I'm trying to deploy and ELK stack on AKS that will take messages from RabbitMQ and ultimately end up in Kibana. To do this I'm using the Elastic operator via

kubectl apply -f https://download.elastic.co/downloads/eck/1.3.0/all-in-one.yaml

Everything is working except the connection between Logstash and Elasticsearch. I can log in to Kibana, I can get the default Elasticsearch message in the browser, all the logs look fine so I think the issue lies in the logstash configuration. My configuration is at the end of the question, you can see I'm using secrets to get the various passwords and access the public certificates to make the https work.

Most confusingly, I can bash into the running logstash pod and with the exact same certificate run

curl --cacert /etc/logstash/certificates/tls.crt -u elastic:<redacted-password> https://rabt-db-es-http:9200

This gives me the response:

{
  "name" : "rabt-db-es-default-0",
  "cluster_name" : "rabt-db",
  "cluster_uuid" : "9YoWLsnMTwq5Yor1ak2JGw",
  "version" : {
    "number" : "7.10.0",
    "build_flavor" : "default",
    "build_type" : "docker",
    "build_hash" : "51e9d6f22758d0374a0f3f5c6e8f3a7997850f96",
    "build_date" : "2020-11-09T21:30:33.964949Z",
    "build_snapshot" : false,
    "lucene_version" : "8.7.0",
    "minimum_wire_compatibility_version" : "6.8.0",
    "minimum_index_compatibility_version" : "6.0.0-beta1"
  },
  "tagline" : "You Know, for Search"
}

To me, this verifies that I the pod can communicate with the database, and has the correct user, password and certificates in place to do it securely. Why then does this fail using a logstash conf file?

The error from the logstash end is

[WARN ] 2021-01-14 15:24:38.360 [Ruby-0-Thread-6: /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.6.2-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:241] elasticsearch - Attempted to resurrect connection to dead ES instance, but got an error. {:url=>"https://rabt-db-es-http:9200/", :error_type=>LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError, :error=>"Got response code '401' contacting Elasticsearch at URL 'https://rabt-db-es-http:9200/'"}

From Elasticsearch I can see the failed requests as

{"type": "server", "timestamp": "2021-01-14T15:36:13,725Z", "level": "WARN", "component": "o.e.x.s.t.n.SecurityNetty4HttpServerTransport", "cluster.name": "rabt-db", "node.name": "rabt-db-es-default-0", "message": "http client did not trust this server's certificate, closing connection Netty4HttpChannel{localAddress=/10.244.0.30:9200, remoteAddress=/10.244.0.15:37766}", "cluster.uuid": "9YoWLsnMTwq5Yor1ak2JGw", "node.id": "9w3fXZBZQGeBMeFYGqYUXw"  }

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: logstash-config
  labels:
    app.kubernetes.io/name: rabt
    app.kubernetes.io/component: logstash
data:
  logstash.yml: |
    http.host: "0.0.0.0"
    path.config: /usr/share/logstash/pipeline
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: logstash-pipeline
  labels:
    app.kubernetes.io/name: rabt
    app.kubernetes.io/component: logstash
data:
  logstash.conf: |
    input {

      rabbitmq {
        host => "rabt-mq"
        port => 5672
        durable => true
        queue => "rabt-rainfall-queue"
        exchange => "rabt-rainfall-exchange"
        exchange_type => "direct"
        heartbeat => 30
        durable => true
        user => "${RMQ_USERNAME}"
        password => "${RMQ_PASSWORD}"
      }

      file {
        path => "/usr/share/logstash/config/intensity.csv"
        start_position => "beginning"
        codec => plain {
            charset => "ISO-8859-1"
        }
        type => "intensity"
      }
    }

    filter {
      csv {
        separator => ","
        columns => ["Duration", "Intensity"]
      }
    }

    output {
      if [type] == "rainfall" {
        elasticsearch {
          hosts => [ "${ES_HOSTS}" ]
          ssl => true
          cacert => "/etc/logstash/certificates/tls.crt"
          index => "rabt-rainfall-%{+YYYY.MM.dd}"
        }	
      }
      else if[type] == "intensity"{
        elasticsearch {
          hosts => [ "${ES_HOSTS}" ]
          ssl => true
          cacert => "/etc/logstash/certificates/tls.crt"
          index => "intensity-%{+YYYY.MM.dd}"
        }	
      }
    }
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: rainfall-intensity-threshold
  labels:
    app.kubernetes.io/name: rabt
    app.kubernetes.io/component: logstash
data:
  intensity.csv: |
    Duration,Intensity
    0.1,7.18941593
    0.2,6.34611898
    0.3,5.89945352
    0.4,5.60173824
    0.5,5.38119846
    0.6,5.20746530
    0.7,5.06495933
    0.8,4.94467113
    0.9,4.84094288
    1,4.75000000
    2,4.19283923
    3,3.89773029
    4,3.70103175
    5,3.55532256
    6,3.44053820
    7,3.34638544
    8,3.26691182
    9,3.19837924
    10,3.13829388
    20,2.77018141
    30,2.57520486
    40,2.44524743
    50,2.34897832
    60,2.27314105
    70,2.21093494
    80,2.15842723
    90,2.11314821
    100,2.07345020 
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: rabt-logstash
  labels:
    app.kubernetes.io/name: rabt
    app.kubernetes.io/component: logstash
spec:
  replicas: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: rabt
      app.kubernetes.io/component: logstash
  template:
    metadata:
      labels:
        app.kubernetes.io/name: rabt
        app.kubernetes.io/component: logstash
    spec:
      containers:
        - name: logstash
          image: docker.elastic.co/logstash/logstash:7.9.2
          ports:
            - name: "tcp-beats"
              containerPort: 5044
          env:
            - name: ES_HOSTS
              value: "https://rabt-db-es-http:9200"
            - name: ES_USER
              value: "elastic"
            - name: ES_PASSWORD
              valueFrom:
                secretKeyRef:
                  name: rabt-db-es-elastic-user
                  key: elastic
            - name: RMQ_USERNAME
              valueFrom:
                secretKeyRef:
                  name: rabt-mq-default-user
                  key: username
            - name: RMQ_PASSWORD
              valueFrom:
                secretKeyRef:
                  name: rabt-mq-default-user
                  key: password
          volumeMounts:
            - name: config-volume
              mountPath: /usr/share/logstash/config
            - name: pipeline-volume
              mountPath: /usr/share/logstash/pipeline
            - name: ca-certs
              mountPath: /etc/logstash/certificates
              readOnly: true
      volumes:
        - name: config-volume
          projected:
            sources:
            - configMap:
                name: logstash-config
            - configMap:
                name: rainfall-intensity-threshold
        - name: pipeline-volume
          configMap:
            name: logstash-pipeline
        - name: ca-certs
          secret:
            secretName: rabt-db-es-http-certs-public
---
apiVersion: v1
kind: Service
metadata:
  name: rabt-logstash
  labels:
    app.kubernetes.io/name: rabt
    app.kubernetes.io/component: logstash
spec:
  ports:
    - name: "tcp-beats"
      port: 5044
      targetPort: 5044
  selector:
    app.kubernetes.io/name: rabt
    app.kubernetes.io/component: logstash