Elasticsearch pods failing at readiness probe

12/19/2021

Elasticsearch pod is not becoming active.

logging-es-data-master-ilmz5zyt-3-deploy   1/1       Running   0          5m
logging-es-data-master-ilmz5zyt-3-qxkml    1/2       Running   0          5m

and events are.

Events:
  Type     Reason     Age                From                     Message
  ----     ------     ----               ----                     -------
  Normal   Scheduled  5m                 default-scheduler        Successfully assigned logging-es-data-master-ilmz5zyt-3-qxkml to digi-srv-pp-01
  Normal   Pulled     5m                 kubelet, digi-srv-pp-01  Container image "docker.io/openshift/origin-logging-elasticsearch:v3.10" already present on machine
  Normal   Created    5m                 kubelet, digi-srv-pp-01  Created container
  Normal   Started    5m                 kubelet, digi-srv-pp-01  Started container
  Normal   Pulled     5m                 kubelet, digi-srv-pp-01  Container image "docker.io/openshift/oauth-proxy:v1.0.0" already present on machine
  Normal   Created    5m                 kubelet, digi-srv-pp-01  Created container
  Normal   Started    5m                 kubelet, digi-srv-pp-01  Started container
  Warning  Unhealthy  13s (x55 over 4m)  kubelet, digi-srv-pp-01  Readiness probe failed: Elasticsearch node is not ready to accept HTTP requests yet [response code: 000]

Deployment config is

# oc export dc/logging-es-data-master-ilmz5zyt -o yaml
Command "export" is deprecated, use the oc get --export
apiVersion: v1
kind: DeploymentConfig
metadata:
  creationTimestamp: null
  generation: 5
  labels:
    component: es
    deployment: logging-es-data-master-ilmz5zyt
    logging-infra: elasticsearch
    provider: openshift
  name: logging-es-data-master-ilmz5zyt
spec:
  replicas: 1
  revisionHistoryLimit: 0
  selector:
    component: es
    deployment: logging-es-data-master-ilmz5zyt
    logging-infra: elasticsearch
    provider: openshift
  strategy:
    activeDeadlineSeconds: 21600
    recreateParams:
      timeoutSeconds: 600
    resources: {}
    type: Recreate
  template:
    metadata:
      creationTimestamp: null
      labels:
        component: es
        deployment: logging-es-data-master-ilmz5zyt
        logging-infra: elasticsearch
        provider: openshift
      name: logging-es-data-master-ilmz5zyt
    spec:
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - podAffinityTerm:
              labelSelector:
                matchExpressions:
                - key: logging-infra
                  operator: In
                  values:
                  - elasticsearch
              topologyKey: kubernetes.io/hostname
            weight: 100
      containers:
      - env:
        - name: DC_NAME
          value: logging-es-data-master-ilmz5zyt
        - name: NAMESPACE
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: metadata.namespace
        - name: KUBERNETES_TRUST_CERTIFICATES
          value: "true"
        - name: SERVICE_DNS
          value: logging-es-cluster
        - name: CLUSTER_NAME
          value: logging-es
        - name: INSTANCE_RAM
          value: 12Gi
        - name: HEAP_DUMP_LOCATION
          value: /elasticsearch/persistent/heapdump.hprof
        - name: NODE_QUORUM
          value: "1"
        - name: RECOVER_EXPECTED_NODES
          value: "1"
        - name: RECOVER_AFTER_TIME
          value: 5m
        - name: READINESS_PROBE_TIMEOUT
          value: "30"
        - name: POD_LABEL
          value: component=es
        - name: IS_MASTER
          value: "true"
        - name: HAS_DATA
          value: "true"
        - name: PROMETHEUS_USER
          value: system:serviceaccount:openshift-metrics:prometheus
        image: docker.io/openshift/origin-logging-elasticsearch:v3.10
        imagePullPolicy: IfNotPresent
        name: elasticsearch
        ports:
        - containerPort: 9200
          name: restapi
          protocol: TCP
        - containerPort: 9300
          name: cluster
          protocol: TCP
        readinessProbe:
          exec:
            command:
            - /usr/share/java/elasticsearch/probe/readiness.sh
          failureThreshold: 3
          initialDelaySeconds: 30
          periodSeconds: 5
          successThreshold: 1
          timeoutSeconds: 120
        resources:
          limits:
            memory: 12Gi
          requests:
            cpu: "1"
            memory: 12Gi
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
        volumeMounts:
        - mountPath: /etc/elasticsearch/secret
          name: elasticsearch
          readOnly: true
        - mountPath: /usr/share/java/elasticsearch/config
          name: elasticsearch-config
          readOnly: true
        - mountPath: /elasticsearch/persistent
          name: elasticsearch-storage
      - args:
        - --upstream-ca=/etc/elasticsearch/secret/admin-ca
        - --https-address=:4443
        - -provider=openshift
        - -client-id=system:serviceaccount:openshift-logging:aggregated-logging-elasticsearch
        - -client-secret-file=/var/run/secrets/kubernetes.io/serviceaccount/token
        - -cookie-secret=endzaVczSWMzb0NoNlVtVw==
        - -basic-auth-password=NXd9xTjg4npjIM0E
        - -upstream=https://localhost:9200
        - '-openshift-sar={"namespace": "openshift-logging", "verb": "view", "resource":
          "prometheus", "group": "metrics.openshift.io"}'
        - '-openshift-delegate-urls={"/": {"resource": "prometheus", "verb": "view",
          "group": "metrics.openshift.io", "namespace": "openshift-logging"}}'
        - --tls-cert=/etc/tls/private/tls.crt
        - --tls-key=/etc/tls/private/tls.key
        - -pass-access-token
        - -pass-user-headers
        image: docker.io/openshift/oauth-proxy:v1.0.0
        imagePullPolicy: IfNotPresent
        name: proxy
        ports:
        - containerPort: 4443
          name: proxy
          protocol: TCP
        resources:
          limits:
            memory: 64Mi
          requests:
            cpu: 100m
            memory: 64Mi
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
        volumeMounts:
        - mountPath: /etc/tls/private
          name: proxy-tls
          readOnly: true
        - mountPath: /etc/elasticsearch/secret
          name: elasticsearch
          readOnly: true
      dnsPolicy: ClusterFirst
      nodeSelector:
        node-role.kubernetes.io/compute: "true"
      restartPolicy: Always
      schedulerName: default-scheduler
      securityContext:
        supplementalGroups:
        - 65534
      serviceAccount: aggregated-logging-elasticsearch
      serviceAccountName: aggregated-logging-elasticsearch
      terminationGracePeriodSeconds: 30
      volumes:
      - name: proxy-tls
        secret:
          defaultMode: 420
          secretName: prometheus-tls
      - name: elasticsearch
        secret:
          defaultMode: 420
          secretName: logging-elasticsearch
      - configMap:
          defaultMode: 420
          name: logging-elasticsearch
        name: elasticsearch-config
      - name: elasticsearch-storage
        persistentVolumeClaim:
          claimName: logging-es-0
  test: false
  triggers: []
status:
  availableReplicas: 0
  latestVersion: 0
  observedGeneration: 0
  replicas: 0
  unavailableReplicas: 0
  updatedReplicas: 0
-- Mohit Kumar
elasticsearch
kubernetes
readinessprobe

0 Answers