So I think I sort of have this partly working. However, while I think it's collecting the logs, it does not seem to be saving the output to where I have told it to go.

I have built my own docker image from <code>fluent/fluentd:v1.11.5-debian-1.0</code> This image is built with Azure Blob Storage plugins added, while I have tried to save the output there its not loading. However, I have also done a test to output the logs to a file, which does not seem to be created?

I have tried this locally in a VM with K3s and on my test Azure K8s Cluster, with the same result, no output.

With the configmap, I only have the file option loaded, but when going to that location, there is no logs! But in <code>/var/logs/containers</code> I can see all the logs for all the running pods.

So what have I not done?

This is the daemonset config file

apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: fluentd
  namespace: fluentd
  labels:
    k8s-app: fluentd-logging
    version: v1
spec:
  selector:
    matchLabels:
      k8s-app: fluentd-logging
      version: v1
  template:
    metadata:
      labels:
        k8s-app: fluentd-logging
        version: v1
    spec:
      serviceAccount: fluentd
      serviceAccountName: fluentd
      tolerations:
      - key: node-role.kubernetes.io/master
        effect: NoSchedule
      containers:
      - name: fluentd-azureblob
        image: my-image-repo/fluentd:v2
        imagePullPolicy: Always
        env:
          - name:  AZUREBLOB_ACCOUNT_NAME
            value: "xxx"
          - name:  AZUREBLOB_ACCOUNT_KEY
            value: "xxx"
          - name:  AZUREBLOB_CONNECTION_STRING
            value: "xxxx"
          - name:  AZUREBLOB_CONTAINER
            value: "xxx"
          - name:  AZUREBLOB_TOKEN_REFRESH_INTERVAL
            value: "xxx"
        resources:
          limits:
            memory: 200Mi
          requests:
            cpu: 100m
            memory: 200Mi
        volumeMounts:
        - name: varlog
          mountPath: /var/log
        - name: varlibdockercontainers
          mountPath: /var/lib/docker/containers
          readOnly: true
      imagePullSecrets:
      - name: crlogin
      terminationGracePeriodSeconds: 30
      volumes:
      - name: fluentd-config
        configMap:
          name: fluentd-config
      - name: varlog
        hostPath:
          path: /var/log
      - name: varlibdockercontainers
        hostPath:
          path: /var/lib/docker/containers

Service Roles:

apiVersion: v1
kind: ServiceAccount
metadata:
  name: fluentd
  namespace: fluentd
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: fluentd
  namespace: fluentd
rules:
- apiGroups:
  - ""
  resources:
  - pods
  - namespaces
  verbs:
  - get
  - list
  - watch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: fluentd
roleRef:
  kind: ClusterRole
  name: fluentd
  apiGroup: rbac.authorization.k8s.io
subjects:
- kind: ServiceAccount
  name: fluentd
  namespace: fluentd

The ConfigMap:

apiVersion: v1
kind: ConfigMap
metadata:
  name: fluentd-config
  namespace: fluentd
data:
  fluent.conf: |-
    @include pods-fluent.conf
    @include file-fluent.conf
    #@include blob-fluent.conf
  pods-fluent.conf: |-
    <source>
      @type tail
      read_from_head true
      tag kubernetes.*
      path /var/log/containers/*.log
      pos_file /var/log/fluentd-containers.log.pos
      exclude_path ["/var/log/containers/fluent*"]
      <parse>
        @type kubernetes
        @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}"
        time_format %Y-%m-%dT%H:%M:%S.%NZ
      </parse>
    </source>

    <filter kubernetes.**>
      @type kubernetes_metadata
      @id filter_kube_metadata
      kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || 'https://' + ENV.fetch('KUBERNETES_SERVICE_HOST') + ':' + ENV.fetch('KUBERNETES_SERVICE_PORT') + '/api'}"
      verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}"
      ca_file "#{ENV['KUBERNETES_CA_FILE']}"
      skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}"
      skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}"
      skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}"
      skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}"
    </filter>
  file-fluent.conf: |-
    <match **>
      @type file
      path /var/tmp/test.log
    </match>
  blob-fluent.conf: |-
    <match **>
      @type azure-storage-append-blob
      @id out_azure_storage_append_blob

      azure_storage_account           "#{ENV['AZUREBLOB_ACCOUNT_NAME']}"
      azure_storage_access_key        "#{ENV['AZUREBLOB_ACCOUNT_KEY']}"
      azure_storage_connection_string "#{ENV['AZUREBLOB_CONNECTION_STRING']}"
      azure_storage_sas_token         "#{ENV['AZUREBLOB_SAS_TOKEN']}"
      azure_container                 "#{ENV['AZUREBLOB_CONTAINER']}"
      azure_imds_api_version          "#{ENV['AZUREBLOB_IMDS_API_VERSION']}"
      azure_token_refresh_interval    "#{ENV['AZUREBLOB_TOKEN_REFRESH_INTERVAL']}"
      auto_create_container           true
      path                            "#{ENV['AZUREBLOB_LOG_PATH']}"
      azure_object_key_format         %{path}%{time_slice}_%{index}.log
      time_slice_format               %Y%m%d-%H

      <buffer>
        @type file
        path /var/log/fluent/azurestorageappendblob
        timekey 60 # 1 minute
        timekey_wait 60
        timekey_use_utc true # use utc
        chunk_limit_size 256m
      </buffer>
    </match>