I have noticed that when I delete and then rebuild my redis pod in kubernetes, the previously scheduled Celery tasks do not persist and are lost. I can see this in flower and also just by manually going into the pod. This is strange as I think I have configured my redis pod to persist with both AOF and RDB. I can see the .aof and the dump file increase in size frequently so I do think data is being saved as appropriate, but maybe the connection between celery and redis is off or something.

Is there something wrong with my setup? I have included my .yaml files for redis, celery, celery-beat, flower, backend and my backend config.

redis.yaml:

apiVersion: v1
kind: Service
metadata:
  name: redis-service
spec:
  type: NodePort
  selector:
    type: redis
  ports:
    - port: 6379
      targetPort: 6379
      protocol: TCP
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: redis-statefulset
  labels:
    type: redis
spec:
  selector:
    matchLabels:
      type: redis
  serviceName: redis
  replicas: 1
  template:
    metadata:
      labels:
        type: redis
    spec:
      containers:
        - name: redis
          image: redis:6.0
          args: ["--appendonly", "yes", "--save", "900", "1", "--save", "30", "1"]
          ports:
            - containerPort: 6379
          volumeMounts:
            - name: redis-volume
              mountPath: /data
      volumes:
        - name: redis-volume
          gcePersistentDisk:
            pdName: redis-staging-persist
            fsType: ext4

celery.yaml

apiVersion: v1
kind: Service
metadata:
  name: celery-service
spec:
  type: ClusterIP
  selector:
    type: celery
  ports:
  - protocol: TCP
    port: 8000
    targetPort: 8000
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: celery-deployment
  labels:
    type: celery
spec:
  replicas: 2
  selector:
    matchLabels:
      type: celery
  template:
    metadata:
      labels:
        type: celery
    spec:
      volumes:
        - name: storage-credentials-volume
          secret:
            secretName: storage-credentials
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            podAffinityTerm:
              labelSelector:
                matchExpressions:
                - key: type
                  operator: In
                  values:
                  - celery
                  - backend
              topologyKey: "kubernetes.io/hostname"
      containers:
        - name: celery
          imagePullPolicy: "Always"
          image: gcr.io/app-123456/staging/backend:latest
          command: ["celery", "-A", "app", "worker", "-l", "info"]
          volumeMounts:
            - mountPath: /storage-credentials
              name: storage-credentials-volume
              readOnly: true
          livenessProbe:
            exec:
              command: ["celery", "-A", "app", "inspect", "ping"]
            initialDelaySeconds: 30
            periodSeconds: 60
            timeoutSeconds: 15
          readinessProbe:
            exec:
              command: ["celery", "-A", "app", "inspect", "ping"]
            initialDelaySeconds: 20
            periodSeconds: 10
          envFrom:
            - secretRef:
                name: db-secrets
            - configMapRef:
                name: backend-config
          env:
            - name: GOOGLE_APPLICATION_CREDENTIALS
              value: /storage-credentials/storage-credentials.json
            - name: EXPORT_EXPIRY_IN_DAYS
              value: "1"

celery-beat.yaml

apiVersion: v1
kind: Service
metadata:
  name: celery-beat-service
spec:
  type: ClusterIP
  selector:
    type: celery-beat
  ports:
  - protocol: TCP
    port: 8000
    targetPort: 8000
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: celery-beat-deployment
  labels:
    type: celery-beat
spec:
  replicas: 1
  selector:
    matchLabels:
      type: celery-beat
  template:
    metadata:
      labels:
        type: celery-beat
    spec:
      volumes:
        - name: storage-credentials-volume
          secret:
            secretName: storage-credentials
      containers:
        - name: celery-beat
          imagePullPolicy: "Always"
          image: gcr.io/app-123456/staging/backend:latest
          command: ["celery", "-A", "app", "beat", "-l", "info"]
          volumeMounts:
            - mountPath: /storage-credentials
              name: storage-credentials-volume
              readOnly: true
          livenessProbe:
            exec:
              command: ["celery", "-A", "app", "inspect", "ping"]
            initialDelaySeconds: 30
            periodSeconds: 60
            timeoutSeconds: 15
          readinessProbe:
            exec:
              command: ["celery", "-A", "app", "inspect", "ping"]
            initialDelaySeconds: 20
            periodSeconds: 10
          envFrom:
            - secretRef:
                name: db-secrets
            - configMapRef:
                name: backend-config
          env:
            - name: GOOGLE_APPLICATION_CREDENTIALS
              value: /storage-credentials/storage-credentials.json
            - name: EXPORT_EXPIRY_IN_DAYS
              value: "1"

flower.yaml

apiVersion: v1
kind: Service
metadata:
  name: flower-service
spec:
  type: NodePort
  selector:
    type: flower
  ports:
  - protocol: TCP
    port: 8888
    targetPort: 8888
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: flower-deployment
  labels:
    type: flower
spec:
  replicas: 1
  selector:
    matchLabels:
      type: flower
  template:
    metadata:
      labels:
        type: flower
    spec:
      volumes:
        - name: storage-credentials-volume
          secret:
            secretName: storage-credentials
      containers:
        - name: flower
          image: mher/flower:0.9.4
          command: [
            "celery",
            "flower",
            "--basic_auth=$(FLOWER_USER):$(FLOWER_PASSWORD)",
            "--broker=redis://redis-service:6379/0",
            "--port=8888",
            "--url-prefix=flower"
          ]
          ports:
            - containerPort: 8888
          volumeMounts:
            - mountPath: /storage-credentials
              name: storage-credentials-volume
              readOnly: true
          envFrom:
            - secretRef:
                name: flower-secrets

backend.yaml

apiVersion: v1
kind: Service
metadata:
  name: backend-service
spec:
  type: NodePort
  selector:
    type: backend
  ports:
  - protocol: TCP
    port: 8000
    targetPort: 8000
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: backend-deployment
  labels:
    type: backend
spec:
  replicas: 2
  selector:
    matchLabels:
      type: backend
  template:
    metadata:
      labels:
        type: backend
    spec:
      volumes:
        - name: storage-credentials-volume
          secret:
            secretName: storage-credentials
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            podAffinityTerm:
              labelSelector:
                matchExpressions:
                - key: type
                  operator: In
                  values:
                  - backend
                  - celery
              topologyKey: "kubernetes.io/hostname"
      containers:
        - name: backend
          imagePullPolicy: "Always"
          command: ["/bin/sh", "-c", "python manage.py migrate &&
                                      python manage.py collectstatic --noinput &&
                                      gunicorn app.wsgi -b 0.0.0.0:8000"
          ]
          image: gcr.io/app-123456/staging/backend:latest
          volumeMounts:
            - mountPath: /storage-credentials
              name: storage-credentials-volume
              readOnly: true
          ports:
            - containerPort: 8000
          livenessProbe:
            httpGet:
              port: 8000
              path: "/"
              httpHeaders:
                - name: Host
                  value: localhost
            initialDelaySeconds: 20
            periodSeconds: 5
          readinessProbe:
            httpGet:
              port: 8000
              path: "/"
              httpHeaders:
                - name: Host
                  value: localhost
          envFrom:
            - secretRef:
                name: db-secrets
            - configMapRef:
                name: backend-config
          env:
            - name: GOOGLE_APPLICATION_CREDENTIALS
              value: /storage-credentials/storage-credentials.json
            - name: EXPORT_THRESHOLD
              value: "10"

backend_config.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: backend-config
data:

...

 REDIS_HOST: "redis-service"

...

Cheers!