I am working on Kubernetes application where I am running RabbitMQ.. Kubernetes deployment.yaml has a Volume attached, but during container restarts, messages are getting lost. Even though messages are marked as persistent and queues are durable.

deployment.yaml :

apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  labels:
    app.kubernetes.io/name: rabbitmq-develop
    app.kubernetes.io/instance: rabbitmq-develop-instance
    app.kubernetes.io/version: '1.0.0'
    app.kubernetes.io/managed-by: kubectl
  name: rabbitmq-deployment
spec:
  replicas: 3
  strategy:
     type: RollingUpdate
     rollingUpdate:
        maxUnavailable: 25%
        maxSurge: 1
  selector:
    matchLabels: 
      app: rabbitmq-develop
  template: 
    metadata:
      labels: 
        app: rabbitmq-develop
    spec:
      containers:
        - image: IMAGE_LOCATION
          imagePullPolicy: Always
          name: rabbitmq-develop
          ports: 
            - containerPort: 80
            - containerPort: 443
            - containerPort: 5672
            - containerPort: 15672
            - containerPort: 4369
            - containerPort: 25672
      volumes:
      - name: mysqlvol
        persistentVolumeClaim:
          claimName: mysqlvol
(END)

volume details :

apiVersion: v1
kind: PersistentVolume
metadata:
  name: mysqlvol
spec:
  storageClassName: manual
  capacity:
    storage: 2Gi #Size of the volume
  accessModes:
    - ReadWriteOnce #type of access
  hostPath:
    path: "/var/lib/rabbitmq/" #host location

---

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: mysqlvol
spec:
  storageClassName: manual
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 2Gi

DockerFile :

FROM rabbitmq:3.8.2

MAINTAINER ME

RUN touch /tmp/firsttimerunning

ENV RABBITMQ_DATA_DIR=/var/lib/rabbitmq/

ENV HOME $RABBITMQ_DATA_DIR

VOLUME $RABBITMQ_DATA_DIR

RUN apt-get update && apt-get install -y curl

RUN apt-get update && apt-get install -y curl

# Expose ports inside the container to the host
EXPOSE 4369
EXPOSE 5672
EXPOSE 5671
EXPOSE 15672
EXPOSE 25672

ADD startclusternode.sh /opt/rabbit/

RUN chmod 777 /opt/rabbit/startclusternode.sh

RUN apt-get update && apt-get install -y curl


# Set ownership permissions on files in the container
#RUN chown rabbitmq:rabbitmq /var/lib/rabbitmq/.erlang.cookie

# Expose ports inside the container to the host
EXPOSE 4369
EXPOSE 5672
EXPOSE 5671
EXPOSE 15672
EXPOSE 25672

ADD startclusternode.sh /opt/rabbit/

RUN chmod 777 /opt/rabbit/startclusternode.sh

startclusternode.sh :

#!/bin/bash

logfile="/var/log/rabbitmq/rabbitnode.log"
firsttimefile="/tmp/firsttimerunning"

curhostname=""
username=""
password=""
echo "" > $logfile
echo "New Start Date:" >> $logfile
date >> $logfile
echo "" >> $logfile

 # echo "Starting Clustered Server Instance" >> $logfile
  # if not clustered then start it normally as if it is a single server
  rabbitmq-server -detached
  sleep 10
  rabbitmqctl start_app &
# sleep 10
  echo "Done Starting Clustered Server Instance" >> $logfile
  # Tail to keep the foreground process active.
#  tail -f /var/log/rabbitmq/*

echo "adding users now"
rabbitmqctl add_user $username $password; \
rabbitmqctl set_user_tags $username administrator; \
rabbitmqctl add_vhost $curhostname; \
rabbitmqctl set_permissions -p $curhostname $username ".*" ".*" ".*" ; \
rabbitmqctl set_policy ha-all "" '{"ha-mode":"all","ha-sync-mode":"automatic"}'
echo "user addition complete"
# For version 3.5.6 the first time running the cluster needs to enable the plugins
if [ -f $firsttimefile ]; then
  echo "Enabling Plugins" >> $logfile
  rabbitmq-plugins enable rabbitmq_management  rabbitmq_management_agent rabbitmq_federation rabbitmq_federation_management >> $logfile
  echo "Waiting for Plugins to finish" >> $logfile
  sleep 1

  echo "Done First Time Running Enabling Plugins" >> $logfile
  rm -f $firsttimefile >> $logfile
  echo "Done Cleanup First Time File" >> $logfile
fi

curl -i -u $username:$password -H "content-type:application/json" \
-XPUT -d'{"durable":true}' \
http://127.0.0.1:15672/api/queues/$curhostname/OUR_QUEUE
/// 3 more such queues


echo "in last step"
tail -f /var/log/rabbitmq/rabbitnode.log

Apologies for lot of files, dunno where exactly the problem is. What am I doing wrong? Thank you. :-)