We have deployed MongoDB(3 replicas) on the Kubernetes and configured replication manually using the below commands.

    rs0:PRIMARY> rs.config()
    rs0:PRIMARY> var cfg = rs.conf()
    rs0:PRIMARY> cfg.members[0].host="mongo-0.mongo:27017"
    rs0:PRIMARY> rs.reconfig(cfg)
    rs0:PRIMARY> rs.status()
    rs0:PRIMARY> rs.add("mongo-1.mongo:27017")
    rs0:PRIMARY> rs.add("mongo-2.mongo:27017")
    rs0:PRIMARY> rs.status()

MongoDB was working fine and today all of a sudden, ReplicaSet is broken. Two pods are in "rs0:OTHER>" status and one pod is in "rs0:SECONDARY>" state. Please find rs.status() command output here.

rs0:OTHER> rs.status()
{
        "state" : 10,
        "stateStr" : "REMOVED",
        "uptime" : 2045,
        "optime" : {
                "ts" : Timestamp(1585535836, 1),
                "t" : NumberLong(124)
        },
        "optimeDate" : ISODate("2020-03-30T02:37:16Z"),
        "lastHeartbeatMessage" : "",
        "syncingTo" : "",
        "syncSourceHost" : "",
        "syncSourceId" : -1,
        "infoMessage" : "",
        "ok" : 0,
        "errmsg" : "Our replica set config is invalid or we are not a member of it",
        "code" : 93,
        "codeName" : "InvalidReplicaSetConfig"
}
rs0:OTHER>

rs0:SECONDARY> rs.status()
{
        "set" : "rs0",
        "date" : ISODate("2020-03-30T04:52:29.952Z"),
        "myState" : 2,
        "term" : NumberLong(124),
        "syncingTo" : "",
        "syncSourceHost" : "",
        "syncSourceId" : -1,
        "heartbeatIntervalMillis" : NumberLong(2000),
        "optimes" : {
                "lastCommittedOpTime" : {
                        "ts" : Timestamp(0, 0),
                        "t" : NumberLong(-1)
                },
                "appliedOpTime" : {
                        "ts" : Timestamp(1585422009, 1),
                        "t" : NumberLong(108)
                },
                "durableOpTime" : {
                        "ts" : Timestamp(1585422009, 1),
                        "t" : NumberLong(108)
                }
        },
        "members" : [
                {
                        "_id" : 0,
                        "name" : "mongo-0.mongo:27017",
                        "health" : 0,
                        "state" : 8,
                        "stateStr" : "(not reachable/healthy)",
                        "uptime" : 0,
                        "optime" : {
                                "ts" : Timestamp(0, 0),
                                "t" : NumberLong(-1)
                        },
                        "optimeDurable" : {
                                "ts" : Timestamp(0, 0),
                                "t" : NumberLong(-1)
                        },
                        "optimeDate" : ISODate("1970-01-01T00:00:00Z"),
                        "optimeDurableDate" : ISODate("1970-01-01T00:00:00Z"),
                        "lastHeartbeat" : ISODate("2020-03-30T04:52:25.376Z"),
                        "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"),
                        "pingMs" : NumberLong(0),
                        "lastHeartbeatMessage" : "Our replica set configuration is invalid or does not include us",
                        "syncingTo" : "",
                        "syncSourceHost" : "",
                        "syncSourceId" : -1,
                        "infoMessage" : "",
                        "configVersion" : -1
                },
                {
                        "_id" : 1,
                        "name" : "mongo-1.mongo:27017",
                        "health" : 0,
                        "state" : 8,
                        "stateStr" : "(not reachable/healthy)",
                        "uptime" : 0,
                        "optime" : {
                                "ts" : Timestamp(0, 0),
                                "t" : NumberLong(-1)
                        },
                        "optimeDurable" : {
                                "ts" : Timestamp(0, 0),
                                "t" : NumberLong(-1)
                        },
                        "optimeDate" : ISODate("1970-01-01T00:00:00Z"),
                        "optimeDurableDate" : ISODate("1970-01-01T00:00:00Z"),
                        "lastHeartbeat" : ISODate("2020-03-30T04:52:25.332Z"),
                        "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"),
                        "pingMs" : NumberLong(0),
                        "lastHeartbeatMessage" : "Our replica set configuration is invalid or does not include us",
                        "syncingTo" : "",
                        "syncSourceHost" : "",
                        "syncSourceId" : -1,
                        "infoMessage" : "",
                        "configVersion" : -1
                },
                {
                        "_id" : 2,
                        "name" : "mongo-2.mongo:27017",
                        "health" : 1,
                        "state" : 2,
                        "stateStr" : "SECONDARY",
                        "uptime" : 3448,
                        "optime" : {
                                "ts" : Timestamp(1585422009, 1),
                                "t" : NumberLong(108)
                        },
                        "optimeDate" : ISODate("2020-03-28T19:00:09Z"),
                        "syncingTo" : "",
                        "syncSourceHost" : "",
                        "syncSourceId" : -1,
                        "infoMessage" : "",
                        "configVersion" : 100696,
                        "self" : true,
                        "lastHeartbeatMessage" : ""
                }
        ],
        "ok" : 1
}
rs0:SECONDARY>

Please find my Deployment file here

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: mongo
spec:
  selector:
    matchLabels:
      app: mongo
  serviceName: "mongo"
  replicas: 3
  template:
    metadata:
      labels:
        app: mongo
    spec:
      terminationGracePeriodSeconds: 10
      containers:
      - name: mongo
        image: mongo:3.4.23
        command:
        - mongod
        - "--bind_ip"
        - "0.0.0.0"
        - "--replSet"
        - rs0
        - "--smallfiles"
        - "--noprealloc"
        ports:
        - containerPort: 27017
        resources:
          requests:
           cpu: 1
           memory: 2Gi
        volumeMounts:
        - name: mongo-persistent-storage
          mountPath: /data/db
  volumeClaimTemplates:
  - metadata:
      name: mongo-persistent-storage
    spec:
      accessModes: [ "ReadWriteOnce" ]
      resources:
        requests:
          storage: 1Gi

Could you please help me to resolve an issue.