With kubernetes, I'm trying to deploy jenkins image & a persistent volume mapped to a NFS share (which is mounted on all my workers)

• So, this is my share on my workers :

[root@pp-tmp-test24 /opt]# df -Th /opt/jenkins.persistent
Filesystem                                        Type  Size  Used Avail Use% Mounted on
xxx.xxx.xxx.xxx:/VR_C_CS003_NFS_KUBERNETESPV_TMP_PP nfs4   10G  9.5M   10G   1% /opt/jenkins.persistent

• And My data on this share

[root@pp-tmp-test24 /opt/jenkins.persistent]# ls -l
total 0
-rwxr-xr-x. 1 root root 0 Oct  2 11:53 newfile

[root@pp-tmp-test24 /opt/jenkins.persistent]# cat newfile
hello

• Here It is my yaml files to deploy it

My PersistentVolume yaml

apiVersion: v1
kind: PersistentVolume
metadata:
  name: jenkins-pv-nfs
  labels:
    type: type-nfs
spec:
  storageClassName: class-nfs
  capacity:
    storage: 10Gi
  volumeMode: Filesystem
  accessModes:
    - ReadWriteMany
  persistentVolumeReclaimPolicy: Recycle
  hostPath:
    path: /opt/jenkins.persistent

My PersistentVolumeClaim yaml

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: jenkins-pvc-nfs
  namespace: ns-jenkins
spec:
  storageClassName: class-nfs
  volumeMode: Filesystem
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 10Gi
  selector:
    matchLabels:
      type: type-nfs

And my deployment

apiVersion: apps/v1
kind: Deployment
metadata:
  name: jenkins
  namespace: ns-jenkins
spec:
  replicas: 1
  selector:
    matchLabels:
      app: jenkins
  template:
    metadata:
      labels:
        app: jenkins
    spec:
      containers:
      - image: jenkins
     #- image: httpd:latest
        name: jenkins
        ports:
        - containerPort: 8080
          protocol: TCP
          name: jenkins-web
        volumeMounts:
        - name: jenkins-persistent-storage
          mountPath: /var/foo
      volumes:
      - name: jenkins-persistent-storage
        persistentVolumeClaim:
          claimName: jenkins-pvc-nfs

• After kubectl create -f command, all is looking good :

# kubectl get pv
NAME             CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM                        STORAGECLASS   REASON   AGE
jenkins-pv-nfs   10Gi       RWX            Recycle          Bound    ns-jenkins/jenkins-pvc-nfs   class-nfs               37s

# kubectl get pvc -A
NAMESPACE    NAME              STATUS   VOLUME           CAPACITY   ACCESS MODES   STORAGECLASS   AGE
ns-jenkins   jenkins-pvc-nfs   Bound    jenkins-pv-nfs   10Gi       RWX            class-nfs      35s

# kubectl get pods -A |grep jenkins
ns-jenkins             jenkins-5bdb8678c-x6vht                                                  1/1     Running   0          14s

# kubectl describe pod jenkins-5bdb8678c-x6vht -n ns-jenkins

Name:           jenkins-5bdb8678c-x6vht
Namespace:      ns-jenkins
Priority:       0
Node:           pp-tmp-test25.mydomain/172.31.68.225
Start Time:     Wed, 02 Oct 2019 11:48:23 +0200
Labels:         app=jenkins
                pod-template-hash=5bdb8678c
Annotations:    <none>
Status:         Running
IP:             10.244.5.47
Controlled By:  ReplicaSet/jenkins-5bdb8678c
Containers:
  jenkins:
    Container ID:   docker://8a3e4871ed64b371818bac59e24d6912e5d2b13c8962c1639d36797fbce8082e
    Image:          jenkins
    Image ID:       docker-pullable://docker.io/jenkins@sha256:eeb4850eb65f2d92500e421b430ed1ec58a7ac909e91f518926e02473904f668
    Port:           8080/TCP
    Host Port:      0/TCP
    State:          Running
      Started:      Wed, 02 Oct 2019 11:48:26 +0200
    Ready:          True
    Restart Count:  0
    Environment:    <none>
    Mounts:
      /var/foo from jenkins-persistent-storage (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from default-token-dz6cd (ro)
Conditions:
  Type              Status
  Initialized       True
  Ready             True
  ContainersReady   True
  PodScheduled      True
Volumes:
  jenkins-persistent-storage:
    Type:       PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
    ClaimName:  jenkins-pvc-nfs
    ReadOnly:   false
  default-token-dz6cd:
    Type:        Secret (a volume populated by a Secret)
    SecretName:  default-token-dz6cd
    Optional:    false
QoS Class:       BestEffort
Node-Selectors:  <none>
Tolerations:     node.kubernetes.io/not-ready:NoExecute for 300s
                 node.kubernetes.io/unreachable:NoExecute for 300s
Events:
  Type    Reason     Age   From                                                     Message
  ----    ------     ----  ----                                                     -------
  Normal  Scheduled  39s   default-scheduler                                        Successfully assigned ns-jenkins/jenkins-5bdb8678c-x6vht to pp-tmp-test25.mydomain
  Normal  Pulling    38s   kubelet, pp-tmp-test25.mydomain  Pulling image "jenkins"
  Normal  Pulled     36s   kubelet, pp-tmp-test25.mydomain  Successfully pulled image "jenkins"
  Normal  Created    36s   kubelet, pp-tmp-test25.mydomain  Created container jenkins
  Normal  Started    36s   kubelet, pp-tmp-test25.mydomain  Started container jenkins

• On my worker, this is my container

# docker ps |grep jenkins
8a3e4871ed64        docker.io/jenkins@sha256:eeb4850eb65f2d92500e421b430ed1ec58a7ac909e91f518926e02473904f668           "/bin/tini -- /usr..."   2 minutes ago       Up 2 minutes                            k8s_jenkins_jenkins-5bdb8678c-x6vht_ns-jenkins_64b66dae-a1da-4d90-83fd-ff433638dc9c_0

So I launch a shell on my container, and I can see my data on /var/foo :

# docker exec -t -i 8a3e4871ed64 /bin/bash


jenkins@jenkins-5bdb8678c-x6vht:/$ df -h /var/foo
Filesystem                                                                                           Size  Used Avail Use% Mounted on
xxx.xxx.xxx.xxx:/VR_C_CS003_NFS_KUBERNETESPV_TMP_PP                                                     10G  9.5M   10G   1% /var/foo


jenkins@jenkins-5bdb8678c-x6vht:/var/foo$ ls -lZ /var/foo -d
drwxr-xr-x. 2 root root system_u:object_r:nfs_t:s0 4096 Oct  2 10:06 /var/foo


jenkins@jenkins-5bdb8678c-x6vht:/var/foo$ ls -lZ /var/foo
-rwxr-xr-x. 1 root root system_u:object_r:nfs_t:s0 12 Oct  2 10:05 newfile

jenkins@jenkins-5bdb8678c-x6vht:/var/foo$ cat newfile
hello

I'm trying to write data in my /var/foo/newfile but the Permission is denied

jenkins@jenkins-5bdb8678c-x6vht:/var/foo$ echo "world" >> newfile
bash: newfile: Permission denied

Same thing in my /var/foo/ directory, I can't write data

jenkins@jenkins-5bdb8678c-x6vht:/var/foo$ touch newfile2
touch: cannot touch 'newfile2': Permission denied

So, I tried an another image like httpd:latest in my deployment yaml (keeping the same name in my yaml definition)

[...]
      containers:
      #- image: jenkins
      - image: httpd:latest
[...]

# docker ps |grep jenkins
fa562400405d        docker.io/httpd@sha256:39d7d9a3ab93c0ad68ee7ea237722ed1b0016ff6974d80581022a53ec1e58797             "httpd-foreground"       50 seconds ago      Up 48 seconds                           k8s_jenkins_jenkins-7894877f96-6dj85_ns-jenkins_540b12bd-69df-44d8-b3df-20a0a96cc851_0

In my new container, this time I can Read-Write data :

root@jenkins-7894877f96-6dj85:/usr/local/apache2# df -h /var/foo
Filesystem                                         Size  Used Avail Use% Mounted on
xxx.xxx.xxx.xxx:/VR_C_CS003_NFS_KUBERNETESPV_TMP_PP   10G  9.6M   10G   1% /var/foo

root@jenkins-7894877f96-6dj85:/var/foo# ls -lZ
total 0
-rwxr-xr-x. 1 root root system_u:object_r:nfs_t:s0 12 Oct  2 10:05 newfile
-rw-r--r--. 1 root root system_u:object_r:nfs_t:s0  0 Oct  2 10:06 newfile2

root@jenkins-7894877f96-6dj85:/var/foo# ls -lZ /var/foo -d
drwxr-xr-x. 2 root root system_u:object_r:nfs_t:s0 4096 Oct  2 10:06 /var/foo


root@jenkins-7894877f96-6dj85:/var/foo# ls -l
total 0
-rwxr-xr-x. 1 root root 6 Oct  2 09:55 newfile

root@jenkins-7894877f96-6dj85:/var/foo# echo "world" >> newfile
root@jenkins-7894877f96-6dj85:/var/foo# touch newfile2
root@jenkins-7894877f96-6dj85:/var/foo# ls -l
total 0
-rwxr-xr-x. 1 root root 12 Oct  2 10:05 newfile
-rw-r--r--. 1 root root  0 Oct  2 10:06 newfile2

What I'm doing wrong ? Does the pb is due to jenkins images who do not allow RW access ? Same pb with a local storage (on my worker) with persistent volume.

Other thing, perhaps it is stupid : with my jenkins image, I would like to mount the /var/jenkins_home dir to a persistent volume in order to keep jenkins's configuration files. But if I try to mount /var/jenkins_home instead of /var/foo, pod is crashinglookbackoff (because there is already data stored in /var/jenkins_home).

thank you all for your help !