I'm trying to setup monitoring for my microservices and I've created a simple microservice just to try out the Prometheus monitoring. But I'm running into issues of prometheus not registering the service.

I'm deploying an example service like so:

apiVersion: apps/v1
kind: Deployment
metadata:
  name: pinger-svc
spec:
  replicas: 3
  selector:
    matchLabels:
      app: pinger-svc
  template:
    metadata:
      labels:
        app: pinger-svc
    spec:
      restartPolicy: Always
      containers:
      - name: pinger-svc
        image: pingerservice:latest
        ports:
          - containerPort: 3000
            protocol: TCP
---
kind: Service
apiVersion: v1
metadata:
  name:  pinger-svc
  labels:
    app: pinger-svc
spec:
  selector:
    app:  pinger-svc
  type:  NodePort
  ports:
  - name: web
    port:  80
    targetPort:  3000
    nodePort: 32333

And trying to setup a service monitor with:

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: prometheus-kube-prometheus-pinger-service
spec:
  selector:
    matchLabels:
      app: pinger-svc
  endpoints:
    - port: web
      path: /metrics
      interval: "10s"

In the app I'm also exposing the metrics in the app at endpoint /metrics:

const promBundle = require('express-prom-bundle');
const metricsMiddleware = promBundle({includeMethod: true});

app.use(metricsMiddleware);

This is all I can see under the targets at the Prometheus endpoint:

Here's my prometheus-operator pod config:

apiVersion: v1
kind: Pod
metadata:
  name: prometheus-kube-prometheus-operator-cb55c97b9-4rh4f
  generateName: prometheus-kube-prometheus-operator-cb55c97b9-
  namespace: default
  uid: c3c710b9-868f-41d9-b35e-7d55a212dd6f
  resourceVersion: '176924'
  creationTimestamp: '2021-09-16T13:34:16Z'
  labels:
    app: kube-prometheus-stack-operator
    app.kubernetes.io/instance: prometheus
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/part-of: kube-prometheus-stack
    app.kubernetes.io/version: 18.0.8
    chart: kube-prometheus-stack-18.0.8
    heritage: Helm
    pod-template-hash: cb55c97b9
    release: prometheus
  ownerReferences:
    - apiVersion: apps/v1
      kind: ReplicaSet
      name: prometheus-kube-prometheus-operator-cb55c97b9
      uid: 62dbdb1e-ed3f-4f6f-a4e6-6da48d084681
      controller: true
      blockOwnerDeletion: true
  managedFields:
    - manager: k3s
      operation: Update
      apiVersion: v1
      time: '2021-09-16T13:34:47Z'
      fieldsType: FieldsV1
      fieldsV1:
        'f:metadata':
          'f:generateName': {}
          'f:labels':
            .: {}
            'f:app': {}
            'f:app.kubernetes.io/instance': {}
            'f:app.kubernetes.io/managed-by': {}
            'f:app.kubernetes.io/part-of': {}
            'f:app.kubernetes.io/version': {}
            'f:chart': {}
            'f:heritage': {}
            'f:pod-template-hash': {}
            'f:release': {}
          'f:ownerReferences':
            .: {}
            'k:{"uid":"62dbdb1e-ed3f-4f6f-a4e6-6da48d084681"}':
              .: {}
              'f:apiVersion': {}
              'f:blockOwnerDeletion': {}
              'f:controller': {}
              'f:kind': {}
              'f:name': {}
              'f:uid': {}
        'f:spec':
          'f:containers':
            'k:{"name":"kube-prometheus-stack"}':
              .: {}
              'f:args': {}
              'f:image': {}
              'f:imagePullPolicy': {}
              'f:name': {}
              'f:ports':
                .: {}
                'k:{"containerPort":10250,"protocol":"TCP"}':
                  .: {}
                  'f:containerPort': {}
                  'f:name': {}
                  'f:protocol': {}
              'f:resources': {}
              'f:securityContext':
                .: {}
                'f:allowPrivilegeEscalation': {}
                'f:readOnlyRootFilesystem': {}
              'f:terminationMessagePath': {}
              'f:terminationMessagePolicy': {}
              'f:volumeMounts':
                .: {}
                'k:{"mountPath":"/cert"}':
                  .: {}
                  'f:mountPath': {}
                  'f:name': {}
                  'f:readOnly': {}
          'f:dnsPolicy': {}
          'f:enableServiceLinks': {}
          'f:restartPolicy': {}
          'f:schedulerName': {}
          'f:securityContext':
            .: {}
            'f:fsGroup': {}
            'f:runAsGroup': {}
            'f:runAsNonRoot': {}
            'f:runAsUser': {}
          'f:serviceAccount': {}
          'f:serviceAccountName': {}
          'f:terminationGracePeriodSeconds': {}
          'f:volumes':
            .: {}
            'k:{"name":"tls-secret"}':
              .: {}
              'f:name': {}
              'f:secret':
                .: {}
                'f:defaultMode': {}
                'f:secretName': {}
        'f:status':
          'f:conditions':
            'k:{"type":"ContainersReady"}':
              .: {}
              'f:lastProbeTime': {}
              'f:lastTransitionTime': {}
              'f:status': {}
              'f:type': {}
            'k:{"type":"Initialized"}':
              .: {}
              'f:lastProbeTime': {}
              'f:lastTransitionTime': {}
              'f:status': {}
              'f:type': {}
            'k:{"type":"Ready"}':
              .: {}
              'f:lastProbeTime': {}
              'f:lastTransitionTime': {}
              'f:status': {}
              'f:type': {}
          'f:containerStatuses': {}
          'f:hostIP': {}
          'f:phase': {}
          'f:podIP': {}
          'f:podIPs':
            .: {}
            'k:{"ip":"10.42.0.53"}':
              .: {}
              'f:ip': {}
          'f:startTime': {}
  selfLink: >-
    /api/v1/namespaces/default/pods/prometheus-kube-prometheus-operator-cb55c97b9-4rh4f
status:
  phase: Running
  conditions:
    - type: Initialized
      status: 'True'
      lastProbeTime: null
      lastTransitionTime: '2021-09-16T13:34:16Z'
    - type: Ready
      status: 'True'
      lastProbeTime: null
      lastTransitionTime: '2021-09-16T13:34:46Z'
    - type: ContainersReady
      status: 'True'
      lastProbeTime: null
      lastTransitionTime: '2021-09-16T13:34:46Z'
    - type: PodScheduled
      status: 'True'
      lastProbeTime: null
      lastTransitionTime: '2021-09-16T13:34:16Z'
  hostIP: 192.168.50.85
  podIP: 10.42.0.53
  podIPs:
    - ip: 10.42.0.53
  startTime: '2021-09-16T13:34:16Z'
  containerStatuses:
    - name: kube-prometheus-stack
      state:
        running:
          startedAt: '2021-09-16T13:34:46Z'
      lastState: {}
      ready: true
      restartCount: 0
      image: 'quay.io/prometheus-operator/prometheus-operator:v0.50.0'
      imageID: >-
        quay.io/prometheus-operator/prometheus-operator@sha256:ab4f480f2cc65e98f1b4dfb93eb3a41410036359c238fdd60bb3f59deca8d522
      containerID: >-
        containerd://4f03044b1013f18f918034e96950908ee3fc31f8de901a740a8594b273b957bc
      started: true
  qosClass: BestEffort
spec:
  volumes:
    - name: tls-secret
      secret:
        secretName: prometheus-kube-prometheus-admission
        defaultMode: 420
    - name: kube-api-access-6rbnp
      projected:
        sources:
          - serviceAccountToken:
              expirationSeconds: 3607
              path: token
          - configMap:
              name: kube-root-ca.crt
              items:
                - key: ca.crt
                  path: ca.crt
          - downwardAPI:
              items:
                - path: namespace
                  fieldRef:
                    apiVersion: v1
                    fieldPath: metadata.namespace
        defaultMode: 420
  containers:
    - name: kube-prometheus-stack
      image: 'quay.io/prometheus-operator/prometheus-operator:v0.50.0'
      args:
        - '--kubelet-service=kube-system/prometheus-kube-prometheus-kubelet'
        - '--localhost=127.0.0.1'
        - >-
          --prometheus-config-reloader=quay.io/prometheus-operator/prometheus-config-reloader:v0.50.0
        - '--config-reloader-cpu-request=100m'
        - '--config-reloader-cpu-limit=100m'
        - '--config-reloader-memory-request=50Mi'
        - '--config-reloader-memory-limit=50Mi'
        - '--thanos-default-base-image=quay.io/thanos/thanos:v0.17.2'
        - '--web.enable-tls=true'
        - '--web.cert-file=/cert/cert'
        - '--web.key-file=/cert/key'
        - '--web.listen-address=:10250'
        - '--web.tls-min-version=VersionTLS13'
      ports:
        - name: https
          containerPort: 10250
          protocol: TCP
      resources: {}
      volumeMounts:
        - name: tls-secret
          readOnly: true
          mountPath: /cert
        - name: kube-api-access-6rbnp
          readOnly: true
          mountPath: /var/run/secrets/kubernetes.io/serviceaccount
      terminationMessagePath: /dev/termination-log
      terminationMessagePolicy: File
      imagePullPolicy: IfNotPresent
      securityContext:
        readOnlyRootFilesystem: true
        allowPrivilegeEscalation: false
  restartPolicy: Always
  terminationGracePeriodSeconds: 30
  dnsPolicy: ClusterFirst
  serviceAccountName: prometheus-kube-prometheus-operator
  serviceAccount: prometheus-kube-prometheus-operator
  nodeName: ubuntu
  securityContext:
    runAsUser: 65534
    runAsGroup: 65534
    runAsNonRoot: true
    fsGroup: 65534
  schedulerName: default-scheduler
  tolerations:
    - key: node.kubernetes.io/not-ready
      operator: Exists
      effect: NoExecute
      tolerationSeconds: 300
    - key: node.kubernetes.io/unreachable
      operator: Exists
      effect: NoExecute
      tolerationSeconds: 300
  priority: 0
  enableServiceLinks: true
  preemptionPolicy: PreemptLowerPriority