I am using microk8s and have an nginx frontend service connect to a headless webapplication (ClusterIP = None). However, the nginx service is refused connection to the backend service.

nginx configuration:

apiVersion: v1
kind: ConfigMap
metadata:
  name: nginx-config
data:
  nginx.config: |
    user nginx;
    worker_processes  auto;
    # set open fd limit to 30000
    #worker_rlimit_nofile 10000;
    error_log  /var/log/nginx/error.log;
    events {
      worker_connections  10240;
    }
    http {
      log_format  main
              'remote_addr:$remote_addr\t'
              'time_local:$time_local\t'
              'method:$request_method\t'
              'uri:$request_uri\t'
              'host:$host\t'
              'status:$status\t'
              'bytes_sent:$body_bytes_sent\t'
              'referer:$http_referer\t'
              'useragent:$http_user_agent\t'
              'forwardedfor:$http_x_forwarded_for\t'
              'request_time:$request_time';
      access_log /var/log/nginx/access.log main;
      rewrite_log on;
      upstream svc-web {
        server localhost:8080;
        keepalive 1024;
      }
      server {
        listen 80;
        access_log /var/log/nginx/app.access_log main;
        error_log /var/log/nginx/app.error_log;
        location / {
          proxy_pass http://svc-web;
          proxy_http_version 1.1;
        }
      }
    }

$ k get all
NAME                                          READY   STATUS    RESTARTS   AGE
pod/blazegraph-0                              1/1     Running   0          19h
pod/default-http-backend-587b7d64b5-c4rzj     1/1     Running   0          19h
pod/mysql-0                                   1/1     Running   0          19h
pod/nginx-7fdcdfcc7d-nlqc2                    1/1     Running   0          12s
pod/nginx-ingress-microk8s-controller-b9xcd   1/1     Running   0          19h
pod/web-0                                     1/1     Running   0          13s

NAME                           TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)                      AGE
service/default-http-backend   ClusterIP   10.152.183.94    <none>        80/TCP                       19h
service/kubernetes             ClusterIP   10.152.183.1     <none>        443/TCP                      22h
service/svc-db                 ClusterIP   None             <none>        3306/TCP,9999/TCP            19h
service/svc-frontend           NodePort    10.152.183.220   <none>        80:32282/TCP,443:31968/TCP   12s
service/svc-web                ClusterIP   None             <none>        8080/TCP,8443/TCP            15s

NAME                                               DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE   NODE SELECTOR   AGE
daemonset.apps/nginx-ingress-microk8s-controller   1         1         1       1            1           <none>          19h

NAME                                   DESIRED   CURRENT   UP-TO-DATE   AVAILABLE   AGE
deployment.apps/default-http-backend   1         1         1            1           19h
deployment.apps/nginx                  1         1         1            1           12s

NAME                                              DESIRED   CURRENT   READY   AGE
replicaset.apps/default-http-backend-587b7d64b5   1         1         1       19h
replicaset.apps/nginx-7fdcdfcc7d                  1         1         1       12s

NAME                          DESIRED   CURRENT   AGE
statefulset.apps/blazegraph   1         1         19h
statefulset.apps/mysql        1         1         19h
statefulset.apps/web          1         1         15s

NAME                                              REFERENCE                TARGETS         MINPODS   MAXPODS   REPLICAS   AGE
horizontalpodautoscaler.autoscaling/istio-pilot   Deployment/istio-pilot   <unknown>/55%   1         1         0          19h

$ k describe pod web-0
Name:               web-0
Namespace:          default
Priority:           0
PriorityClassName:  <none>
Node:               khteh-t580/192.168.86.93
Start Time:         Fri, 30 Nov 2018 09:19:53 +0800
Labels:             app=app-web
                    controller-revision-hash=web-5b9476f774
                    statefulset.kubernetes.io/pod-name=web-0
Annotations:        <none>
Status:             Running
IP:                 10.1.1.203
Controlled By:      StatefulSet/web
Containers:
  web-service:
    Container ID:   docker://b5c68ba1d9466c352af107df69f84608aaf233d117a9d71ad307236d10aec03a
    Image:          khteh/tomcat:tomcat-webapi
    Image ID:       docker-pullable://khteh/tomcat@sha256:c246d322872ab315948f6f2861879937642a4f3e631f75e00c811afab7f4fbb9
    Ports:          8080/TCP, 8443/TCP
    Host Ports:     0/TCP, 0/TCP
    State:          Running
      Started:      Fri, 30 Nov 2018 09:20:02 +0800
    Ready:          True
    Restart Count:  0
    Environment:    <none>
    Mounts:
      /usr/share/web/html from web-persistent-storage (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from default-token-s6bpp (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  web-persistent-storage:
    Type:       PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
    ClaimName:  web-persistent-storage-web-0
    ReadOnly:   false
  default-token-s6bpp:
    Type:        Secret (a volume populated by a Secret)
    SecretName:  default-token-s6bpp
    Optional:    false
QoS Class:       BestEffort
Node-Selectors:  <none>
Tolerations:     node.kubernetes.io/not-ready:NoExecute for 300s
                 node.kubernetes.io/unreachable:NoExecute for 300s
Events:
  Type    Reason     Age   From                 Message
  ----    ------     ----  ----                 -------
  Normal  Scheduled  11m   default-scheduler    Successfully assigned default/web-0 to khteh-t580
  Normal  Pulling    11m   kubelet, khteh-t580  pulling image "khteh/tomcat:tomcat-webapi"
  Normal  Pulled     11m   kubelet, khteh-t580  Successfully pulled image "khteh/tomcat:tomcat-webapi"
  Normal  Created    11m   kubelet, khteh-t580  Created container
  Normal  Started    11m   kubelet, khteh-t580  Started container
$ k describe svc svc-frontend
Name:                     svc-frontend
Namespace:                default
Labels:                   <none>
Annotations:              kubectl.kubernetes.io/last-applied-configuration:
                            {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{},"name":"svc-frontend","namespace":"default"},"spec":{"ports":[{"name":"ht...
Selector:                 app=nginx,tier=frontend
Type:                     NodePort
IP:                       10.152.183.159
Port:                     http  80/TCP
TargetPort:               80/TCP
NodePort:                 http  30879/TCP
Endpoints:                10.1.1.204:80
Port:                     https  443/TCP
TargetPort:               443/TCP
NodePort:                 https  31929/TCP
Endpoints:                10.1.1.204:443
Session Affinity:         None
External Traffic Policy:  Cluster
Events:                   <none>

curl <nodportIP>:32282/webapi/greeting would hang.

curl <pod IP>:8080/webapi/greeting WORKS.

curl <endpoint IP>:80/webapi/greeting results in "Bad Gateway":

$ curl http://10.1.1.204/webapi/greeting
<html>
<head><title>502 Bad Gateway</title></head>
<body>
<center><h1>502 Bad Gateway</h1></center>
<hr><center>nginx/1.15.7</center>
</body>
</html>

Inside the nginx container:

root@nginx-7fdcdfcc7d-nlqc2:/var/log/nginx# tail -f app.error_log 
2018/11/24 08:17:04 [error] 8#8: *1 connect() failed (111: Connection refused) while connecting to upstream, client: 10.1.1.1, server: , request: "GET / HTTP/1.1", upstream: "http://127.0.0.1:8080/", host: "localhost:32282"
2018/11/24 08:17:04 [error] 8#8: *1 connect() failed (111: Connection refused) while connecting to upstream, client: 10.1.1.1, server: , request: "GET / HTTP/1.1", upstream: "http://[::1]:8080/", host: "localhost:32282"

$ k get endpoints
NAME                   ENDPOINTS                                                     AGE
default-http-backend   10.1.1.246:80                                                 6d20h
kubernetes             192.168.86.93:6443                                            6d22h
svc-db                 10.1.1.248:9999,10.1.1.253:9999,10.1.1.248:3306 + 1 more...   5h48m
svc-frontend           10.1.1.242:80,10.1.1.242:443                                  6h13m
svc-web                10.1.1.245:8443,10.1.1.245:8080                               6h13m
khteh@khteh-T580:/usr/src/kubernetes/cluster1 2950 $ curl 10.1.1.242:80/webapi/greeting
<html>
<head><title>502 Bad Gateway</title></head>
<body>
<center><h1>502 Bad Gateway</h1></center>
<hr><center>nginx/1.15.7</center>
</body>
</html>
khteh@khteh-T580:/usr/src/kubernetes/cluster1 2951 $