Minikube with Istio Service Unavailable (http status 503) Node.js connecting to Etcd

6/8/2017

I have been working on a simple Node.js application that SETs and GETs a key from etcd using Istio to connect the two services together. I have tried a few variations but keep seeing the same error returned.

nodeAppTesting failed(etcd-operator) ->{"errors":[{"server":"http://etcd-operator:2379","httperror":null,"httpstatus":503,"httpbody":"upstream connect error or disconnect/reset before headers","response":{"statusCode":503,"body":"upstream connect error or disconnect/reset before headers","headers":{"content-length":"57","content-type":"text/plain","date":"Thu, 08 Jun 2017 17:17:04 GMT","server":"envoy","x-envoy-upstream-service-time":"5"},"request":{"uri":{"protocol":"http:","slashes":true,"auth":null,"host":"etcd-operator:2379","port":"2379","hostname":"etcd-operator","hash":null,"search":null,"query":null,"pathname":"/v2/keys/testKey","path":"/v2/keys/testKey","href":"http://etcd-operator:2379/v2/keys/testKey"},"method":"GET","headers":{"accept":"application/json"}}},"timestamp":"2017-06-08T17:17:04.544Z"}],"retries":0}

Looking at the proxy logs, I can see that client and server proxies are involved in the communication (and this is verified I think in seeing envoy in the server header).

Attaching the Node.js app and the deployment.yaml. server.js

var http = require('http');
var Etcd = require('node-etcd');
var fs = require('fs');
var httpClient = require('request');

var handleRequest = function(request, response) {


    var scheme = "http";
    var ipAddress = "etcd-operator"
    var port = "2379";
    var connectionAddress = scheme +"://" + ipAddress +":" + port;

    console.log('Received request for URL: ' + request.url + " connecting to " + connectionAddress);
    var etcd = new Etcd([connectionAddress] /*, options */);
    etcd.set("testKey" , "foo");
    etcd.get("testKey", function(err, res){
        if(!err){
            response.writeHead(200);
            response.write("nodeAppTesting("+ ipAddress+") ->"+ JSON.stringify(res) ) ;
            response.end();
        }else{
            response.writeHead(500);
            response.write("nodeAppTesting failed("+ ipAddress+") ->"+ JSON.stringify(err) ) ;
            console.log("Encountered error during runtime", JSON.stringify(err));
            response.end();
        }
    });

}
var www = http.createServer(handleRequest);
www.listen(8080);
console.log("App up and running on port 8080")

deployment.yaml

apiVersion: v1
kind: Service
metadata:
  name: etcd-node
  labels:
    app: etcd-node
spec:
  ports:
  - port: 8080
    name: http
  selector:
    app: etcd-node
---

apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: etcd-node-deployment
spec:
  replicas: 1
  template:
    metadata:
      labels:
        app: etcd-node
    spec:
      containers:
      - name: etcd-node
        image: todkap/etcd-node:v1
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 8080
---
##################################################################################################
# etcd service
##################################################################################################
apiVersion: v1
kind: Service
metadata:
  name: etcd-operator
  labels:
    app: etcd-operator
spec:
  ports:
  - port: 2379
    targetPort: 2379
    name: http
  selector:
    app: etcd-operator
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: etcd-operator
spec:
  replicas: 1
  template:
    metadata:
      labels:
        name: etcd-operator
        app: etcd-operator
        version: v1
    spec:
      containers:
      - name: etcd-operator
        image: quay.io/coreos/etcd-operator:v0.2.6
        imagePullPolicy: IfNotPresent
        env:
        - name: MY_POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        - name: MY_POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        ports:
        - containerPort: 2379
---        

apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: gateway2
  annotations:
    kubernetes.io/ingress.class: "istio"
spec:
  rules:
  - http:
      paths:
      - path: /
        backend:
          serviceName: etcd-node
          servicePort: 8080
  - http:
      paths:
      - path: /
        backend:
          serviceName: etcd-operator
          servicePort: 2379
      - path: /v2/keys/*
        backend:
          serviceName: etcd-operator
          servicePort: 2379

---
-- Todd Kaplinger
etcd
istio
kubernetes
node.js

2 Answers

6/9/2017

This is probably a better question for https://groups.google.com/forum/#!forum/istio-users or https://github.com/istio/issues/issues

but it looks like your application isn't up - can you check kubectl get pods and see nothing is still pending ?

-- Laurent Demailly
Source: StackOverflow

6/11/2017

I was able to resolve this issues reported here. I will be publishing a recipe demonstrating the flow sometime this week. For now, we can consider this closed. In the future, I will move to the forums or post on issues. Be on the look out for the article (I will update this post with the link when available). Thanks for the help, guidance and suggestions.

Main issues were consistency in referencing the etcd service, consistency in referencing my node app as a Deployment, Service and Ingress and then finally the exposing the NodePort.

update

Published a article demonstrating the working flow.

-- Todd Kaplinger
Source: StackOverflow