Pulumi pods failed to fully initilialize or become live in gcp

8/5/2020

When trying to run deploying a scdf chart using helm, pulumi up returns errors with the services not being able to find pods. Here is an example of the errors

the Kubernetes API server reported that "jc-scdf-prometheus-server" failed to fully initialize or become live: 'jc-scdf-prometheus-server' timed out waiting to be Ready

Service does not target any Pods. Selected Pods may not be ready, or field '.spec.selector' may not match labels on any Pods

'''

k8s_nodepool=NodePool("demo-k8s-cluster-nodepool", 
    autoscaling={
          "maxNodeCount":16, 
          "minNodeCount":1,
    },
    location=zone,
    cluster=k8s_cluster.name,
    node_count=NODE_COUNT,
    name="demo-k8s-node",
    management={
        "autoRepair":True,
        "autoUpgrade":False,
    },
)

# Manufacture a GKE-style Kubeconfig. Note that this is slightly "different" because of the way GKE requires
# gcloud to be in the picture for cluster authentication (rather than using the client cert/key directly).
k8s_info = Output.all(k8s_cluster.name, k8s_cluster.endpoint, k8s_cluster.master_auth)
k8s_config = k8s_info.apply(
    lambda info: """apiVersion: v1
clusters:
- cluster:
    certificate-authority-data: {0}
    server: https://{1}
  name: {2}
contexts:
- context:
    cluster: {2}
    user: {2}
  name: {2}
current-context: {2}
kind: Config
preferences: {{}}
users:
- name: {2}
  user:
    auth-provider:
      config:
        cmd-args: config config-helper --format=json
        cmd-path: gcloud
        expiry-key: '{{.credential.token_expiry}}'
        token-key: '{{.credential.access_token}}'
      name: gcp
""".format(info[2]['clusterCaCertificate'], info[1], '{0}_{1}_{2}'.format(project, zone, info[0])))

# Make a Kubernetes provider instance that uses our cluster from above.
k8s_provider = Provider('gke_k8s', kubeconfig=k8s_config)

crb = ClusterRoleBinding("cluster-admin-binding", 
    metadata={ 'name': 'cluster-admin-binding' },
    role_ref={
        "api_group":"rbac.authorization.k8s.io", 
        "kind":"ClusterRole",
        "name":"cluster-admin",
    }, 
    subjects=[{
        "kind":"User", 
        "name":gcp.organizations.get_client_open_id_user_info().email
        },],
    opts=ResourceOptions(provider=k8s_provider)
)

# Create Helm Chart
nginx_ingress = Chart("demo-nginx-ingress", 
        config=ChartOpts(
            chart="nginx-ingress",
            #namespace=system_namespace.id,
            fetch_opts=FetchOpts(
                  repo="https://kubernetes-charts.storage.googleapis.com/",
            ),
            values={
                "controller": {
                    "metrics": {
                        "enabled": True,
                    },
                },
            },
        ),
)

# Deploy Spring Cloud Data Flow using Helm Chart
scdf = Chart("jc-scdf", 
        config=ChartOpts(
            chart="spring-cloud-data-flow",
            #namespace=system_namespace.id,
            #path="./spring-cloud-data-flow/spring-cloud-data-flow",
            fetch_opts=FetchOpts(
                   repo="https://kubernetes-charts.storage.googleapis.com/",
             ),
            values={
                "kafka": {
                    "enabled":True,
                    "persistence": {"size":"20Gi",},
                },
                "rabbitmq": {"enabled":False,},
                "features": {"monitoring": {"enabled":True,},},
                "server": {"service": {"type":"ClusterIP",},},
                "grafana":{"service":{"type":"ClusterIP",},},
                "prometheus":{"proxy":{"service":{"type":"ClusterIP",},},},
                "ingress":{
                    "enabled":True,
                    "protocol":"http",
                },
            },
        ),
)

data_flow_ingress = Ingress("data-flow-ingress", 
    metadata={
        'name':'scdf-dashboard-ingress',
        'annotations':{
            "kubernetes.io/ingress.class": "nginx",
            "cert-manager.io/cluster-issuer": "letsencrypt-staging"
#            "cert-manager.io/cluster-issuer": "letsencrypt-prod"
        },
    },
    spec={
        'rules': [{
            'host': "scdf.jc.paradymelabs.com",
            'http': {
                'paths': [{
                    "path": "/",
                    "backend": {
                        "serviceName": "jc-scdf-data-flow-server",
                        "servicePort": 80,
                    }
                },],
            },
        },],
        'tls': [{
             'hosts': ["scdf.jc.paradymelabs.com",],
             'secretName':'dataflow-certificate',
        },],
    },
    opts=ResourceOptions(provider=k8s_provider)
)

prometheus_ingress = Ingress("prometheus-ingress", 
    metadata={
        'name':'prometheus-dashboard-ingress',
        'annotations':{
            "kubernetes.io/ingress.class": "nginx",
            "cert-manager.io/cluster-issuer": "letsencrypt-staging"
#            "cert-manager.io/cluster-issuer": "letsencrypt-prod"
        },
    },
    spec={
        'rules': [{
            'host': "prometheus.jc.paradymelabs.com",
            'http': {
                'paths': [{
                    "path": "/",
                    "backend": {
                        "serviceName": "jc-scdf-prometheus-server",
                        "servicePort": 80,
                    }
                },],
            },
        },],
        'tls': [{
             'hosts': ["prometheus.jc.paradymelabs.com",],
             'secretName':'prometheus-certificate',
        },],
    },
    opts=ResourceOptions(provider=k8s_provider)
)

grafana_ingress = Ingress("grafana-ingress", 
    metadata={
        'name':'grafana-dashboard-ingress',
        'annotations':{
            "kubernetes.io/ingress.class": "nginx",
            "cert-manager.io/cluster-issuer": "letsencrypt-staging"
#            "cert-manager.io/cluster-issuer": "letsencrypt-prod"
        },
    },
    spec={
        'rules': [{
            'host': "grafana.jc.paradymelabs.com",
            'http': {
                'paths': [{
                    "path": "/",
                    "backend": {
                        "serviceName": "jc-scdf-grafana-server",
                        "servicePort": 80,
                    }
                },],
            },
        },],
        'tls': [{
             'hosts': ["grafana.jc.paradymelabs.com",],
             'secretName':'grafana-certificate',
        },],
    },
    opts=ResourceOptions(provider=k8s_provider)
)

grafana_storage = PersistentVolumeClaim("grafana-storage",
    metadata={
        'name':'grafana-storage',
    },
    spec={
        'accessModes': ['ReadWriteOnce',],
        'resources':{
            'requests': {'storage': '10Gi'},
        },
    },
    system_namespace], delete_before_replace=True),
    opts=ResourceOptions(provider=k8s_provider)
)

'''

apiVersion: v1
kind: Service
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"v1","kind":"Service","metadata":{"labels":{"app":"spring-cloud-data-flow","app.kubernetes.io/managed-by":"pulumi","chart":"spring-cloud-data-flow-2.7.1","component":"server","heritage":"Helm","release":"jc-scdf"},"name":"jc-scdf-data-flow-server"},"spec":{"ports":[{"name":"http","port":80,"targetPort":"http"}],"selector":{"app":"spring-cloud-data-flow","component":"server","release":"jc-scdf"},"type":"ClusterIP"}}
  creationTimestamp: "2020-08-06T12:43:57Z"
  labels:
    app: spring-cloud-data-flow
    app.kubernetes.io/managed-by: pulumi
    chart: spring-cloud-data-flow-2.7.1
    component: server
    heritage: Helm
    release: jc-scdf
  name: jc-scdf-data-flow-server
  namespace: default
  resourceVersion: "1701"
  selfLink: /api/v1/namespaces/default/services/jc-scdf-data-flow-server
  uid: 785dce7c-7325-4e3c-8442-0631c5526699
spec:
  clusterIP: 10.59.240.165
  ports:
  - name: http
    port: 80
    protocol: TCP
    targetPort: http
  selector:
    app: spring-cloud-data-flow
    component: server
    release: jc-scdf
  sessionAffinity: None
  type: ClusterIP
status:
  loadBalancer: {}
-- josh-pmi
google-cloud-platform
kubernetes
pulumi

0 Answers