How to use multiple nodes for hazelcast kubernetes discovery plugin?

4/4/2019

I setup the hazelcast kubernetes configuration as per the explanations given in the below link,

https://vertx.io/docs/vertx-hazelcast/java/#_configuring_for_kubernetes

But, hazelcast can identify all members on one node and not able to find across all the nodes in the cluster.

Please help us to solve this issue

Following is the service file for hazelcast of type ClusterIP,

apiVersion: v1
kind: Service
metadata:
 name: cb-hazelcast-service
spec:
 selector:
   component: cb-hazelcast-service
 type: ClusterIP
 clusterIP: None
 ports:
 - name: hz-port-name
   port: 5701
   protocol: TCP

Following is the deployment file for microservice 1,

apiVersion: apps/v1
kind: Deployment
metadata:
  name: cb-agent-service
spec:
  replicas: 1
  minReadySeconds: 30
  strategy:
    rollingUpdate:
      maxSurge: 1
      maxUnavailable: 1
    type: RollingUpdate
  selector:
    matchLabels:
      app: cb-agent-service
  template:
    metadata:
      labels:
        app: cb-agent-service
        component: cb-hazelcast-service
    spec:
      containers:
      - name: cb-agent-service
        image: <docker-image-hub>/agent-service:hz-dns-001
        #imagePullPolicy: IfNotPresent
        volumeMounts:
        - mountPath: /usr/data/logs
          name: shared-logs
        ports:
        - containerPort: 8085
          name: cbport
        ports:
        - name: hazelcast
          containerPort: 5701
      volumes:
      - name: shared-logs
        hostPath:
          path: /usr/data/logs
---
apiVersion: v1
kind: Service
metadata:
  name: cb-agent-service
  labels:
    vertx-cluster: "true"
spec:
  type: NodePort
  ports:
  - port: 80
    targetPort: 8085
  selector:
    app: cb-agent-service

following is deployment for another microservice,

apiVersion: apps/v1
kind: Deployment
metadata:
  name: cb-transaction-service
spec:
  replicas: 1
  minReadySeconds: 30
  strategy:
    rollingUpdate:
      maxSurge: 1
      maxUnavailable: 1
    type: RollingUpdate
  selector:
    matchLabels:
      app: cb-transaction-service
  template:
    metadata:
      labels:
        app: cb-transaction-service
        component: cb-hazelcast-service
    spec:
      containers:
      - name: cb-transaction-service
        image: <docker-iamge-hub>/transaction-service:hz-dns-001
        #imagePullPolicy: IfNotPresent
        volumeMounts:
        - mountPath: /usr/data/logs
          name: shared-logs
        ports:
        - containerPort: 8085
          name: cbport
        ports:
        - name: hazelcast
          containerPort: 5701
      nodeSelector:
        service: transaction
      volumes:
      - name: shared-logs
        hostPath:
          path: /usr/data/logs
---
apiVersion: v1
kind: Service
metadata:
  name: cb-transaction-service
  labels:
    vertx-cluster: "true"
spec:
  type: NodePort
  ports:
  - port: 80
    targetPort: 8085
  selector:
    app: cb-transaction-service

Following is the cluster.xml file for all the microservices

<?xml version="1.0" encoding="UTF-8"?>
<hazelcast xsi:schemaLocation="http://www.hazelcast.com/schema/config hazelcast-config-3.6.xsd"
           xmlns="http://www.hazelcast.com/schema/config"
           xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
    <properties>
        <property name="hazelcast.memcache.enabled">false</property>
        <property name="hazelcast.wait.seconds.before.join">0</property>
        <property name="hazelcast.logging.type">slf4j</property>

        <property name="hazelcast.health.monitoring.delay.seconds">2</property>

        <property name="hazelcast.max.no.heartbeat.seconds">5</property>
        <property name="hazelcast.max.no.master.confirmation.seconds">10</property>
        <property name="hazelcast.master.confirmation.interval.seconds">10</property>
        <property name="hazelcast.member.list.publish.interval.seconds">10</property>
        <property name="hazelcast.connection.monitor.interval">10</property>
        <property name="hazelcast.connection.monitor.max.faults">2</property>
        <property name="hazelcast.partition.migration.timeout">10</property>
        <property name="hazelcast.migration.min.delay.on.member.removed.seconds">3</property>

        <!-- at the moment the discovery needs to be activated explicitly -->
        <property name="hazelcast.discovery.enabled">true</property>
        <property name="hazelcast.rest.enabled">false</property>
    </properties>
    <network>
        <port auto-increment="true" port-count="10000">5701</port>
        <outbound-ports>
            <ports>0</ports>
        </outbound-ports>
        <join>
            <multicast enabled="false"/>

            <tcp-ip enabled="false"/>
            <discovery-strategies>
                <discovery-strategy enabled="true"
                                    class="com.hazelcast.kubernetes.HazelcastKubernetesDiscoveryStrategy">
                    <properties>
                        <property name="service-dns">cb-hazelcast-service.default.svc.cluster.local</property>
                    </properties>
                </discovery-strategy>
            </discovery-strategies>
        </join>
    </network>
    <partition-group enabled="false"/>
    <executor-service name="default">
        <pool-size>16</pool-size>
        <!--Queue capacity. 0 means Integer.MAX_VALUE.-->
        <queue-capacity>0</queue-capacity>
    </executor-service>
    <map name="__vertx.subs">
        <backup-count>1</backup-count>
        <time-to-live-seconds>0</time-to-live-seconds>
        <max-idle-seconds>0</max-idle-seconds>
        <max-size policy="PER_NODE">0</max-size>
        <eviction-percentage>25</eviction-percentage>
        <merge-policy>com.hazelcast.map.merge.LatestUpdateMapMergePolicy</merge-policy>
    </map>
    <semaphore name="__vertx.*">
        <initial-permits>1</initial-permits>
    </semaphore>
</hazelcast>
-- Hari
hazelcast
kubernetes

0 Answers