We have a pod which uses Java based HTTP server. HTTP POST requests are sent to the server from JMeter. We are currently measuring scale-out percentage for this http server by increasing the number of replicas. Attached here is deployment yaml which is configured to use both nodePort and hostPort for exposing the service.
Our tests shows nodePort scale-out percentage is 57 whereas hostPort scale-out percentage is ~95.
We would like to know why this difference in behavior and how to tune nodePort to get scale-out percentage same as hostPort.
Given below are test details:
K8S version: v1.9.6,
system details: 1 master, 3 worker, 1 NFS server
OS: CentOS Linux release 7.3
System resource: 16 GiB RAM(in each K8S node), 12 CPU (in each K8S node), 1 TB storage.
apiVersion: v1
kind: Service
metadata:
name: itom-di-receiver-svc
namespace: opsbridge1
spec:
type: NodePort
ports:
- name: receiver-port
nodePort: 30001
port: 8443
protocol: TCP
targetPort: 5050
selector:
app: itom-di-receiver
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: itom-di-receiver-dpl
namespace: opsbridge1
annotations:
deployment.microfocus.com/default-replica-count: "1"
deployment.microfocus.com/runlevel: UP
spec:
replicas: 1
template:
metadata:
labels:
app: itom-di-receiver
annotations:
pod.boostport.com/vault-approle: opsbridge1-di
pod.boostport.com/vault-init-container: install
spec:
containers:
- name: itom-di-receiver-cnt
image: localhost:5000/hpeswitomsandbox/itom-data-ingestion-receiver:1.3.0-029
livenessProbe:
exec:
command:
- cat
- /receiver/receiver-status.txt
initialDelaySeconds: 180
periodSeconds: 20
readinessProbe:
exec:
command:
- "/receiver/bin/readiness.sh"
initialDelaySeconds: 30
periodSeconds: 20
failureThreshold: 18
securityContext:
capabilities:
drop:
- all
add:
- CHOWN
- SETGID
- SETUID
imagePullPolicy: IfNotPresent
env:
- name: gid
value: "1999"
- name: uid
value: "1999"
- name: KAFKA_SECURITY_PROTOCOL
value: "SSL"
- name: KAFKA_HOSTNAME
valueFrom:
configMapKeyRef:
name: itom-di-kafka-cm
key: kafka.advertised.host.name
- name: KAFKA_PORT
valueFrom:
configMapKeyRef:
name: itom-di-kafka-cm
key: kafka.advertised.port
- name: KAFKA_DEFAULT_TOPIC
valueFrom:
configMapKeyRef:
name: itom-di-receiver-cm
key: kafka.default.topic
- name: KAFKA_MAP_CONTEXT_TO_TOPIC
valueFrom:
configMapKeyRef:
name: itom-di-receiver-cm
key: kafka.map.context.to.topic
- name: RECEIVER_KAFKA_TLS_ENABLE
valueFrom:
configMapKeyRef:
name: itom-di-receiver-cm
key: receiver.kafka.tls.enable
- name: RECEIVER_JVM_ARGS
valueFrom:
configMapKeyRef:
name: itom-di-receiver-cm
key: receiver.jvm.args
### Receiver Performance Tuning Environment Variables ###
- name: RECEIVER_COMPUTE_THREADS
valueFrom:
configMapKeyRef:
name: itom-di-receiver-cm
key: receiver.compute.threads
- name: KAFKA_PRODUCER_BATCH_SIZE
valueFrom:
configMapKeyRef:
name: itom-di-receiver-cm
key: receiver.kafka.producer.batch.size.kb
- name: RECEIVER_REQUEST_BODY_MAXSIZE_MB
valueFrom:
configMapKeyRef:
name: itom-di-receiver-cm
key: receiver.request.body.maxsize.mb
- name: KAFKA_PRODUCER_LINGER_MS
valueFrom:
configMapKeyRef:
name: itom-di-receiver-cm
key: receiver.kafka.producer.linger.ms
- name: KAFKA_PRODUCER_ACKS
valueFrom:
configMapKeyRef:
name: itom-di-receiver-cm
key: receiver.kafka.producer.acks
- name: KAFKA_PRODUCER_COMPRESSION_TYPE
valueFrom:
configMapKeyRef:
name: itom-di-receiver-cm
key: receiver.kafka.producer.compression.type
- name: KAFKA_PRODUCER_BUFFER_MEMORY
valueFrom:
configMapKeyRef:
name: itom-di-receiver-cm
key: receiver.kafka.producer.buffer.memory.mb
- name: KAFKA_PRODUCER_MAX_BLOCK_MS
valueFrom:
configMapKeyRef:
name: itom-di-receiver-cm
key: receiver.kafka.producer.max.block.ms
- name: RECEIVER_HEADER_FIELDNAME_FOR_TOPIC
valueFrom:
configMapKeyRef:
name: itom-di-receiver-cm
key: receiver.header.fieldname.for.topic
- name: RECEIVER_HEADER_FIELDNAME_FOR_TOPIC_KEY
valueFrom:
configMapKeyRef:
name: itom-di-receiver-cm
key: receiver.header.fieldname.for.topic.key
- name: RECEIVER_TOPIC_FROM_HEADER
valueFrom:
configMapKeyRef:
name: itom-di-receiver-cm
key: receiver.topic.from.header
- name: KAFKA_PRODUCER_RETRIES
valueFrom:
configMapKeyRef:
name: itom-di-receiver-cm
key: receiver.kafka.producer.retries
- name: KAFKA_PRODUCER_MAX_IN_FLIGHT_REQUEST_PER_CONNECTION
valueFrom:
configMapKeyRef:
name: itom-di-receiver-cm
key: receiver.kafka.producer.max.in.flight.requests.per.connection
### Security Environment Variables ###
- name: RECEIVER_PROTOCOL
valueFrom:
configMapKeyRef:
name: itom-di-receiver-cm
key: receiver.protocol
- name: RECEIVER_AUTH_METHOD
valueFrom:
configMapKeyRef:
name: itom-di-receiver-cm
key: receiver.auth.method
- name: RECEIVER_KEYSTORE_TYPE
valueFrom:
configMapKeyRef:
name: itom-di-receiver-cm
key: receiver.keystore.type
- name: RECEIVER_TRUSTSTORE_TYPE
valueFrom:
configMapKeyRef:
name: itom-di-receiver-cm
key: receiver.truststore.type
- name: RECEIVER_EXTERNAL_JAR_ENABLE
valueFrom:
configMapKeyRef:
name: itom-di-receiver-cm
key: receiver.external.jar.enable
- name: RECEIVER_JAR_VALIDATE
valueFrom:
configMapKeyRef:
name: itom-di-receiver-cm
key: receiver.jar.validate
- name: CERTIFICATE_REVOCATION
valueFrom:
configMapKeyRef:
name: itom-di-receiver-cm
key: receiver.cert.revocation.enable
- name: CRL_FILE
valueFrom:
configMapKeyRef:
name: itom-di-receiver-cm
key: receiver.cert.revocation.crlfile.enable
- name: PREFER_OCSP
valueFrom:
configMapKeyRef:
name: itom-di-receiver-cm
key: receiver.cert.revocation.ocsp.enable
- name: ENABLE_SOFT_FAIL
valueFrom:
configMapKeyRef:
name: itom-di-receiver-cm
key: receiver.cert.revocation.softfail.enable
- name: PREFER_CRL
valueFrom:
configMapKeyRef:
name: itom-di-receiver-cm
key: receiver.cert.revocation.preferCRL.enable
- name: RESPONDER_URL
valueFrom:
configMapKeyRef:
name: itom-di-receiver-cm
key: ocsp.responderURL
- name: CERT_SUBJECT_NAME
valueFrom:
configMapKeyRef:
name: itom-di-receiver-cm
key: ocsp.responderCertSubjectName
- name: CERT_ISSUER_NAME
valueFrom:
configMapKeyRef:
name: itom-di-receiver-cm
key: ocsp.responderCertIssuerName
- name: CERT_SERIAL_NUMBER
valueFrom:
configMapKeyRef:
name: itom-di-receiver-cm
key: ocsp.responderCertSerialNumber
- name: RECEIVER_FIPS_ENABLE
valueFrom:
configMapKeyRef:
name: itom-di-receiver-cm
key: receiver.fips.enable
- name: RECEIVER_APIKEY_REFRESH_INTERVAL_MINS
valueFrom:
configMapKeyRef:
name: itom-di-receiver-cm
key: receiver.apikey.refresh.interval.mins
ports:
- containerPort: 5050
hostPort: 5051
resources:
limits:
cpu: "4"
memory: "2048Mi"
requests:
cpu: "1"
memory: "1024Mi"
volumeMounts:
- name: di-receiver-volume
mountPath: /receiver/conf
subPath: di/receiver/conf
- name: di-receiver-volume
mountPath: /receiver/conf/endpoint
subPath: di/receiver/conf/endpoint
- name: di-receiver-volume
mountPath: /receiver/conf/schema
subPath: di/receiver/conf/schema
- name: di-receiver-volume
mountPath: /receiver/conf/crl
subPath: di/receiver/conf/crl
- name: di-receiver-log-volume
mountPath: /receiver/log
subPath: di/receiver/log
- name: di-receiver-volume
mountPath: /receiver/ext
subPath: di/receiver/ext
- name: di-receiver-volume
mountPath: /receiver/data
subPath: di/receiver/data
- name: di-receiver-volume
mountPath: /receiver/samples
subPath: di/receiver/samples
- name: vault-token
mountPath: /var/run/secrets/boostport.com
- name: secret-volume
mountPath: /receiver/ssl/ca
- name: secret-volume-kafka
mountPath: /receiver/ssl/store/receiver-kafka
- name: kubernetes-vault-renew
image: localhost:5000/kubernetes-vault-renew:0.5.0
imagePullPolicy: IfNotPresent
volumeMounts:
- name: vault-token
mountPath: /var/run/secrets/boostport.com
initContainers:
- env:
- name: VAULT_ROLE_ID
value: "66d8c1aa-6079-a65f-38c3-89bd7a6fdd2c"
- name: CERT_COMMON_NAME
value: "smperfqa31.hpeswlab.net"
image: localhost:5000/kubernetes-vault-init:0.5.0
imagePullPolicy: IfNotPresent
name: install
resources: {}
securityContext:
runAsUser: 1999
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /var/run/secrets/boostport.com
name: vault-token
- name: itom-di-init-receiver-cnt
image: localhost:5000/hpeswitomsandbox/itom-data-ingestion-receiver:1.3.0-029
command: ["/receiver/bin/run.sh","dependency"]
env:
- name: KAFKA_HOSTNAME
valueFrom:
configMapKeyRef:
name: itom-di-kafka-cm
key: kafka.advertised.host.name
volumes:
- name: di-receiver-volume
persistentVolumeClaim:
claimName: conf-volume
- name: di-receiver-log-volume
persistentVolumeClaim:
claimName: log-volume
- name: vault-token
emptyDir: {}
- name: secret-volume
secret:
secretName: receiver-secret
- name: secret-volume-kafka
secret:
secretName: receiver-kafka-secret