I followed some tutorials on how to set up an HTTP server, and test it in a local Kubernetes cluster (using minikube).

I also implemented graceful shutdown from some examples I found, and expected that there would be no downtime from a Kubernetes rolling restart.

To verify that, I started performing load tests (using Apache Benchmark, by running ab -n 100000 -c 20 <addr>) and running kubectl rollout restart during the benchmarking, but ab stops running as soon as the rolling restart is performed.

Here is my current project setup:

Dockerfile

FROM golang:1.13.4-alpine3.10

RUN mkdir /app
ADD . /app
WORKDIR /app

RUN go build -o main src/main.go
CMD ["/app/main"]

src/main.go

package main

import (
    "context"
    "fmt"
    "log"
    "net/http"
    "os"
    "os/signal"
    "syscall"

    "github.com/gorilla/mux"
)

func main() {
    srv := &http.Server{
        Addr:    ":8080",
        Handler: NewHTTPServer(),
    }

    idleConnsClosed := make(chan struct{})
    go func() {
        sigint := make(chan os.Signal, 1)
        signal.Notify(sigint, os.Interrupt, syscall.SIGTERM, syscall.SIGINT)
        <-sigint

        // We received an interrupt signal, shut down.
        if err := srv.Shutdown(context.Background()); err != nil {
            // Error from closing listeners, or context timeout:
            log.Printf("HTTP server Shutdown: %v", err)
        }

        close(idleConnsClosed)
    }()

    log.Printf("Starting HTTP server")
    running = true
    if err := srv.ListenAndServe(); err != http.ErrServerClosed {
        // Error starting or closing listener:
        log.Fatalf("HTTP server ListenAndServe: %v", err)
    }

    <-idleConnsClosed
}

func NewHTTPServer() http.Handler {
    r := mux.NewRouter()

    // Ping
    r.HandleFunc("/", handler)

    return r
}

func handler(w http.ResponseWriter, r *http.Request) {
    fmt.Fprintf(w, "Hello World!")
}

kubernetes/deployment.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: myapp
  name: myapp
spec:
  replicas: 10
  selector:
    matchLabels:
      app: myapp
  strategy:
    rollingUpdate:
      maxSurge: 1
      maxUnavailable: 5
    type: RollingUpdate
  template:
    metadata:
      labels:
        app: myapp
    spec:
      containers:
      - name: myapp
        image: dickster/graceful-shutdown-test:latest
        imagePullPolicy: Never
        ports:
        - containerPort: 8080

kubernetes/service.yaml

apiVersion: v1
kind: Service
metadata:
  labels:
    app: myapp
  name: myapp
spec:
  ports:
  - port: 8080
    protocol: TCP
  selector:
    app: myapp
  sessionAffinity: None
  type: NodePort

Is there something missing in this setup? According to the rollingUpdate strategy, there should be at least five running pods that should serve the incoming requests, but ab exits with an apr_socket_recv: Connection reset by peer (54) error. I also tried adding readiness/liveness probes, but no luck. I suspect they're not needed here, either.