I have an upstream server that simply starts a chunked encoding response and sends a small chunk every second for 100 chunks.
When I hit the upstream server directly like so, all works fine.
curl --raw 10.244.7.248:4000/test/streamHowever, when I go through nginx like so, the response gets cut off at random times.
curl --raw localhost/test/streamCurl returns
curl: (18) transfer closed with outstanding read data remainingand the nginx logs only show
::1 - [::1] - - [24/Oct/2017:11:09:02 +0000] "GET /test/stream HTTP/1.1" 200 440 "-" "curl/7.47.0" 95 44.783 [bar-bar-80] 10.244.7.248:4000 440 44.783 200Notice in this example, the response appears to complete successfully with a 200 but only after 44.783 seconds when it should have been roughly 100 seconds. Every time I try this I get a cut off at a different number of seconds.
I am running Kubernetes Nginx Ingress Controller 0.9.0-beta.14 with gcr.io/google_containers/nginx-ingress-controller:0.9.0-beta.14
The nginx.conf is very long, but here are some snippets
daemon off;
worker_processes 2;
pid /run/nginx.pid;
worker_rlimit_nofile 498976;
worker_shutdown_timeout 10s ;
events {
multi_accept on;
worker_connections 16384;
use epoll;
}
http {
real_ip_header X-Forwarded-For;
real_ip_recursive on;
set_real_ip_from 0.0.0.0/0;
geoip_country /etc/nginx/GeoIP.dat;
geoip_city /etc/nginx/GeoLiteCity.dat;
geoip_proxy_recursive on;
sendfile on;
aio threads;
aio_write on;
tcp_nopush on;
tcp_nodelay on;
log_subrequest on;
reset_timedout_connection on;
keepalive_timeout 75s;
keepalive_requests 100;
client_header_buffer_size 1k;
client_header_timeout 60s;
large_client_header_buffers 4 8k;
client_body_buffer_size 8k;
client_body_timeout 60s;
http2_max_field_size 4k;
http2_max_header_size 16k;
types_hash_max_size 2048;
server_names_hash_max_size 8192;
server_names_hash_bucket_size 128;
map_hash_bucket_size 64;
proxy_headers_hash_max_size 512;
proxy_headers_hash_bucket_size 64;
variables_hash_bucket_size 64;
variables_hash_max_size 2048;
underscores_in_headers off;
ignore_invalid_headers on;
include /etc/nginx/mime.types;
default_type text/html;
gzip on;
gzip_comp_level 5;
gzip_http_version 1.1;
gzip_min_length 256;
gzip_types application/atom+xml application/javascript application/x-javascript application/json application/rss+xml application/vnd.ms-fontobject application/x-font-ttf application/x-web-app-manifest
+json application/xhtml+xml application/xml font/opentype image/svg+xml image/x-icon text/css text/plain text/x-component;
gzip_proxied any;
# Custom headers for response
server_tokens on;
# disable warnings
uninitialized_variable_warn off;
# Additional available variables:
# $namespace
# $ingress_name
# $service_name
log_format upstreaminfo '$the_real_ip - [$the_real_ip] - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" $request_length $request_time [$proxy_upstrea
m_name] $upstream_addr $upstream_response_length $upstream_response_time $upstream_status';
map $request_uri $loggable {
default 1;
}
access_log /var/log/nginx/access.log upstreaminfo if=$loggable;
error_log /var/log/nginx/error.log notice;
resolver 10.0.0.10 valid=30s;
# Retain the default nginx handling of requests without a "Connection" header
map $http_upgrade $connection_upgrade {
default upgrade;
'' close;
}
# Trust HTTP X-Forwarded-* Headers, but use direct values if they're missing.
map $http_x_forwarded_for $the_real_ip {
# Get IP address from X-Forwarded-For HTTP header
default $realip_remote_addr;
'' $remote_addr;
}
# trust http_x_forwarded_proto headers correctly indicate ssl offloading
map $http_x_forwarded_proto $pass_access_scheme {
default $http_x_forwarded_proto;
'' $scheme;
}
map $http_x_forwarded_port $pass_server_port {
default $http_x_forwarded_port;
'' $server_port;
}
map $http_x_forwarded_host $best_http_host {
default $http_x_forwarded_host;
'' $this_host;
}
map $pass_server_port $pass_port {
443 443;
default $pass_server_port;
}
# Map a response error watching the header Content-Type
map $http_accept $httpAccept {
default html;
application/json json;
application/xml xml;
text/plain text;
}
map $httpAccept $httpReturnType {
default text/html;
json application/json;
xml application/xml;
text text/plain;
}
# Obtain best http host
map $http_host $this_host {
default $http_host;
'' $host;
}
server_name_in_redirect off;
port_in_redirect off;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
# turn on session caching to drastically improve performance
ssl_session_cache builtin:1000 shared:SSL:10m;
ssl_session_timeout 10m;
# allow configuring ssl session tickets
ssl_session_tickets on;
# slightly reduce the time-to-first-byte
ssl_buffer_size 4k;
# allow configuring custom ssl ciphers
ssl_ciphers 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-R
SA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:D
HE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-C
BC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA';
ssl_prefer_server_ciphers on;
ssl_ecdh_curve auto;
proxy_ssl_session_reuse on;
upstream bar-bar-80 {
# Load balance algorithm; empty for round robin, which is the default
least_conn;
keepalive 32;
server 10.244.7.248:4000 max_fails=0 fail_timeout=0;
}
server {
server_name bar.gigalixir.com;
listen 80;
listen [::]:80;
set $proxy_upstream_name "-";
listen 443 ssl http2;
listen [::]:443 ssl http2;
ssl_certificate /ingress-controller/ssl/bar-bar-tls.pem;
ssl_certificate_key /ingress-controller/ssl/bar-bar-tls.pem;
more_set_headers "Strict-Transport-Security: max-age=15724800; includeSubDomains;";
location / {
set $proxy_upstream_name "bar-bar-80";
set $namespace "bar";
set $ingress_name "bar";
set $service_name "bar";
# enforce ssl on server side
if ($pass_access_scheme = http) {
return 301 https://$best_http_host$request_uri;
}
port_in_redirect off;
client_max_body_size "0";
proxy_set_header Host $best_http_host;
# Pass the extracted client certificate to the backend
# Allow websocket connections
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
proxy_set_header X-Real-IP $the_real_ip;
proxy_set_header X-Forwarded-For $the_real_ip;
proxy_set_header X-Forwarded-Host $best_http_host;
proxy_set_header X-Forwarded-Port $pass_port;
proxy_set_header X-Forwarded-Proto $pass_access_scheme;
proxy_set_header X-Original-URI $request_uri;
proxy_set_header X-Scheme $pass_access_scheme;
proxy_set_header X-Auth-Request-Redirect $request_uri;
# mitigate HTTPoxy Vulnerability
# https://www.nginx.com/blog/mitigating-the-httpoxy-vulnerability-with-nginx/
proxy_set_header Proxy "";
# Custom headers to proxied server
proxy_connect_timeout 5s;
proxy_send_timeout 60s;
proxy_read_timeout 60s;
proxy_redirect off;
proxy_buffering off;
proxy_buffer_size "4k";
proxy_buffers 4 "4k";
proxy_request_buffering "on";
proxy_http_version 1.1;
proxy_cookie_domain off;
proxy_cookie_path off;
# In case of errors try the next upstream server before returning an error
proxy_next_upstream error timeout invalid_header http_502 http_503 http_504;
proxy_pass http://bar-bar-80;
}
}
I determined that the connection is closed when the nginx configuration is reloaded, which happens periodically with the Kubernetes Nginx Ingress Controller. The nginx workers drain connections for 10 seconds which is not enough in this case. I filed an issue here https://github.com/kubernetes/kubernetes/issues/54505
From the github issue above
you can adjust the drain of connections in the workers adjusting the value of the setting
worker-shutdown-timeout: XXXsin the configuration configmap. The default value is 10 seconds.