Nginx chunked add number before body

3/22/2018

I'm using nginx to proxy Icecast web server.

When I'm starting to stream data from the server, Nginx add a number between headers and real body. When I'm making the same request directly to Icecast this number isn't present.

Note: I'm using Kubernetes nginx ingress manager to generate config

My Nginx configuration:

daemon off;

worker_processes 2;
pid /run/nginx.pid;

worker_rlimit_nofile 100981;

worker_shutdown_timeout 10s ;

events {
    multi_accept on;
    worker_connections 16384;
    use epoll;
}

http {

    real_ip_header X-Forwarded-For;
    real_ip_recursive on;
    set_real_ip_from 0.0.0.0/0;

    aio threads;
    aio_write on;

    tcp_nopush on;
    tcp_nodelay on;

    log_subrequest on;

    reset_timedout_connection on;

    keepalive_timeout 75s;
    keepalive_requests 100;

    client_header_buffer_size 1k;
    client_header_timeout 60s;
    large_client_header_buffers 4 8k;
    client_body_buffer_size 8k;
    client_body_timeout 60s;

    types_hash_max_size 2048;
    server_names_hash_max_size 1024;
    server_names_hash_bucket_size 64;
    map_hash_bucket_size 64;

    proxy_headers_hash_max_size 512;
    proxy_headers_hash_bucket_size 64;

    variables_hash_bucket_size 128;
    variables_hash_max_size 2048;

    underscores_in_headers off;
    ignore_invalid_headers on;

    limit_req_status 503;

    include /etc/nginx/mime.types;
    default_type text/html;

    gzip on;
    gzip_comp_level 5;
    gzip_http_version 1.1;
    gzip_min_length 256;
    gzip_types application/atom+xml application/javascript application/x-javascript application/json application/rss+xml application/vnd.ms-fontobject application/x-font-ttf application/x-web-app-manifest+json application/xhtml+xml application/xml font/opentype image/svg+xml image/x-icon text/css text/plain text/x-component;
    gzip_proxied any;
    gzip_vary on;

    # Custom headers for response
    server_tokens on;

    # disable warnings
    uninitialized_variable_warn off;

    # Additional available variables:
    # $namespace
    # $ingress_name
    # $service_name
    log_format upstreaminfo '$the_real_ip - [$the_real_ip] - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" $request_length $request_time [$proxy_upstream_name] $upstream_addr $upstream_response_length $upstream_response_time $upstream_status';

    map $request_uri $loggable {
        default 1;
    }

    access_log /var/log/nginx/access.log upstreaminfo if=$loggable;

    error_log /var/log/nginx/error.log notice;

    resolver 10.6.30.8 valid=30s;

    # Retain the default nginx handling of requests without a "Connection" header
    map $http_upgrade $connection_upgrade {
        default upgrade;
        '' close;
    }

    map $http_x_forwarded_for $the_real_ip {
        default $remote_addr;
    }

    # trust http_x_forwarded_proto headers correctly indicate ssl offloading
    map $http_x_forwarded_proto $pass_access_scheme {
        default $http_x_forwarded_proto;
        '' $scheme;
    }

    # validate $pass_access_scheme and $scheme are http to force a redirect
    map "$scheme:$pass_access_scheme" $redirect_to_https {
        default 0;
        "http:http" 1;
        "https:http" 1;
    }

    map $http_x_forwarded_port $pass_server_port {
        default $http_x_forwarded_port;
        '' $server_port;
    }

    map $pass_server_port $pass_port {
        443 443;
        default $pass_server_port;
    }

    # Obtain best http host
    map $http_host $this_host {
        default $http_host;
        '' $host;
    }

    map $http_x_forwarded_host $best_http_host {
        default $http_x_forwarded_host;
        '' $this_host;
    }

    server_name_in_redirect off;
    port_in_redirect off;

    rewrite_log on;

    upstream upstream-default-backend {
        # Load balance algorithm; empty for round robin, which is the default
        least_conn;
        keepalive 32;
        server 10.244.5.2:8080 max_fails=0 fail_timeout=0;
    }

    upstream oma-clients-paj-icecast-icecast-client {
        # Load balance algorithm; empty for round robin, which is the default
        least_conn;
        keepalive 32;
        server 10.244.5.6:8000 max_fails=0 fail_timeout=0;
    }

    ## start server listen.oma-radio.fr
    server {
        server_name listen.oma-radio.fr ;

        listen 80;
        listen [::]:80;

        set $proxy_upstream_name "-";

        location /paj.ogg {
            port_in_redirect off;

            set $proxy_upstream_name "oma-clients-paj-icecast-icecast-client";

            client_max_body_size "2m";

            proxy_set_header Host $best_http_host;

            # Pass the extracted client certificate to the backend
            proxy_set_header ssl-client-cert "";
            proxy_set_header ssl-client-verify "";
            proxy_set_header ssl-client-dn "";

            # Allow websocket connections
            proxy_set_header Upgrade $http_upgrade;
            proxy_set_header Connection $connection_upgrade;
            proxy_set_header X-Real-IP $the_real_ip;
            proxy_set_header X-Forwarded-For $the_real_ip;
            proxy_set_header X-Forwarded-Host $best_http_host;
            proxy_set_header X-Forwarded-Port $pass_port;
            proxy_set_header X-Forwarded-Proto $pass_access_scheme;
            proxy_set_header X-Original-URI $request_uri;
            proxy_set_header X-Scheme $pass_access_scheme;

            # Pass the original X-Forwarded-For
            proxy_set_header X-Original-Forwarded-For $http_x_forwarded_for;

            # mitigate HTTPoxy Vulnerability
            # https://www.nginx.com/blog/mitigating-the-httpoxy-vulnerability-with-nginx/
            proxy_set_header Proxy "";

            # Custom headers to proxied server
            proxy_connect_timeout 5s;
            proxy_send_timeout 60s;
            proxy_read_timeout 60s;

            proxy_buffering "off";
            proxy_buffer_size "4k";
            proxy_buffers 4 "4k";
            proxy_request_buffering "on";

            proxy_http_version 1.1;

            proxy_cookie_domain off;
            proxy_cookie_path off;

            # In case of errors try the next upstream server before returning an error
            proxy_next_upstream error timeout invalid_header http_502 http_503 http_504;

            proxy_pass http://oma-clients-paj-icecast-icecast-client;

            proxy_redirect off;
        }
    }
}

Here the TCP stream received with this configuration: TCP capture

What is the cause of the presence of this number?
How to remove it from there?

-- Nicolas Perraut
http
kubernetes
nginx
tcp

1 Answer

3/22/2018

That number, along with the

Transfer-Encoding: chunked

header is added when nginx doesn't know the size of the resource it is returning- which, in the case of request proxying, can occur when upstream doesn't return a Content-Length header. This is a common and normal state of affairs for streaming upstreams.

When that occurs, nginx returns the data in "chunks", each of which is a byte length, followed by a carriage return, followed by that promised number of bytes of data. The number in the question is the length in bytes of the first chunk of data nginx is returning from icecast.

The use of chunking is to allow both nginx and clients to be smart about their management of buffers.

HTTP/1.1 clients are required to be able to process chunked responses.

-- Jonah Benton
Source: StackOverflow