load-balancer

What this skill does

# Load Balancing: Nginx & HAProxy Core Knowledge

## Nginx vs HAProxy — When to Use Each

| Dimension | Nginx | HAProxy |
|---|---|---|
| Primary role | Web server + reverse proxy + LB | Dedicated load balancer + proxy |
| Config complexity | Low-medium | Medium-high |
| HTTP modes | HTTP/1.1, HTTP/2 | HTTP/1.1, HTTP/2 (enterprise), HTTP/3 (1.9+) |
| TCP/UDP LB | Nginx Plus or stream module | Native, very mature |
| Active health checks | Nginx Plus only (open-source: passive only) | Built-in, free |
| Stats/metrics UI | Third-party (nginx-lua, stub_status) | Built-in stats page |
| Sticky sessions | Nginx Plus (cookie) or ip_hash | Stick tables (any key), free |
| Connection reuse | Keepalive to upstream | Reuse connections, queue management |
| Dynamic reconfiguration | Nginx Plus (`upstream_conf` API) | Runtime API (HAProxy 2.0+) |
| Ecosystem / docs | Very mature, massive | Mature, industry standard for pure LB |

**Use Nginx when**: you already use Nginx as your web server, you want a single tool for
serving files + proxying + LB, or your team knows Nginx.

**Use HAProxy when**: you need advanced health checks, fine-grained ACL routing, TCP load
balancing, or maximum LB performance and observability.

---

## Nginx Upstream Configuration

### Basic Upstream Block

```nginx
# /etc/nginx/nginx.conf or included conf

http {
    # Shared memory zone for upstream state across workers
    # Required for proper load balancing with multiple workers
    upstream app_backend {
        zone app_zone 256k;         # Shared state (round_robin works without it too)

        # Balancing method (default is round_robin if nothing specified)
        # least_conn;               # Route to backend with fewest active connections
        # ip_hash;                  # Sticky: same client IP always → same backend
        # hash $request_uri consistent;  # Consistent hashing by URI (good for caching)
        # random two least_conn;    # Pick 2 random servers, send to less-loaded one

        server 10.0.1.10:3000 weight=3 max_fails=3 fail_timeout=30s;
        server 10.0.1.11:3000 weight=1 max_fails=3 fail_timeout=30s;
        server 10.0.1.12:3000 weight=1 max_fails=3 fail_timeout=30s;

        # Backup server — only used when all primaries are down
        server 10.0.1.20:3000 backup;

        # Permanently excluded (maintenance)
        # server 10.0.1.13:3000 down;

        # Keepalive connections to upstream (dramatically reduces TCP overhead)
        keepalive 64;               # Max idle keepalive connections per worker
        keepalive_requests 1000;    # Max requests per keepalive connection
        keepalive_timeout 60s;
    }

    server {
        listen 80;
        server_name api.example.com;

        # Logging with upstream info
        log_format upstream_log '$remote_addr - $upstream_addr [$time_local] '
                                 '"$request" $status $body_bytes_sent '
                                 'rt=$request_time urt=$upstream_response_time';
        access_log /var/log/nginx/api_access.log upstream_log;

        location / {
            proxy_pass         http://app_backend;
            proxy_http_version 1.1;                     # Required for keepalive
            proxy_set_header   Connection "";           # Required for keepalive
            proxy_set_header   Host              $host;
            proxy_set_header   X-Real-IP         $remote_addr;
            proxy_set_header   X-Forwarded-For   $proxy_add_x_forwarded_for;
            proxy_set_header   X-Forwarded-Proto $scheme;

            # Timeouts
            proxy_connect_timeout  5s;
            proxy_send_timeout     30s;
            proxy_read_timeout     30s;

            # Passive health check: try next upstream on errors
            proxy_next_upstream     error timeout http_502 http_503 http_504;
            proxy_next_upstream_tries 3;
            proxy_next_upstream_timeout 10s;

            # Buffering
            proxy_buffering    on;
            proxy_buffer_size  16k;
            proxy_buffers      8 16k;
        }

        # Health check endpoint for external monitors
        location /nginx-health {
            access_log off;
            return 200 "OK\n";
        }
    }
}
```

### Active Health Check Workaround (Open-Source Nginx)

Nginx OSS only supports passive health checks. Simulate active checks with a small
service or use the `nginx_upstream_check_module` (third-party).

```bash
# Install lua-nginx-module + lua-resty-upstream-healthcheck
# OR use OpenResty (Nginx + LuaJIT bundle)
# Simple approach: use a separate monitoring tool (Consul, HAProxy) alongside Nginx
```

### SSL Termination at Nginx

```nginx
server {
    listen 443 ssl http2;
    server_name api.example.com;

    # Certificate (from Let's Encrypt / Certbot)
    ssl_certificate     /etc/letsencrypt/live/api.example.com/fullchain.pem;
    ssl_certificate_key /etc/letsencrypt/live/api.example.com/privkey.pem;

    # Modern TLS settings
    ssl_protocols       TLSv1.2 TLSv1.3;
    ssl_ciphers         ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384;
    ssl_prefer_server_ciphers off;
    ssl_session_cache   shared:SSL:10m;
    ssl_session_timeout 1d;
    ssl_session_tickets off;

    # HSTS
    add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload" always;

    location / {
        proxy_pass http://app_backend;      # Plain HTTP to backend (internal network)
        proxy_http_version 1.1;
        proxy_set_header Connection "";
        proxy_set_header X-Forwarded-Proto https;
        proxy_set_header X-Real-IP $remote_addr;
    }
}

# HTTP → HTTPS redirect
server {
    listen 80;
    server_name api.example.com;
    return 301 https://$host$request_uri;
}
```

### Stub Status (Metrics Endpoint)

```nginx
server {
    listen 127.0.0.1:8080;   # Bind to localhost only
    location /nginx_status {
        stub_status;
        allow 127.0.0.1;
        deny all;
    }
}
```

---

## HAProxy Configuration

### Full HTTP Load Balancer Config

```haproxy
# /etc/haproxy/haproxy.cfg

global
    log         /dev/log local0 info
    log         /dev/log local0 notice notice
    chroot      /var/lib/haproxy
    pidfile     /var/run/haproxy.pid
    maxconn     50000               # Total concurrent connections
    user        haproxy
    group       haproxy
    daemon
    stats socket /run/haproxy/admin.sock mode 660 level admin expose-fd listeners

defaults
    log         global
    mode        http                # http | tcp
    option      httplog             # Structured HTTP log format
    option      dontlognull         # Don't log health checks
    option      forwardfor          # Add X-Forwarded-For header
    option      http-server-close   # Close server-side connection after each request
    option      redispatch          # Retry on different server if session fails
    timeout     connect  5s
    timeout     client   30s
    timeout     server   30s
    timeout     http-request 10s    # Max time to receive full HTTP request
    timeout     http-keep-alive 5s
    timeout     queue   1m          # Max wait in queue when all servers full
    timeout     tunnel  1h          # For WebSocket / long-lived connections
    retries     3

#──────────────────────────────────────
# Stats page
#──────────────────────────────────────
frontend stats
    bind *:8404
    stats enable
    stats uri /stats
    stats refresh 10s
    stats auth admin:strongpassword    # CHANGE THIS
    stats show-legends
    stats show-node
    # Restrict to internal IPs
    acl internal_nets src 10.0.0.0/8 172.16.0.0/12 192.168.0.0/16
    tcp-request connection reject if !internal_nets

#──────────────────────────────────────
# HTTPS frontend (SSL termination)
#──────────────────────────────────────
frontend https_in
    bind *:443 ssl crt /etc/ssl/certs/example.com.pem  # Combined cert+key PEM
    bind *:80
    http-request redi