Nginx WebSocket长连接及数据容量配置实践

Nginx WebSocket长连接及数据容量配置实践

一、WebSocket基础配置

http {
    # 启用WebSocket代理
    map $http_upgrade $connection_upgrade {
        default upgrade;
        ''      close;
    }

    upstream websocket_backend {
        # 负载均衡配置
        server 192.168.1.10:8080;
        server 192.168.1.11:8080;

        # 最少连接数算法（适合长连接）
        least_conn;

        # 保持长连接配置
        keepalive 100;  # 保持的连接数
    }

    server {
        listen 80;
        server_name ws.example.com;

        location /ws/ {
            # WebSocket代理配置
            proxy_pass http://websocket_backend;
            proxy_http_version 1.1;

            # 必须的WebSocket头
            proxy_set_header Upgrade $http_upgrade;
            proxy_set_header Connection $connection_upgrade;

            # 传递客户端真实信息
            proxy_set_header Host $host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;

            # 长连接超时设置
            proxy_read_timeout 3600s;  # WebSocket读取超时
            proxy_send_timeout 3600s;  # WebSocket发送超时
            proxy_connect_timeout 30s; # 后端连接超时
        }
    }
}

二、数据容量优化配置

http {
    # 缓冲区配置
    proxy_buffer_size 16k;
    proxy_buffers 4 64k;
    proxy_busy_buffers_size 128k;

    # 临时文件配置（处理大消息）
    proxy_temp_file_write_size 256k;
    proxy_max_temp_file_size 1024m;

    # 请求体大小限制（针对WebSocket初始握手）
    client_max_body_size 100m;

    # TCP优化
    tcp_nopush on;
    tcp_nodelay on;

    server {
        location /ws/ {
            # WebSocket特定缓冲区配置
            proxy_buffering off;  # WebSocket通常禁用缓冲

            # 大消息分片配置
            proxy_websocket_buffer_size 128k;
            proxy_request_buffering off;

            # 限制帧大小（可选）
            # proxy_set_header Sec-WebSocket-Protocol $http_sec_websocket_protocol;
        }

        # 针对大文件传输的WebSocket
        location /ws/file-transfer {
            proxy_buffering on;
            proxy_buffer_size 256k;
            proxy_buffers 8 512k;
            proxy_busy_buffers_size 1m;

            # 增加超时时间
            proxy_read_timeout 7200s;
            proxy_send_timeout 7200s;
        }
    }
}

三、连接数限制和负载均衡

events {
    worker_connections 4096;  # 每个worker的连接数
    use epoll;                # Linux高性能事件模型
    multi_accept on;
}

http {
    # 全局连接限制
    limit_conn_zone $binary_remote_addr zone=ws_conn:10m;
    limit_conn_zone $server_name zone=ws_server:10m;

    upstream ws_cluster {
        zone upstream_zone 64k;
        least_conn;

        # 健康检查
        server 192.168.1.10:8080 max_fails=3 fail_timeout=30s;
        server 192.168.1.11:8080 max_fails=3 fail_timeout=30s;
        server 192.168.1.12:8080 backup;  # 备份服务器

        # 会话保持（如果需要）
        hash $remote_addr consistent;

        # 连接数限制
        keepalive 500;
    }

    server {
        location /ws/ {
            # 连接数限制
            limit_conn ws_conn 1000;      # 单个IP最多1000连接
            limit_conn ws_server 10000;   # 服务器总连接数限制

            # 速率限制
            limit_req zone=ws_req burst=50 nodelay;

            proxy_pass http://ws_cluster;

            # 连接保持
            proxy_set_header Connection "";
            proxy_set_header Keep-Alive "";
        }
    }
}

四、SSL/TLS配置（WSS）

server {
    listen 443 ssl http2;
    server_name wss.example.com;

    # SSL证书
    ssl_certificate /etc/nginx/ssl/server.crt;
    ssl_certificate_key /etc/nginx/ssl/server.key;

    # SSL优化
    ssl_protocols TLSv1.2 TLSv1.3;
    ssl_ciphers ECDHE-RSA-AES256-GCM-SHA512:ECDHE-RSA-AES256-GCM-SHA384;
    ssl_prefer_server_ciphers on;
    ssl_session_cache shared:SSL:10m;
    ssl_session_timeout 1h;

    # HSTS
    add_header Strict-Transport-Security "max-age=31536000" always;

    location /wss/ {
        # WebSocket over TLS
        proxy_pass http://websocket_backend;
        proxy_http_version 1.1;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection "upgrade";

        # SSL传递
        proxy_set_header X-Forwarded-Proto https;
        proxy_set_header X-Forwarded-Ssl on;

        # 增强安全
        proxy_set_header Sec-WebSocket-Key $http_sec_websocket_key;
        proxy_set_header Sec-WebSocket-Version $http_sec_websocket_version;
        proxy_set_header Sec-WebSocket-Protocol $http_sec_websocket_protocol;
    }
}

五、监控和日志配置

http {
    # WebSocket专用日志格式
    log_format websocket '$remote_addr - $remote_user [$time_local] '
                        '"$request" $status $body_bytes_sent '
                        '"$http_referer" "$http_user_agent" '
                        '$upstream_addr $upstream_response_time '
                        '$connection $connection_requests '
                        'upgrade:$http_upgrade '
                        'sec_key:$http_sec_websocket_key';

    server {
        # 访问日志
        access_log /var/log/nginx/websocket_access.log websocket;
        error_log /var/log/nginx/websocket_error.log warn;

        location /ws/ {
            # 连接状态监控
            proxy_set_header X-Connection-ID $connection;
            proxy_set_header X-Request-ID $request_id;

            # 统计信息
            stub_status on;  # 在单独location开启状态页

            # 慢连接日志
            log_format slow_ws '$remote_addr [$time_local] '
                              'rt=$request_time uct=$upstream_connect_time '
                              'uht=$upstream_header_time urt=$upstream_response_time';
            access_log /var/log/nginx/ws_slow.log slow_ws if=$slow_connection;
        }
    }
}

六、完整配置示例

# 用户和进程配置
user nginx;
worker_processes auto;  # 自动根据CPU核心数设置
worker_rlimit_nofile 65535;  # 文件描述符限制

events {
    worker_connections 4096;
    use epoll;
    multi_accept on;
}

http {
    include       mime.types;
    default_type  application/octet-stream;

    # 日志格式
    log_format main '$remote_addr - $remote_user [$time_local] "$request" '
                    '$status $body_bytes_sent "$http_referer" '
                    '"$http_user_agent" "$http_x_forwarded_for"';

    # WebSocket升级映射
    map $http_upgrade $connection_upgrade {
        default upgrade;
        ''      close;
    }

    # 连接限制区域
    limit_conn_zone $binary_remote_addr zone=perip:10m;
    limit_conn_zone $server_name zone=perserver:10m;

    # 上游服务器
    upstream websocket_cluster {
        zone ws_cluster 64k;
        least_conn;
        keepalive 500;

        server 10.0.1.10:8080 max_fails=3 fail_timeout=30s weight=10;
        server 10.0.1.11:8080 max_fails=3 fail_timeout=30s weight=10;
        server 10.0.1.12:8080 backup;
    }

    # 主服务器配置
    server {
        listen 80;
        listen 443 ssl http2;
        server_name ws.yourdomain.com;

        # SSL配置
        ssl_certificate /path/to/cert.pem;
        ssl_certificate_key /path/to/key.pem;

        # WebSocket路径
        location /ws/ {
            # 连接限制
            limit_conn perip 1000;
            limit_conn perserver 10000;

            # WebSocket代理
            proxy_pass http://websocket_cluster;
            proxy_http_version 1.1;

            # 升级头
            proxy_set_header Upgrade $http_upgrade;
            proxy_set_header Connection $connection_upgrade;

            # 传递客户端信息
            proxy_set_header Host $host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;

            # 超时设置
            proxy_read_timeout 3600s;
            proxy_send_timeout 3600s;
            proxy_connect_timeout 30s;

            # 缓冲区配置
            proxy_buffering off;
            proxy_request_buffering off;

            # WebSocket特定
            proxy_set_header Sec-WebSocket-Key $http_sec_websocket_key;
            proxy_set_header Sec-WebSocket-Version $http_sec_websocket_version;
            proxy_set_header Sec-WebSocket-Extensions $http_sec_websocket_extensions;

            # 健康检查头
            proxy_set_header X-Health-Check "websocket";
        }

        # 状态监控端点
        location /nginx_status {
            stub_status on;
            access_log off;
            allow 127.0.0.1;
            allow 10.0.0.0/8;
            deny all;
        }

        # 健康检查端点
        location /health {
            access_log off;
            return 200 "healthy\n";
            add_header Content-Type text/plain;
        }
    }
}

七、优化建议和注意事项

性能调优参数：

# 内核参数调整（/etc/sysctl.conf）
net.core.somaxconn = 65535
net.ipv4.tcp_max_syn_backlog = 65535
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_fin_timeout = 30

Nginx配置

worker_processes auto; # CPU核心数 worker_rlimit_nofile 65535; # 文件描述符

2. **监控指标**：
   - 活跃WebSocket连接数
   - 消息传输速率
   - 连接持续时间
   - 错误率和重连频率

3. **安全建议**：
   - 启用WSS（WebSocket Secure）
   - 实施连接数限制
   - 添加身份验证
   - 设置合适的超时时间

4. **故障排查命令**：
```bash
# 查看连接状态
ss -tuna | grep :80
netstat -an | grep ESTABLISHED

# 监控Nginx状态
curl http://localhost/nginx_status

# 日志分析
tail -f /var/log/nginx/websocket_access.log
grep "101 Switching Protocols" access.log

这个配置可以根据实际业务需求进行调整，特别是超时时间、缓冲区大小和连接数限制等参数。