Client (Web/App) 
   │
   ▼
[ Nginx Cluster (L4/L7) ] ──► [ Signaling Server Cluster ] (WebSocket/HTTP)
   │                              │
   │                              ▼
   └──────────► [ Media Server Cluster (SFU/MCU) ] (UDP/TCP, TURN/ICE)

./configure 
  --prefix=/usr/local/nginx 
  --with-http_ssl_module 
  --with-http_v2_module 
  --with-http_realip_module 
  --with-http_stub_status_module 
  --with-stream 
  --with-stream_ssl_module 
  --with-stream_realip_module 
  --with-threads 
  --with-compat 
  --add-module=../ngx_brotli 
  --add-module=../nginx-module-vts
make && make install

user  nginx;
worker_processes  auto;                 # 自动匹配 CPU 核心数
worker_cpu_affinity auto;               # 绑核提升缓存命中
worker_rlimit_nofile 1000000;           # 突破文件描述符限制

error_log  /var/log/nginx/error.log warn;
pid        /var/run/nginx.pid;

events {
    use epoll;                          # Linux 专用高性能事件模型
    worker_connections  65535;
    multi_accept on;                    # 一次性接受多个新连接
}

http {
    include       mime.types;
    default_type  application/octet-stream;

    # ---- 通用性能调优 ----
    sendfile            on;
    tcp_nopush          on;
    tcp_nodelay         on;
    keepalive_timeout   65;
    keepalive_requests  10000;
    types_hash_max_size 2048;
    server_tokens       off;            # 隐藏版本号，安全加固

    # ---- 压缩与缓存 ----
    gzip  on;
    gzip_vary on;
    gzip_min_length 1k;
    gzip_comp_level 4;
    gzip_types text/plain application/javascript application/json application/xml text/css application/wasm;

    # ---- 上游定义 (信令层) ----
    upstream signaling_backend {
        zone signaling_zone 64k;        # 共享内存区，支持动态重配
        least_conn;                     # 最少连接算法，适合长连接
        server 10.0.1.11:8080 max_fails=3 fail_timeout=10s;
        server 10.0.1.12:8080 max_fails=3 fail_timeout=10s;
        keepalive 512;                  # 后端长连接池
    }

    # ---- 虚拟主机：信令/REST API ----
    server {
        listen 443 ssl http2 reuseport;
        server_name meet.example.com;

        ssl_certificate     /etc/nginx/certs/meet.example.com.pem;
        ssl_certificate_key /etc/nginx/certs/meet.example.com.key;
        ssl_protocols       TLSv1.2 TLSv1.3;
        ssl_ciphers         HIGH:!aNULL:!MD5;
        ssl_session_cache   shared:SSL:20m;
        ssl_session_timeout 10m;

        # 安全响应头
        add_header X-Frame-Options SAMEORIGIN;
        add_header X-Content-Type-Options nosniff;
        add_header Content-Security-Policy "default-src 'self'; connect-src 'self' wss:;";

        # 信令 WebSocket 代理
        location /signaling/ {
            proxy_pass http://signaling_backend;
            proxy_http_version 1.1;
            proxy_set_header Upgrade $http_upgrade;
            proxy_set_header Connection "upgrade";
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_read_timeout 3600s;   # 长连接超时
            proxy_send_timeout 3600s;
        }

        # REST API 透传
        location /api/ {
            proxy_pass http://signaling_backend;
            proxy_set_header Host $host;
            proxy_set_header X-Real-IP $remote_addr;
        }

        # 健康检查端点
        location /nginx-health {
            access_log off;
            return 200 "healthyn";
        }
    }
}

# ---- 四层 Stream 模块：媒体流量 (UDP/TCP) ----
stream {
    log_format  proxy  '$remote_addr [$time_local] $protocol $status $bytes_sent $bytes_received $session_time "$upstream_addr"';
    access_log  /var/log/nginx/stream.log proxy;

    upstream media_udp {
        zone media_udp_zone 64k;
        least_conn;
        server 10.0.2.11:3478;  # TURN/STUN
        server 10.0.2.12:3478;
    }

    upstream media_tcp {
        zone media_tcp_zone 64k;
        least_conn;
        server 10.0.2.11:443;   # TURN over TLS / WebRTC TCP fallback
        server 10.0.2.12:443;
    }

    server {
        listen 3478 udp reuseport;
        proxy_pass media_udp;
        proxy_timeout 30s;
        proxy_responses 1;
    }

    server {
        listen 443 ssl reuseport;
        proxy_pass media_tcp;
        ssl_preread on;                    # SNI 路由前置，不解密
        proxy_ssl on;
        proxy_ssl_verify off;
    }
}

# 连接队列与 TIME_WAIT 复用
net.core.somaxconn = 65535
net.core.netdev_max_backlog = 65535
net.ipv4.tcp_max_syn_backlog = 65535
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_fin_timeout = 15

# 缓冲区扩大，适配大帧视频包
net.core.rmem_max = 67108864
net.core.wmem_max = 67108864
net.ipv4.tcp_rmem = 4096 87380 67108864
net.ipv4.tcp_wmem = 4096 65536 67108864

# BBR 拥塞控制，降低弱网丢包重传延迟
net.core.default_qdisc = fq
net.ipv4.tcp_congestion_control = bbr

# 文件描述符与端口范围
fs.file-max = 2000000
net.ipv4.ip_local_port_range = 1024 65535

nginx  soft  nofile  1000000
nginx  hard  nofile  1000000

# HTTP 层：基于 Key 的漏桶限流
limit_req_zone $binary_remote_addr zone=api_limit:10m rate=200r/s;
limit_conn_zone $binary_remote_addr zone=conn_limit:10m;

server {
    location /api/ {
        limit_req zone=api_limit burst=400 nodelay;
        limit_conn conn_limit 200;
        limit_req_status 429;
        limit_conn_status 429;
        proxy_pass http://signaling_backend;
    }
}

# Stream 层：UDP 连接数限制 (OpenResty ngx_stream_limit_conn_module)
stream {
    limit_conn_zone $binary_remote_addr zone=udp_limit:10m;
    server {
        listen 3478 udp;
        limit_conn udp_limit 100;
        proxy_pass media_udp;
    }
}

Client → Cloud LB (L4 TCP/UDP, 会话保持: Source IP) 
          → Nginx Cluster (L7 HTTP/WS + L4 Stream) 
          → Backend Services

# nginx.conf 主进程配置
worker_shutdown_timeout 300s;  # 给存量连接 5 分钟自然消亡窗口

# 优雅关闭信号处理 (systemd 场景)
# ExecReload=/usr/local/nginx/sbin/nginx -s reload
# KillSignal=SIGQUIT  (快速关闭) vs SIGTERM (优雅关闭)
# 建议 systemd 服务单元设置: KillSignal=SIGTERM, TimeoutStopSec=300

stream {
    server {
        listen 3478 udp reuseport;
        proxy_pass media_turn_cluster;
        # 关键：允许单会话多次交互，防止 Nginx 会话表过早老化
        proxy_timeout 300s;           # 空闲超时 5 分钟，覆盖 ICE 保活周期
        proxy_responses 10;           # 允许单 UDP "会话" 多次请求-响应 (默认 1)
        # 启用 PREROUTING 标记 (需内核/iptables 配合) 实现对称 NAT 穿透辅助
    }
}

stream {
    map $ssl_preread_server_name $turn_backend {
        turn.example.com   turn_tls_cluster;
        meet.example.com   signaling_https_cluster; # 复用 443 端口
        default            reject;
    }

    server {
        listen 443 reuseport;
        proxy_pass $turn_backend;
        ssl_preread on;  # 仅解析 ClientHello SNI，不卸载证书，极低延迟
    }
}

http {
    # 七层信令/HTTP 优化
    proxy_buffering off;          # 关闭缓冲，低延迟直通
    proxy_request_buffering off;  # 上传不缓冲
    tcp_nodelay on;               # 禁用 Nagle 算法
}

stream {
    # 四层媒体流优化
    proxy_socket_keepalive on;    # TCP Keepalive 探测死连
    # UDP 无缓冲参数，依赖内核 net.core.wmem_max / rmem_max
    # 建议配合 TC (Traffic Control) 在宿主机网卡出向做 FQ_Codel / Cake AQM
}

# 假设网卡 eth0 上行 1Gbps，为视频流留 800Mbps，其余给信令/管理
tc qdisc add dev eth0 root handle 1: htb default 10
tc class add dev eth0 parent 1: classid 1:1 htb rate 1gbit ceil 1gbit
tc class add dev eth0 parent 1:1 classid 1:10 htb rate 800mbit ceil 900mbit prio 1  # Media
tc class add dev eth0 parent 1:1 classid 1:20 htb rate 200mbit ceil 1gbit prio 2    # Control
tc qdisc add dev eth0 parent 1:10 handle 10: fq_codel  # 流公平队列 + AQM
tc qdisc add dev eth0 parent 1:20 handle 20: fq_codel
tc filter add dev eth0 protocol ip parent 1:0 prio 1 u32 match ip dport 3478 0xffff flowid 1:10  # UDP Media
tc filter add dev eth0 protocol ip parent 1:0 prio 1 u32 match ip dport 443 0xffff flowid 1:10  # TURN TLS

# 依赖 OpenResty / nginx-plus / ngx_http_auth_jwt_module
location /signaling/ {
    auth_jwt "Meeting Realm";
    auth_jwt_key_file /etc/nginx/keys/jwks.json;  # 支持 JWKS 自动轮换
    auth_jwt_claim_custom $claim_room_id room_id; # 提取自定义 Claim 透传给上游
    
    proxy_set_header X-User-ID $jwt_claim_sub;
    proxy_set_header X-Room-ID $claim_room_id;
    proxy_set_header X-User-Role $jwt_claim_role;
    
    # 细粒度权限：仅允许主持人访问控制接口
    location /signaling/control/ {
        if ($jwt_claim_role != "host") { return 403; }
        proxy_pass http://signaling_backend;
    }
    proxy_pass http://signaling_backend;
}

http {
    geoip2 /usr/share/GeoIP/GeoLite2-Country.mmdb {
        $geoip_country_code country/iso_code;
        $geoip_is_anonymous proxy/is_anonymous;  # 代理/VPN/Tor 检测
    }

    # 威胁情报 IP 集合 (通过 API 定期更新 /etc/nginx/blocklist.conf)
    geo $is_malicious {
        default 0;
        include /etc/nginx/blocklist.conf;  # 格式: 1.2.3.0/24 1;
    }

    map $http_user_agent $is_bot {
        default 0;
        "~*(bot|crawler|spider|scanner|curl|wget|python|go-http)" 1;
    }

    server {
        # 统一拦截入口
        if ($is_malicious = 1) { return 444; }  # 静默丢弃，不返回 TCP RST
        if ($geoip_is_anonymous = "true") { 
            # 匿名代理强制挑战或降级
            set $challenge_required 1; 
        }
        if ($is_bot = 1) { return 403 "Automated access denied"; }
        
        # 针对登录/创建会议接口的 JS Challenge / CAPTCHA (需 OpenResty lua-resty-captcha)
        location /api/v1/meetings/create {
            access_by_lua_block {
                if ngx.var.challenge_required == "1" then
                    -- 返回 JS Challenge / Turnstile / reCAPTCHA 验证页面
                end
            }
            proxy_pass http://signaling_backend;
        }
    }
}

# Nginx 作为 Client 向上游发起 mTLS
upstream signaling_backend {
    server 10.0.1.11:8443;
    server 10.0.1.12:8443;
    keepalive 512;
    zone signaling_zone 64k;
}

server {
    location /api/ {
        proxy_pass https://signaling_backend;  # 注意 https
        proxy_ssl_certificate     /etc/nginx/mtls/client.pem;
        proxy_ssl_certificate_key /etc/nginx/mtls/client.key;
        proxy_ssl_trusted_certificate /etc/nginx/mtls/ca.pem;
        proxy_ssl_verify on;
        proxy_ssl_verify_depth 2;
        proxy_ssl_session_reuse on;
        proxy_ssl_name backend-signaling.example.com; # SNI 校验
    }
}

# k8s Ingress 示例：信令服务灰度发布
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: signaling-ingress
  annotations:
    nginx.ingress.kubernetes.io/canary: "true"
    nginx.ingress.kubernetes.io/canary-weight: "10"  # 10% 流量灰度
    nginx.ingress.kubernetes.io/canary-by-header: "X-Canary-User"
    nginx.ingress.kubernetes.io/websocket-services: "signaling"
    nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
    nginx.ingress.kubernetes.io/proxy-send-timeout: "3600"
spec:
  ingressClassName: nginx
  rules:
  - host: meet.example.com
    http:
      paths:
      - path: /signaling
        pathType: Prefix
        backend:
          service:
            name: signaling-canary
            port:
              number: 8080
      - path: /signaling
        pathType: Prefix
        backend:
          service:
            name: signaling-stable
            port:
              number: 8080
  tls:
  - hosts: [meet.example.com]
    secretName: tls-meet-example-com

-- init_worker_by_lua_block 启动定时器同步服务注册中心
local function sync_upstreams()
    local nodes = discovery_client:get_nodes("signaling") -- 从 Consul/Etcd/Nacos 获取
    local upstream = ngx.upstream.get("signaling_backend")
    upstream:set_peers(nodes) -- 原子性替换上游节点列表
end
ngx.timer.every(10, sync_upstreams)

用户请求 → 智能 DNS (GeoIP/EDNS Client Subnet) → 最近 POP Nginx 集群
                ↓
         Nginx 七层二次精准调度 (基于实时延迟探测)

-- access_by_lua_block
local user_ip = ngx.var.remote_addr
-- 查询 Redis 中维护的 "用户IP -> 最优媒体集群延迟" 映射 (由后台探测任务更新)
local best_dc = redis:get("route:latency:" .. user_ip) 
if best_dc then
    ngx.var.target_media_dc = best_dc  -- 透传给上游或直接 proxy_pass 变量
else
    -- 兜底：GeoIP 国家码映射
    local country = ngx.var.geoip_country_code
    ngx.var.target_media_dc = geo_map[country] or "dc-default"
end

stream {
    upstream media_relay_sg {
        zone relay_sg 64k;
        least_conn;
        server media-sg-1.example.com:3478;  # 新加坡媒体集群公网/专线地址
        server media-sg-2.example.com:3478;
    }

    server {
        listen 3478 udp reuseport;
        # 仅当本地无媒体节点或检测到跨域时启用中转
        # 可结合 Lua 动态决定 proxy_pass 目标
        proxy_pass media_relay_sg;
        proxy_timeout 300s;
        proxy_responses 10;
        # 开启 Proxy Protocol v2 透传真实客户端 IP 给远端媒体服务器
        proxy_protocol on; 
    }
}

Nginx 单实例极限并发连接数 ≈ (可用内存 - 系统预留) / (每连接内存开销)
每连接内存开销 ≈ 2KB (HTTP/1.1) ~ 5KB (HTTP/2 + WebSocket + Lua 状态)
建议单 Worker 进程维护 15k-25k 活跃长连接
CPU 瓶颈通常先于内存：TLS 握手 (RSA/ECDSA) + gzip + Lua 逻辑

log_format compliance_json escape=json
  '{'
    '"@timestamp": "$time_iso8601",'
    '"remote_ip": "$remote_addr",'          # 存储前在 Logstash/Fluent Bit 脱敏后两段
    '"user_id": "$http_x_user_id",'         # 来自 JWT 透传 Header，非 Cookie
    '"room_id": "$http_x_room_id",'
    '"request_uri": "$request_uri",'        # 仅记录路径，Query String 敏感参数 (token) 需过滤
    '"status": $status,'
    '"latency_ms": $request_time,'
    '"upstream_addr": "$upstream_addr",'
    '"proto": "$server_protocol",'
    '"ua_hash": "$http_user_agent_hash"'    # 仅存 UA 哈希值，用于设备指纹统计
  '}';

access_log /var/log/nginx/access.log compliance_json buffer=64k flush=5s;