Nginx配置Spring Boot集群：负载均衡+静态资源分离实战

一、架构设计与原理

1.1 整体架构图

1.2 技术选型对比

二、Nginx核心配置详解

2.1 主配置文件(nginx.conf)

# 运行用户和组
user nginx nginx;

# 工作进程数（建议设置为CPU核心数）
worker_processes auto;

# 错误日志级别和路径
error_log /var/log/nginx/error.log warn;

# PID文件位置
pid /var/run/nginx.pid;

# 事件模块配置
events {
    # 每个worker的最大连接数
    worker_connections 10240;
    
    # 使用epoll模型（Linux）
    use epoll;
    
    # 开启多连接接受
    multi_accept on;
    
    # 连接处理优化
    accept_mutex on;
}

# HTTP模块配置
http {
    # 基础文件类型映射
    include /etc/nginx/mime.types;
    default_type application/octet-stream;
    
    # 日志格式定义
    log_format main '$remote_addr - $remote_user [$time_local] "$request" '
                  '$status $body_bytes_sent "$http_referer" '
                  '"$http_user_agent" "$http_x_forwarded_for"';
    
    access_log /var/log/nginx/access.log main;
    
    # 性能优化参数
    sendfile on;
    tcp_nopush on;
    tcp_nodelay on;
    keepalive_timeout 65;
    types_hash_max_size 2048;
    server_tokens off;
    
    # 客户端请求体大小限制
    client_max_body_size 20m;
    client_body_buffer_size 128k;
    
    # 代理相关超时设置
    proxy_connect_timeout 5s;
    proxy_read_timeout 30s;
    proxy_send_timeout 30s;
    proxy_buffer_size 4k;
    proxy_buffers 8 32k;
    proxy_busy_buffers_size 64k;
    
    # Gzip压缩配置
    gzip on;
    gzip_min_length 1k;
    gzip_comp_level 6;
    gzip_types text/plain text/css application/json application/javascript;
    gzip_vary on;
    gzip_proxied any;
    
    # 包含其他配置文件
    include /etc/nginx/conf.d/*.conf;
    include /etc/nginx/sites-enabled/*;
}

2.2 负载均衡配置(upstream.conf)

upstream springboot_cluster {
    # 负载均衡算法（可选：ip_hash, least_conn, hash等）
    least_conn;
    
    # 后端服务器配置
    server 192.168.1.101:8080 weight=5 max_fails=3 fail_timeout=30s;
    server 192.168.1.102:8081 weight=3 max_fails=3 fail_timeout=30s;
    server 192.168.1.103:8082 weight=2 max_fails=3 fail_timeout=30s;
    
    # 健康检查配置（需要nginx_upstream_check_module模块）
    check interval=3000 rise=2 fall=3 timeout=2000 type=http;
    check_http_send "HEAD /actuator/health HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n";
    check_http_expect_alive http_2xx http_3xx;
    
    # 长连接优化
    keepalive 32;
}

2.3 静态资源分离配置(static.conf)

server {
    listen 80;
    server_name static.example.com;
    
    # 静态资源根目录
    root /data/static-resources;
    
    # 访问控制
    location / {
        # 缓存控制
        expires 1y;
        add_header Cache-Control "public, immutable";
        
        # 防盗链配置
        valid_referers none blocked *.example.com;
        if ($invalid_referer) {
            return 403;
        }
        
        # 开启目录浏览（可选）
        autoindex off;
        
        # 尝试文件存在性检查
        try_files $uri $uri/ =404;
    }
    
    # 图片资源特殊处理
    location ~* \.(jpg|jpeg|png|gif|webp)$ {
        # 图片优化处理
        image_filter resize 800 -;
        image_filter_buffer 10M;
        
        # WebP自动转换
        if ($http_accept ~* "webp") {
            add_header Vary Accept;
            rewrite ^(.*)\.(jpg|jpeg|png|gif)$ $1.webp last;
        }
    }
    
    # 禁止访问的文件类型
    location ~* \.(env|git|svn|htaccess|bak|swp)$ {
        deny all;
    }
    
    # 错误页面配置
    error_page 404 /404.html;
    error_page 500 502 503 504 /50x.html;
    
    # 访问日志单独记录
    access_log /var/log/nginx/static-access.log main buffer=32k flush=5m;
}

2.4 动态请求代理配置(api.conf)

server {
    listen 443 ssl http2;
    server_name api.example.com;
    
    # SSL证书配置
    ssl_certificate /etc/letsencrypt/live/api.example.com/fullchain.pem;
    ssl_certificate_key /etc/letsencrypt/live/api.example.com/privkey.pem;
    ssl_session_timeout 1d;
    ssl_session_cache shared:SSL:50m;
    ssl_protocols TLSv1.2 TLSv1.3;
    ssl_ciphers 'ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384';
    ssl_prefer_server_ciphers on;
    
    # HSTS安全头
    add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload" always;
    
    # 动态请求处理
    location / {
        # 负载均衡转发
        proxy_pass http://springboot_cluster;
        
        # 请求头处理
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;
        proxy_set_header X-Forwarded-Port $server_port;
        proxy_set_header X-Request-ID $request_id;
        
        # WebSocket支持
        proxy_http_version 1.1;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection "upgrade";
        
        # 缓冲区优化
        proxy_buffering on;
        proxy_buffer_size 4k;
        proxy_buffers 8 32k;
        proxy_busy_buffers_size 64k;
        proxy_temp_file_write_size 64k;
        
        # 超时控制
        proxy_connect_timeout 5s;
        proxy_read_timeout 30s;
        proxy_send_timeout 30s;
        
        # 错误处理
        proxy_intercept_errors on;
        error_page 502 503 504 /50x.json;
    }
    
    # 健康检查端点
    location /actuator/health {
        access_log off;
        proxy_pass http://springboot_cluster;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
    }
    
    # 管理接口保护
    location /actuator {
        allow 192.168.1.0/24;
        deny all;
        proxy_pass http://springboot_cluster;
    }
    
    # 静态资源重定向到CDN
    location ~* \.(js|css|png|jpg|jpeg|gif|ico|svg)$ {
        return 301 https://static.example.com$request_uri;
    }
    
    # 错误页面配置
    location = /50x.json {
        default_type application/json;
        return 503 '{"status":503,"message":"Service Temporarily Unavailable"}';
    }
}

三、Spring Boot应用适配

3.1 应用配置优化

# application.yml
server:
  port: 8080
  forward-headers-strategy: framework
  tomcat:
    remoteip:
      protocol-header: X-Forwarded-Proto
      remote-ip-header: X-Forwarded-For
    max-threads: 200
    accept-count: 100
    connection-timeout: 5s
    max-connections: 10000
    use-forward-headers: true

management:
  endpoints:
    web:
      exposure:
        include: health,info,metrics,prometheus
  endpoint:
    health:
      show-details: always
      probes:
        enabled: true
    metrics:
      distribution:
        percentiles-histogram: true
        percentiles: 0.5,0.75,0.95,0.99

spring:
  resources:
    static-locations: file:/data/static-resources/
  mvc:
    static-path-pattern: /static/**

3.2 获取真实客户端IP

import javax.servlet.http.HttpServletRequest;
import java.net.InetAddress;
import java.net.UnknownHostException;

public class IpUtils {
    
    private static final String[] IP_HEADERS = {
        "X-Forwarded-For",
        "Proxy-Client-IP",
        "WL-Proxy-Client-IP",
        "HTTP_X_FORWARDED_FOR",
        "HTTP_X_FORWARDED",
        "HTTP_X_CLUSTER_CLIENT_IP",
        "HTTP_CLIENT_IP",
        "HTTP_FORWARDED_FOR",
        "HTTP_FORWARDED",
        "HTTP_VIA",
        "REMOTE_ADDR"
    };
    
    public static String getClientIp(HttpServletRequest request) {
        for (String header : IP_HEADERS) {
            String ip = request.getHeader(header);
            if (ip != null && ip.length() != 0 && !"unknown".equalsIgnoreCase(ip)) {
                if (header.equals("X-Forwarded-For")) {
                    return ip.split(",")[0].trim();
                }
                return ip;
            }
        }
        return request.getRemoteAddr();
    }
    
    public static boolean isInternalIp(String ip) {
        try {
            InetAddress inetAddress = InetAddress.getByName(ip);
            return inetAddress.isSiteLocalAddress() 
                || inetAddress.isLoopbackAddress()
                || ip.startsWith("192.168.")
                || ip.startsWith("10.")
                || ip.startsWith("172.") && ip.substring(6, 8).compareTo("16") >= 0 
                && ip.substring(6, 8).compareTo("31") <= 0;
        } catch (UnknownHostException e) {
            return false;
        }
    }
}

3.3 健康检查增强

import org.springframework.boot.actuate.health.Health;
import org.springframework.boot.actuate.health.HealthIndicator;
import org.springframework.stereotype.Component;

@Component
public class CustomHealthIndicator implements HealthIndicator {
    
    private final DatabaseService databaseService;
    private final CacheService cacheService;
    
    public CustomHealthIndicator(DatabaseService databaseService, 
                               CacheService cacheService) {
        this.databaseService = databaseService;
        this.cacheService = cacheService;
    }
    
    @Override
    public Health health() {
        Health.Builder builder = Health.up();
        
        // 检查数据库连接
        if (!databaseService.isHealthy()) {
            builder.down().withDetail("database", "Connection failed");
        }
        
        // 检查缓存服务
        if (!cacheService.isHealthy()) {
            builder.down().withDetail("cache", "Connection failed");
        }
        
        // 检查磁盘空间
        File root = new File("/");
        long freeSpace = root.getFreeSpace();
        if (freeSpace < 1024 * 1024 * 1024) { // 小于1GB
            builder.withDetail("disk", "Free space low: " + freeSpace / (1024 * 1024) + "MB");
        }
        
        return builder.build();
    }
}

四、性能优化与监控

4.1 Linux内核优化

# /etc/sysctl.conf
# 最大文件描述符
fs.file-max = 65535

# TCP连接优化
net.ipv4.tcp_max_syn_backlog = 8192
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_tw_recycle = 0 # 不建议开启，可能引起NAT问题
net.ipv4.tcp_fin_timeout = 30
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_probes = 5
net.ipv4.tcp_keepalive_intvl = 15

# 网络缓冲区
net.core.somaxconn = 32768
net.core.netdev_max_backlog = 8192
net.core.rmem_max = 16777216
net.core.wmem_max = 16777216
net.ipv4.tcp_rmem = 4096 87380 16777216
net.ipv4.tcp_wmem = 4096 65536 16777216

# 立即生效
sysctl -p

4.2 Nginx性能调优

# 在http模块中添加
http {
    # 文件缓存
    open_file_cache max=10000 inactive=20s;
    open_file_cache_valid 30s;
    open_file_cache_min_uses 2;
    open_file_cache_errors on;
    
    # 连接优化
    reset_timedout_connection on;
    client_header_timeout 15s;
    client_body_timeout 15s;
    send_timeout 10s;
    
    # TCP优化
    tcp_nodelay on;
    tcp_nopush on;
    
    # 输出缓冲区
    output_buffers 4 32k;
    postpone_output 1460;
    
    # 客户端限制
    limit_conn_zone $binary_remote_addr zone=addr:10m;
    limit_conn addr 100;
}

4.3 监控系统集成

Prometheus配置

# prometheus.yml
scrape_configs:
  - job_name: 'nginx'
    metrics_path: '/nginx_status'
    static_configs:
      - targets: ['nginx-server:9113']
    
  - job_name: 'springboot'
    metrics_path: '/actuator/prometheus'
    static_configs:
      - targets: ['app1:8080', 'app2:8080', 'app3:8080']
    relabel_configs:
      - source_labels: [__address__]
        target_label: instance
        regex: '([^:]+)(:\d+)?'
        replacement: '$1'

Grafana仪表盘

1. Nginx关键指标
   • 请求率 (requests/sec)
   • 活跃连接数
   • 请求处理时间 (P95, P99)
   • 错误率 (4xx, 5xx)
2. Spring Boot关键指标
   • JVM内存使用
   • GC次数和时间
   • 线程池状态
   • HTTP请求延迟

五、安全加固措施

5.1 Nginx安全配置

# 在http模块中添加
http {
    # 安全头设置
    add_header X-Frame-Options "SAMEORIGIN";
    add_header X-Content-Type-Options "nosniff";
    add_header X-XSS-Protection "1; mode=block";
    add_header Referrer-Policy "strict-origin-when-cross-origin";
    add_header Content-Security-Policy "default-src 'self'";
    
    # 禁用不安全的HTTP方法
    if ($request_method !~ ^(GET|HEAD|POST|PUT|PATCH|DELETE)$ ) {
        return 405;
    }
    
    # 隐藏服务器信息
    more_clear_headers 'Server';
    more_clear_headers 'X-Powered-By';
}

5.2 防火墙规则

# 只允许HTTP/HTTPS访问
iptables -A INPUT -p tcp --dport 80 -j ACCEPT
iptables -A INPUT -p tcp --dport 443 -j ACCEPT
iptables -A INPUT -j DROP

# 限制连接速率
iptables -A INPUT -p tcp --dport 80 -m connlimit --connlimit-above 50 -j REJECT
iptables -A INPUT -p tcp --dport 443 -m connlimit --connlimit-above 50 -j REJECT

六、高级功能实现

6.1 灰度发布配置

# 根据Cookie分流
map $cookie_gray_release $backend {
    default "production";
    "true" "gray";
}

upstream production {
    server 192.168.1.101:8080;
    server 192.168.1.102:8080;
}

upstream gray {
    server 192.168.1.103:8080;
}

server {
    location / {
        proxy_pass http://$backend;
    }
}

6.2 多机房容灾

upstream backend {
    # 主机房
    server 192.168.1.101:8080 weight=5;
    server 192.168.1.102:8080 weight=5;
    
    # 备机房
    server 192.168.2.101:8080 backup;
    server 192.168.2.102:8080 backup;
    
    # 健康检查
    check interval=3000 rise=2 fall=3 timeout=2000 type=http;
}

6.3 AB测试实现

split_clients "${remote_addr}${http_user_agent}" $variant {
    50%   "A";
    50%   "B";
}

server {
    location / {
        if ($variant = "A") {
            proxy_pass http://backend_a;
        }
        if ($variant = "B") {
            proxy_pass http://backend_b;
        }
    }
}

七、常见问题解决方案

7.1 502 Bad Gateway排查

1. 检查后端服务状态

curl -I http://backend:8080/actuator/health

2. 检查Nginx错误日志

tail -f /var/log/nginx/error.log

3. 检查连接限制

netstat -anp | grep nginx | wc -l

4. 调整缓冲区大小

proxy_buffer_size 16k;
proxy_buffers 4 32k;

7.2 静态资源加载失败

1. 检查文件权限

chown -R nginx:nginx /data/static-resources

2. 检查MIME类型

types {
    application/javascript js;
    text/css css;
}

3. 检查缓存头

expires 1y;
add_header Cache-Control "public, immutable";

八、生产环境检查清单

• 完成压力测试并记录基准指标
• 配置日志轮转和归档
• 设置监控告警阈值
• 实施定期备份策略
• 准备回滚方案文档
• 验证灾备切换流程
• 进行安全扫描和渗透测试

九、性能测试报告示例