diff --git a/docker-compose.yaml b/docker-compose.yaml index d550f9e4..3ac6264d 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -79,11 +79,16 @@ services: CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-replace-with-clickhouse-password} CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1 TZ: ${TZ:-Asia/Shanghai} + ulimits: + nofile: + soft: 262144 + hard: 262144 ports: - "${CLICKHOUSE_HTTP_PORT:-8123}:8123" - "${CLICKHOUSE_NATIVE_PORT:-9000}:9000" volumes: - ./data/clickhouse_data:/var/lib/clickhouse + - ./docker/clickhouse/config.d:/etc/clickhouse-server/config.d healthcheck: test: ["CMD", "clickhouse-client", "--user", "${CLICKHOUSE_USERNAME:-default}", "--password", "${CLICKHOUSE_PASSWORD:-replace-with-clickhouse-password}", "--query", "SELECT 1"] interval: 10s diff --git a/docker/clickhouse/config.d/performance.xml b/docker/clickhouse/config.d/performance.xml new file mode 100644 index 00000000..29e0f617 --- /dev/null +++ b/docker/clickhouse/config.d/performance.xml @@ -0,0 +1,6 @@ + + + 50 + 8 + 2 + \ No newline at end of file diff --git a/docs/changelog/index.md b/docs/changelog/index.md index 9aedc3d3..fddb3d81 100644 --- a/docs/changelog/index.md +++ b/docs/changelog/index.md @@ -20,6 +20,8 @@ sidebar: false ### 修改 +- Docker 部署为 ClickHouse 服务增加 `nofile` ulimits 与 `docker/clickhouse/config.d/performance.xml` 性能配置挂载，限制 `max_concurrent_queries`、`background_pool_size` 与 `background_merges_mutations_concurrency_ratio`，降低高负载下的合并与查询争用。 +- 审计访问日志写入 ClickHouse 时仅保留安全相关请求头（Authorization、Cookie、X-Forwarded-For、X-Real-IP、User-Agent、Content-Type），敏感头字段以 SHA-256 摘要脱敏，并将序列化后的 headers 载荷上限收紧至 2KB，减小 `w_user_access_logs` 行宽与 merge CPU 开销。 - 隐藏侧边栏“文档库”分组中的“规范示例”与“接口文档”，并将“使用文档”及其他相关页面的文档链接统一跳转至外部文档 https://open-flare.pages.dev/ - 修复全局搜索数据源覆盖不全的问题，补全了所有核心业务控制台页面（节点、规则、域名、证书、DNS、源站、WAF、IP组、Pages、版本发布、访问日志、应用记录和性能调优）及缺失的管理员专有页面（存储、数据、推送、日志）的搜索检索支持。 - 修复系统自更新（Updater）检测上游 GitHub Action Release 时，因资产包名称前缀（`openflare-server`）与仓库名不完全一致导致匹配失败并报错“未找到兼容的 Release”的问题。 diff --git a/internal/apps/risk_control/middleware.go b/internal/apps/risk_control/middleware.go index 9de6ba53..237ff4e3 100644 --- a/internal/apps/risk_control/middleware.go +++ b/internal/apps/risk_control/middleware.go @@ -5,8 +5,11 @@ package risk_control import ( + "crypto/sha256" + "encoding/hex" "encoding/json" "net/http" + "strings" "time" "github.com/Rain-kl/Wavelet/internal/apps/oauth" @@ -18,6 +21,61 @@ import ( "github.com/gin-gonic/gin" ) +const maxAuditLogHeadersBytes = 2 * 1024 + +var auditLogHeaderAllowlist = map[string]struct{}{ + "Authorization": {}, + "Cookie": {}, + "X-Forwarded-For": {}, + "X-Real-Ip": {}, + "User-Agent": {}, + "Content-Type": {}, +} + +func marshalAuditLogHeaders(headers http.Header) string { + if headers == nil { + return "" + } + + filtered := make(http.Header) + for key, values := range headers { + if _, ok := auditLogHeaderAllowlist[key]; !ok { + continue + } + filtered[key] = redactAuditLogHeaderValues(key, values) + } + + headersBytes, err := json.Marshal(filtered) + if err != nil { + return "" + } + if len(headersBytes) <= maxAuditLogHeadersBytes { + return string(headersBytes) + } + return string(headersBytes[:maxAuditLogHeadersBytes]) +} + +func redactAuditLogHeaderValues(key string, values []string) []string { + switch key { + case "Authorization", "Cookie": + redacted := make([]string, len(values)) + for i, value := range values { + redacted[i] = hashAuditLogSensitiveValue(value) + } + return redacted + default: + return values + } +} + +func hashAuditLogSensitiveValue(value string) string { + if strings.TrimSpace(value) == "" { + return "" + } + sum := sha256.Sum256([]byte(value)) + return "sha256:" + hex.EncodeToString(sum[:8]) +} + // RiskControlMiddleware 全局日志采集中间件 func RiskControlMiddleware() gin.HandlerFunc { return func(c *gin.Context) { @@ -47,19 +105,7 @@ func RiskControlMiddleware() gin.HandlerFunc { // 4. 计算耗时并异步推送到缓冲队列 latency := time.Since(start).Milliseconds() - var headersStr string - if c.Request.Header != nil { - // 克隆 Header，避免污染原 HTTP 请求的 Header 对象 - clonedHeaders := make(http.Header) - for k, v := range c.Request.Header { - clonedHeaders[k] = v - } - clonedHeaders.Del("Cookie") - - if headersBytes, err := json.Marshal(clonedHeaders); err == nil { - headersStr = string(headersBytes) - } - } + headersStr := marshalAuditLogHeaders(c.Request.Header) const maxHTTPStatus = 999 status := c.Writer.Status() diff --git a/internal/apps/risk_control/middleware_test.go b/internal/apps/risk_control/middleware_test.go index f1381faf..6c5349a0 100644 --- a/internal/apps/risk_control/middleware_test.go +++ b/internal/apps/risk_control/middleware_test.go @@ -94,6 +94,8 @@ func TestRiskControlMiddleware(t *testing.T) { req, _ := http.NewRequest(http.MethodGet, "/test", nil) req.Header.Set("X-Test-Header", "hello") req.Header.Set("Cookie", "session_id=abcdef123456") + req.Header.Set("Authorization", "Bearer secret-token") + req.Header.Set("Content-Type", "application/json") r.ServeHTTP(w, req) assert.Equal(t, http.StatusOK, w.Code) @@ -106,8 +108,11 @@ func TestRiskControlMiddleware(t *testing.T) { assert.Equal(t, http.MethodGet, logItem.Method) assert.Equal(t, int32(http.StatusOK), logItem.Status) assert.NotEmpty(t, logItem.Headers) - assert.Contains(t, logItem.Headers, "X-Test-Header") - assert.NotContains(t, logItem.Headers, "Cookie") + assert.NotContains(t, logItem.Headers, "X-Test-Header") + assert.Contains(t, logItem.Headers, "Content-Type") + assert.Contains(t, logItem.Headers, "sha256:") + assert.NotContains(t, logItem.Headers, "secret-token") + assert.NotContains(t, logItem.Headers, "session_id=abcdef123456") case <-time.After(200 * time.Millisecond): t.Fatal("expected flushed log item, but got none") }