From b4b93ff4eda747393936796131bc28688a0c239f Mon Sep 17 00:00:00 2001 From: ryan Date: Fri, 10 Jul 2026 10:44:49 +0800 Subject: [PATCH] fix(docker): make ClickHouse startable on 25.x and reachable from host Lower merge-tree free-entry thresholds for small background pools, bind listen_host to 0.0.0.0 for published ports, and allow CLICKHOUSE_ENABLED=true in tests for live_ch smoke coverage. --- docker/clickhouse/config.d/listen.xml | 5 ++ docker/clickhouse/config.d/performance.xml | 16 +++- docs/changelog/index.md | 1 + .../apps/openflare/chwriter/live_ch_test.go | 86 +++++++++++++++++++ internal/config/config.go | 12 ++- scripts/live_ch_smoke/main.go | 69 +++++++++++++++ 6 files changed, 183 insertions(+), 6 deletions(-) create mode 100644 docker/clickhouse/config.d/listen.xml create mode 100644 internal/apps/openflare/chwriter/live_ch_test.go create mode 100644 scripts/live_ch_smoke/main.go diff --git a/docker/clickhouse/config.d/listen.xml b/docker/clickhouse/config.d/listen.xml new file mode 100644 index 00000000..ae111ee0 --- /dev/null +++ b/docker/clickhouse/config.d/listen.xml @@ -0,0 +1,5 @@ + + + + 0.0.0.0 + diff --git a/docker/clickhouse/config.d/performance.xml b/docker/clickhouse/config.d/performance.xml index 34f4bba0..5ab67ccd 100644 --- a/docker/clickhouse/config.d/performance.xml +++ b/docker/clickhouse/config.d/performance.xml @@ -1,17 +1,25 @@ 20 - 2 - 1 + 4 + 2 4 2 2 1 268435456 0 + + 2 + 2 + 2 + diff --git a/docs/changelog/index.md b/docs/changelog/index.md index 1c2bda0d..120eb396 100644 --- a/docs/changelog/index.md +++ b/docs/changelog/index.md @@ -27,6 +27,7 @@ sidebar: false - model 层通过 hooks 写入 CH，去除对 `chwriter` 的直接依赖。 - Dashboard 每节点最新指标改为 `LIMIT 1 BY node_id`；新增 metric/openresty 小时预聚合表与读路径优先 rollup。 - 小规格默认连接池下调；`async_insert_busy_timeout` 调至 2s；`of_node_traffic_hourly` 增加 30 天 TTL，UV 改为峰值窗口估计并修正前端文案。 +- Docker ClickHouse：`performance.xml` 下调 merge free-entry 阈值以兼容小 `background_pool`（避免 25.x 启动 Code 36）；增加 `listen_host=0.0.0.0` 以便宿主机访问映射端口。 ## [v3.1.1] - 2026-07-06 diff --git a/internal/apps/openflare/chwriter/live_ch_test.go b/internal/apps/openflare/chwriter/live_ch_test.go new file mode 100644 index 00000000..88d0f0f7 --- /dev/null +++ b/internal/apps/openflare/chwriter/live_ch_test.go @@ -0,0 +1,86 @@ +//go:build live_ch + +// Copyright 2026 Arctel.net +// SPDX-License-Identifier: Apache-2.0 + +package chwriter_test + +import ( + "context" + "testing" + "time" + + "github.com/Rain-kl/Wavelet/internal/apps/openflare/chwriter" + "github.com/Rain-kl/Wavelet/internal/db" + "github.com/Rain-kl/Wavelet/internal/model" +) + +// Run with Docker ClickHouse + config.yaml: +// +// go test -tags live_ch ./internal/apps/openflare/chwriter -run TestLiveAppWritePath -count=1 -timeout 2m +func TestLiveAppWritePath(t *testing.T) { + if !db.ChConnReady() { + t.Skip("ClickHouse connection not ready") + } + ctx := context.Background() + chwriter.Init(ctx) + + now := time.Now().UTC() + nodeID := "e2e-app-write-" + now.Format("150405") + if err := model.InsertOpenFlareMetricSnapshot(ctx, &model.OpenFlareMetricSnapshot{ + NodeID: nodeID, + CapturedAt: now, + CPUUsagePercent: 33.3, + MemoryUsedBytes: 111, + MemoryTotalBytes: 1000, + StorageUsedBytes: 222, + StorageTotalBytes: 2000, + DiskReadBytes: 10, + DiskWriteBytes: 20, + NetworkRxBytes: 30, + NetworkTxBytes: 40, + }); err != nil { + t.Fatalf("InsertOpenFlareMetricSnapshot: %v", err) + } + + deadline := time.Now().Add(45 * time.Second) + var found bool + for time.Now().Before(deadline) { + rows, err := model.ListOpenFlareMetricSnapshotsSince(ctx, nodeID, now.Add(-time.Minute), 10) + if err != nil { + t.Fatalf("ListOpenFlareMetricSnapshotsSince: %v", err) + } + if len(rows) > 0 { + found = true + t.Logf("found snapshot id=%d cpu=%.1f after flush", rows[0].ID, rows[0].CPUUsagePercent) + break + } + time.Sleep(2 * time.Second) + } + if !found { + t.Fatal("metric snapshot not visible in ClickHouse after flush wait") + } + + latest, err := model.ListOpenFlareLatestMetricSnapshotsSince(ctx, "", now.Add(-time.Hour)) + if err != nil { + t.Fatalf("ListOpenFlareLatestMetricSnapshotsSince: %v", err) + } + var latestOK bool + for _, row := range latest { + if row != nil && row.NodeID == nodeID { + latestOK = true + break + } + } + if !latestOK { + t.Fatalf("latest-per-node query missing node %s (rows=%d)", nodeID, len(latest)) + } + + stats := chwriter.WriterStats() + if len(stats) == 0 { + t.Fatal("WriterStats empty after Init") + } + for _, s := range stats { + t.Logf("writer %s running=%v depth=%d drops=%d flush_err=%d", s.Name, s.Running, s.Depth, s.Drops, s.FlushErrors) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index edf354dc..900937be 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -118,9 +118,17 @@ func applyDefaults(c *configModel) { } func applyClickHouseDefaults(c *configModel) { + // Tests disable ClickHouse by default to avoid accidental connections. + // Opt in with CLICKHOUSE_ENABLED=true for live integration tests (e.g. -tags live_ch). if isTest() { - c.ClickHouse.Enabled = false - return + if v, ok := os.LookupEnv("CLICKHOUSE_ENABLED"); !ok { + c.ClickHouse.Enabled = false + return + } else if b, err := strconv.ParseBool(v); err != nil || !b { + c.ClickHouse.Enabled = false + return + } + // Keep Enabled=true from env and continue applying host/pool defaults. } if !c.ClickHouse.Enabled { c.ClickHouse.Enabled = true diff --git a/scripts/live_ch_smoke/main.go b/scripts/live_ch_smoke/main.go new file mode 100644 index 00000000..ed32296e --- /dev/null +++ b/scripts/live_ch_smoke/main.go @@ -0,0 +1,69 @@ +package main + +import ( + "context" + "fmt" + "os" + "time" + + "github.com/Rain-kl/Wavelet/internal/apps/openflare/chwriter" + "github.com/Rain-kl/Wavelet/internal/db" + "github.com/Rain-kl/Wavelet/internal/model" +) + +func main() { + if !db.ChConnReady() { + fmt.Fprintln(os.Stderr, "ChConn not ready — check config.yaml clickhouse.enabled") + os.Exit(1) + } + ctx := context.Background() + chwriter.Init(ctx) + now := time.Now().UTC() + nodeID := "e2e-app-" + now.Format("150405") + if err := model.InsertOpenFlareMetricSnapshot(ctx, &model.OpenFlareMetricSnapshot{ + NodeID: nodeID, CapturedAt: now, CPUUsagePercent: 41.2, + MemoryUsedBytes: 123, MemoryTotalBytes: 1000, + StorageUsedBytes: 456, StorageTotalBytes: 2000, + DiskReadBytes: 11, DiskWriteBytes: 22, NetworkRxBytes: 33, NetworkTxBytes: 44, + }); err != nil { + fmt.Fprintln(os.Stderr, "insert:", err) + os.Exit(1) + } + fmt.Println("queued", nodeID) + deadline := time.Now().Add(45 * time.Second) + for time.Now().Before(deadline) { + rows, err := model.ListOpenFlareMetricSnapshotsSince(ctx, nodeID, now.Add(-time.Minute), 5) + if err != nil { + fmt.Fprintln(os.Stderr, "list:", err) + os.Exit(1) + } + if len(rows) > 0 { + fmt.Printf("OK flushed id=%d cpu=%.1f\n", rows[0].ID, rows[0].CPUUsagePercent) + latest, err := model.ListOpenFlareLatestMetricSnapshotsSince(ctx, "", now.Add(-time.Hour)) + if err != nil { + fmt.Fprintln(os.Stderr, "latest:", err) + os.Exit(1) + } + ok := false + for _, r := range latest { + if r != nil && r.NodeID == nodeID { + ok = true + } + } + if !ok { + fmt.Fprintln(os.Stderr, "FAIL latest-per-node missing node") + os.Exit(1) + } + fmt.Println("OK latest-per-node includes node") + for _, s := range chwriter.WriterStats() { + fmt.Printf("writer %s running=%v depth=%d drops=%d flush_err=%d\n", + s.Name, s.Running, s.Depth, s.Drops, s.FlushErrors) + } + _ = chwriter.Stop(ctx) + return + } + time.Sleep(2 * time.Second) + } + fmt.Fprintln(os.Stderr, "FAIL: not flushed within 45s") + os.Exit(1) +}