fix(docker): make ClickHouse startable on 25.x and reachable from host

Lower merge-tree free-entry thresholds for small background pools, bind
listen_host to 0.0.0.0 for published ports, and allow CLICKHOUSE_ENABLED=true
in tests for live_ch smoke coverage.
This commit is contained in:
ryan
2026-07-10 10:44:49 +08:00
parent 160e63558f
commit b4b93ff4ed
6 changed files with 183 additions and 6 deletions
+5
View File
@@ -0,0 +1,5 @@
<?xml version="1.0"?>
<!-- Allow Docker published ports to reach native/HTTP interfaces (IPv4). -->
<clickhouse>
<listen_host>0.0.0.0</listen_host>
</clickhouse>
+12 -4
View File
@@ -1,17 +1,25 @@
<?xml version="1.0"?>
<!--
Tuned for small control-plane hosts (e.g. 3c6g).
background_pool_size should stay near the vCPU count; oversized pools cause
idle merge threads to burn CPU while absorbing small parts from heartbeats.
background_pool_size * background_merges_mutations_concurrency_ratio must stay
greater than merge_tree number_of_free_entries_in_pool_to_execute_mutation
(ClickHouse 25.x refuses to start otherwise). Keep the merge free-entry
thresholds low so a small pool remains valid.
-->
<clickhouse>
<max_concurrent_queries>20</max_concurrent_queries>
<background_pool_size>2</background_pool_size>
<background_merges_mutations_concurrency_ratio>1</background_merges_mutations_concurrency_ratio>
<background_pool_size>4</background_pool_size>
<background_merges_mutations_concurrency_ratio>2</background_merges_mutations_concurrency_ratio>
<background_schedule_pool_size>4</background_schedule_pool_size>
<background_common_pool_size>2</background_common_pool_size>
<background_fetches_pool_size>2</background_fetches_pool_size>
<background_move_pool_size>1</background_move_pool_size>
<mark_cache_size>268435456</mark_cache_size>
<uncompressed_cache_size>0</uncompressed_cache_size>
<merge_tree>
<number_of_free_entries_in_pool_to_execute_mutation>2</number_of_free_entries_in_pool_to_execute_mutation>
<number_of_free_entries_in_pool_to_lower_max_size_of_merge>2</number_of_free_entries_in_pool_to_lower_max_size_of_merge>
<number_of_free_entries_in_pool_to_execute_optimize_entire_partition>2</number_of_free_entries_in_pool_to_execute_optimize_entire_partition>
</merge_tree>
</clickhouse>
+1
View File
@@ -27,6 +27,7 @@ sidebar: false
- model 层通过 hooks 写入 CH,去除对 `chwriter` 的直接依赖。
- Dashboard 每节点最新指标改为 `LIMIT 1 BY node_id`;新增 metric/openresty 小时预聚合表与读路径优先 rollup。
- 小规格默认连接池下调;`async_insert_busy_timeout` 调至 2s;`of_node_traffic_hourly` 增加 30 天 TTL,UV 改为峰值窗口估计并修正前端文案。
- Docker ClickHouse:`performance.xml` 下调 merge free-entry 阈值以兼容小 `background_pool`(避免 25.x 启动 Code 36);增加 `listen_host=0.0.0.0` 以便宿主机访问映射端口。
## [v3.1.1] - 2026-07-06
@@ -0,0 +1,86 @@
//go:build live_ch
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package chwriter_test
import (
"context"
"testing"
"time"
"github.com/Rain-kl/Wavelet/internal/apps/openflare/chwriter"
"github.com/Rain-kl/Wavelet/internal/db"
"github.com/Rain-kl/Wavelet/internal/model"
)
// Run with Docker ClickHouse + config.yaml:
//
// go test -tags live_ch ./internal/apps/openflare/chwriter -run TestLiveAppWritePath -count=1 -timeout 2m
func TestLiveAppWritePath(t *testing.T) {
if !db.ChConnReady() {
t.Skip("ClickHouse connection not ready")
}
ctx := context.Background()
chwriter.Init(ctx)
now := time.Now().UTC()
nodeID := "e2e-app-write-" + now.Format("150405")
if err := model.InsertOpenFlareMetricSnapshot(ctx, &model.OpenFlareMetricSnapshot{
NodeID: nodeID,
CapturedAt: now,
CPUUsagePercent: 33.3,
MemoryUsedBytes: 111,
MemoryTotalBytes: 1000,
StorageUsedBytes: 222,
StorageTotalBytes: 2000,
DiskReadBytes: 10,
DiskWriteBytes: 20,
NetworkRxBytes: 30,
NetworkTxBytes: 40,
}); err != nil {
t.Fatalf("InsertOpenFlareMetricSnapshot: %v", err)
}
deadline := time.Now().Add(45 * time.Second)
var found bool
for time.Now().Before(deadline) {
rows, err := model.ListOpenFlareMetricSnapshotsSince(ctx, nodeID, now.Add(-time.Minute), 10)
if err != nil {
t.Fatalf("ListOpenFlareMetricSnapshotsSince: %v", err)
}
if len(rows) > 0 {
found = true
t.Logf("found snapshot id=%d cpu=%.1f after flush", rows[0].ID, rows[0].CPUUsagePercent)
break
}
time.Sleep(2 * time.Second)
}
if !found {
t.Fatal("metric snapshot not visible in ClickHouse after flush wait")
}
latest, err := model.ListOpenFlareLatestMetricSnapshotsSince(ctx, "", now.Add(-time.Hour))
if err != nil {
t.Fatalf("ListOpenFlareLatestMetricSnapshotsSince: %v", err)
}
var latestOK bool
for _, row := range latest {
if row != nil && row.NodeID == nodeID {
latestOK = true
break
}
}
if !latestOK {
t.Fatalf("latest-per-node query missing node %s (rows=%d)", nodeID, len(latest))
}
stats := chwriter.WriterStats()
if len(stats) == 0 {
t.Fatal("WriterStats empty after Init")
}
for _, s := range stats {
t.Logf("writer %s running=%v depth=%d drops=%d flush_err=%d", s.Name, s.Running, s.Depth, s.Drops, s.FlushErrors)
}
}
+10 -2
View File
@@ -118,9 +118,17 @@ func applyDefaults(c *configModel) {
}
func applyClickHouseDefaults(c *configModel) {
// Tests disable ClickHouse by default to avoid accidental connections.
// Opt in with CLICKHOUSE_ENABLED=true for live integration tests (e.g. -tags live_ch).
if isTest() {
c.ClickHouse.Enabled = false
return
if v, ok := os.LookupEnv("CLICKHOUSE_ENABLED"); !ok {
c.ClickHouse.Enabled = false
return
} else if b, err := strconv.ParseBool(v); err != nil || !b {
c.ClickHouse.Enabled = false
return
}
// Keep Enabled=true from env and continue applying host/pool defaults.
}
if !c.ClickHouse.Enabled {
c.ClickHouse.Enabled = true
+69
View File
@@ -0,0 +1,69 @@
package main
import (
"context"
"fmt"
"os"
"time"
"github.com/Rain-kl/Wavelet/internal/apps/openflare/chwriter"
"github.com/Rain-kl/Wavelet/internal/db"
"github.com/Rain-kl/Wavelet/internal/model"
)
func main() {
if !db.ChConnReady() {
fmt.Fprintln(os.Stderr, "ChConn not ready — check config.yaml clickhouse.enabled")
os.Exit(1)
}
ctx := context.Background()
chwriter.Init(ctx)
now := time.Now().UTC()
nodeID := "e2e-app-" + now.Format("150405")
if err := model.InsertOpenFlareMetricSnapshot(ctx, &model.OpenFlareMetricSnapshot{
NodeID: nodeID, CapturedAt: now, CPUUsagePercent: 41.2,
MemoryUsedBytes: 123, MemoryTotalBytes: 1000,
StorageUsedBytes: 456, StorageTotalBytes: 2000,
DiskReadBytes: 11, DiskWriteBytes: 22, NetworkRxBytes: 33, NetworkTxBytes: 44,
}); err != nil {
fmt.Fprintln(os.Stderr, "insert:", err)
os.Exit(1)
}
fmt.Println("queued", nodeID)
deadline := time.Now().Add(45 * time.Second)
for time.Now().Before(deadline) {
rows, err := model.ListOpenFlareMetricSnapshotsSince(ctx, nodeID, now.Add(-time.Minute), 5)
if err != nil {
fmt.Fprintln(os.Stderr, "list:", err)
os.Exit(1)
}
if len(rows) > 0 {
fmt.Printf("OK flushed id=%d cpu=%.1f\n", rows[0].ID, rows[0].CPUUsagePercent)
latest, err := model.ListOpenFlareLatestMetricSnapshotsSince(ctx, "", now.Add(-time.Hour))
if err != nil {
fmt.Fprintln(os.Stderr, "latest:", err)
os.Exit(1)
}
ok := false
for _, r := range latest {
if r != nil && r.NodeID == nodeID {
ok = true
}
}
if !ok {
fmt.Fprintln(os.Stderr, "FAIL latest-per-node missing node")
os.Exit(1)
}
fmt.Println("OK latest-per-node includes node")
for _, s := range chwriter.WriterStats() {
fmt.Printf("writer %s running=%v depth=%d drops=%d flush_err=%d\n",
s.Name, s.Running, s.Depth, s.Drops, s.FlushErrors)
}
_ = chwriter.Stop(ctx)
return
}
time.Sleep(2 * time.Second)
}
fmt.Fprintln(os.Stderr, "FAIL: not flushed within 45s")
os.Exit(1)
}