mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-09-29 05:56:38 +08:00
fix(docker): make ClickHouse startable on 25.x and reachable from host
Lower merge-tree free-entry thresholds for small background pools, bind listen_host to 0.0.0.0 for published ports, and allow CLICKHOUSE_ENABLED=true in tests for live_ch smoke coverage.
This commit is contained in:
@@ -0,0 +1,5 @@
|
||||
<?xml version="1.0"?>
|
||||
<!-- Allow Docker published ports to reach native/HTTP interfaces (IPv4). -->
|
||||
<clickhouse>
|
||||
<listen_host>0.0.0.0</listen_host>
|
||||
</clickhouse>
|
||||
@@ -1,17 +1,25 @@
|
||||
<?xml version="1.0"?>
|
||||
<!--
|
||||
Tuned for small control-plane hosts (e.g. 3c6g).
|
||||
background_pool_size should stay near the vCPU count; oversized pools cause
|
||||
idle merge threads to burn CPU while absorbing small parts from heartbeats.
|
||||
|
||||
background_pool_size * background_merges_mutations_concurrency_ratio must stay
|
||||
greater than merge_tree number_of_free_entries_in_pool_to_execute_mutation
|
||||
(ClickHouse 25.x refuses to start otherwise). Keep the merge free-entry
|
||||
thresholds low so a small pool remains valid.
|
||||
-->
|
||||
<clickhouse>
|
||||
<max_concurrent_queries>20</max_concurrent_queries>
|
||||
<background_pool_size>2</background_pool_size>
|
||||
<background_merges_mutations_concurrency_ratio>1</background_merges_mutations_concurrency_ratio>
|
||||
<background_pool_size>4</background_pool_size>
|
||||
<background_merges_mutations_concurrency_ratio>2</background_merges_mutations_concurrency_ratio>
|
||||
<background_schedule_pool_size>4</background_schedule_pool_size>
|
||||
<background_common_pool_size>2</background_common_pool_size>
|
||||
<background_fetches_pool_size>2</background_fetches_pool_size>
|
||||
<background_move_pool_size>1</background_move_pool_size>
|
||||
<mark_cache_size>268435456</mark_cache_size>
|
||||
<uncompressed_cache_size>0</uncompressed_cache_size>
|
||||
<merge_tree>
|
||||
<number_of_free_entries_in_pool_to_execute_mutation>2</number_of_free_entries_in_pool_to_execute_mutation>
|
||||
<number_of_free_entries_in_pool_to_lower_max_size_of_merge>2</number_of_free_entries_in_pool_to_lower_max_size_of_merge>
|
||||
<number_of_free_entries_in_pool_to_execute_optimize_entire_partition>2</number_of_free_entries_in_pool_to_execute_optimize_entire_partition>
|
||||
</merge_tree>
|
||||
</clickhouse>
|
||||
|
||||
@@ -27,6 +27,7 @@ sidebar: false
|
||||
- model 层通过 hooks 写入 CH,去除对 `chwriter` 的直接依赖。
|
||||
- Dashboard 每节点最新指标改为 `LIMIT 1 BY node_id`;新增 metric/openresty 小时预聚合表与读路径优先 rollup。
|
||||
- 小规格默认连接池下调;`async_insert_busy_timeout` 调至 2s;`of_node_traffic_hourly` 增加 30 天 TTL,UV 改为峰值窗口估计并修正前端文案。
|
||||
- Docker ClickHouse:`performance.xml` 下调 merge free-entry 阈值以兼容小 `background_pool`(避免 25.x 启动 Code 36);增加 `listen_host=0.0.0.0` 以便宿主机访问映射端口。
|
||||
|
||||
## [v3.1.1] - 2026-07-06
|
||||
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
//go:build live_ch
|
||||
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package chwriter_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/apps/openflare/chwriter"
|
||||
"github.com/Rain-kl/Wavelet/internal/db"
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
)
|
||||
|
||||
// Run with Docker ClickHouse + config.yaml:
|
||||
//
|
||||
// go test -tags live_ch ./internal/apps/openflare/chwriter -run TestLiveAppWritePath -count=1 -timeout 2m
|
||||
func TestLiveAppWritePath(t *testing.T) {
|
||||
if !db.ChConnReady() {
|
||||
t.Skip("ClickHouse connection not ready")
|
||||
}
|
||||
ctx := context.Background()
|
||||
chwriter.Init(ctx)
|
||||
|
||||
now := time.Now().UTC()
|
||||
nodeID := "e2e-app-write-" + now.Format("150405")
|
||||
if err := model.InsertOpenFlareMetricSnapshot(ctx, &model.OpenFlareMetricSnapshot{
|
||||
NodeID: nodeID,
|
||||
CapturedAt: now,
|
||||
CPUUsagePercent: 33.3,
|
||||
MemoryUsedBytes: 111,
|
||||
MemoryTotalBytes: 1000,
|
||||
StorageUsedBytes: 222,
|
||||
StorageTotalBytes: 2000,
|
||||
DiskReadBytes: 10,
|
||||
DiskWriteBytes: 20,
|
||||
NetworkRxBytes: 30,
|
||||
NetworkTxBytes: 40,
|
||||
}); err != nil {
|
||||
t.Fatalf("InsertOpenFlareMetricSnapshot: %v", err)
|
||||
}
|
||||
|
||||
deadline := time.Now().Add(45 * time.Second)
|
||||
var found bool
|
||||
for time.Now().Before(deadline) {
|
||||
rows, err := model.ListOpenFlareMetricSnapshotsSince(ctx, nodeID, now.Add(-time.Minute), 10)
|
||||
if err != nil {
|
||||
t.Fatalf("ListOpenFlareMetricSnapshotsSince: %v", err)
|
||||
}
|
||||
if len(rows) > 0 {
|
||||
found = true
|
||||
t.Logf("found snapshot id=%d cpu=%.1f after flush", rows[0].ID, rows[0].CPUUsagePercent)
|
||||
break
|
||||
}
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("metric snapshot not visible in ClickHouse after flush wait")
|
||||
}
|
||||
|
||||
latest, err := model.ListOpenFlareLatestMetricSnapshotsSince(ctx, "", now.Add(-time.Hour))
|
||||
if err != nil {
|
||||
t.Fatalf("ListOpenFlareLatestMetricSnapshotsSince: %v", err)
|
||||
}
|
||||
var latestOK bool
|
||||
for _, row := range latest {
|
||||
if row != nil && row.NodeID == nodeID {
|
||||
latestOK = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !latestOK {
|
||||
t.Fatalf("latest-per-node query missing node %s (rows=%d)", nodeID, len(latest))
|
||||
}
|
||||
|
||||
stats := chwriter.WriterStats()
|
||||
if len(stats) == 0 {
|
||||
t.Fatal("WriterStats empty after Init")
|
||||
}
|
||||
for _, s := range stats {
|
||||
t.Logf("writer %s running=%v depth=%d drops=%d flush_err=%d", s.Name, s.Running, s.Depth, s.Drops, s.FlushErrors)
|
||||
}
|
||||
}
|
||||
@@ -118,9 +118,17 @@ func applyDefaults(c *configModel) {
|
||||
}
|
||||
|
||||
func applyClickHouseDefaults(c *configModel) {
|
||||
// Tests disable ClickHouse by default to avoid accidental connections.
|
||||
// Opt in with CLICKHOUSE_ENABLED=true for live integration tests (e.g. -tags live_ch).
|
||||
if isTest() {
|
||||
c.ClickHouse.Enabled = false
|
||||
return
|
||||
if v, ok := os.LookupEnv("CLICKHOUSE_ENABLED"); !ok {
|
||||
c.ClickHouse.Enabled = false
|
||||
return
|
||||
} else if b, err := strconv.ParseBool(v); err != nil || !b {
|
||||
c.ClickHouse.Enabled = false
|
||||
return
|
||||
}
|
||||
// Keep Enabled=true from env and continue applying host/pool defaults.
|
||||
}
|
||||
if !c.ClickHouse.Enabled {
|
||||
c.ClickHouse.Enabled = true
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/apps/openflare/chwriter"
|
||||
"github.com/Rain-kl/Wavelet/internal/db"
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if !db.ChConnReady() {
|
||||
fmt.Fprintln(os.Stderr, "ChConn not ready — check config.yaml clickhouse.enabled")
|
||||
os.Exit(1)
|
||||
}
|
||||
ctx := context.Background()
|
||||
chwriter.Init(ctx)
|
||||
now := time.Now().UTC()
|
||||
nodeID := "e2e-app-" + now.Format("150405")
|
||||
if err := model.InsertOpenFlareMetricSnapshot(ctx, &model.OpenFlareMetricSnapshot{
|
||||
NodeID: nodeID, CapturedAt: now, CPUUsagePercent: 41.2,
|
||||
MemoryUsedBytes: 123, MemoryTotalBytes: 1000,
|
||||
StorageUsedBytes: 456, StorageTotalBytes: 2000,
|
||||
DiskReadBytes: 11, DiskWriteBytes: 22, NetworkRxBytes: 33, NetworkTxBytes: 44,
|
||||
}); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "insert:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println("queued", nodeID)
|
||||
deadline := time.Now().Add(45 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
rows, err := model.ListOpenFlareMetricSnapshotsSince(ctx, nodeID, now.Add(-time.Minute), 5)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "list:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if len(rows) > 0 {
|
||||
fmt.Printf("OK flushed id=%d cpu=%.1f\n", rows[0].ID, rows[0].CPUUsagePercent)
|
||||
latest, err := model.ListOpenFlareLatestMetricSnapshotsSince(ctx, "", now.Add(-time.Hour))
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "latest:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
ok := false
|
||||
for _, r := range latest {
|
||||
if r != nil && r.NodeID == nodeID {
|
||||
ok = true
|
||||
}
|
||||
}
|
||||
if !ok {
|
||||
fmt.Fprintln(os.Stderr, "FAIL latest-per-node missing node")
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println("OK latest-per-node includes node")
|
||||
for _, s := range chwriter.WriterStats() {
|
||||
fmt.Printf("writer %s running=%v depth=%d drops=%d flush_err=%d\n",
|
||||
s.Name, s.Running, s.Depth, s.Drops, s.FlushErrors)
|
||||
}
|
||||
_ = chwriter.Stop(ctx)
|
||||
return
|
||||
}
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
fmt.Fprintln(os.Stderr, "FAIL: not flushed within 45s")
|
||||
os.Exit(1)
|
||||
}
|
||||
Reference in New Issue
Block a user