mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-11 01:36:37 +08:00
fix(docker): make ClickHouse startable on 25.x and reachable from host
Lower merge-tree free-entry thresholds for small background pools, bind listen_host to 0.0.0.0 for published ports, and allow CLICKHOUSE_ENABLED=true in tests for live_ch smoke coverage.
This commit is contained in:
@@ -0,0 +1,5 @@
|
|||||||
|
<?xml version="1.0"?>
|
||||||
|
<!-- Allow Docker published ports to reach native/HTTP interfaces (IPv4). -->
|
||||||
|
<clickhouse>
|
||||||
|
<listen_host>0.0.0.0</listen_host>
|
||||||
|
</clickhouse>
|
||||||
@@ -1,17 +1,25 @@
|
|||||||
<?xml version="1.0"?>
|
<?xml version="1.0"?>
|
||||||
<!--
|
<!--
|
||||||
Tuned for small control-plane hosts (e.g. 3c6g).
|
Tuned for small control-plane hosts (e.g. 3c6g).
|
||||||
background_pool_size should stay near the vCPU count; oversized pools cause
|
|
||||||
idle merge threads to burn CPU while absorbing small parts from heartbeats.
|
background_pool_size * background_merges_mutations_concurrency_ratio must stay
|
||||||
|
greater than merge_tree number_of_free_entries_in_pool_to_execute_mutation
|
||||||
|
(ClickHouse 25.x refuses to start otherwise). Keep the merge free-entry
|
||||||
|
thresholds low so a small pool remains valid.
|
||||||
-->
|
-->
|
||||||
<clickhouse>
|
<clickhouse>
|
||||||
<max_concurrent_queries>20</max_concurrent_queries>
|
<max_concurrent_queries>20</max_concurrent_queries>
|
||||||
<background_pool_size>2</background_pool_size>
|
<background_pool_size>4</background_pool_size>
|
||||||
<background_merges_mutations_concurrency_ratio>1</background_merges_mutations_concurrency_ratio>
|
<background_merges_mutations_concurrency_ratio>2</background_merges_mutations_concurrency_ratio>
|
||||||
<background_schedule_pool_size>4</background_schedule_pool_size>
|
<background_schedule_pool_size>4</background_schedule_pool_size>
|
||||||
<background_common_pool_size>2</background_common_pool_size>
|
<background_common_pool_size>2</background_common_pool_size>
|
||||||
<background_fetches_pool_size>2</background_fetches_pool_size>
|
<background_fetches_pool_size>2</background_fetches_pool_size>
|
||||||
<background_move_pool_size>1</background_move_pool_size>
|
<background_move_pool_size>1</background_move_pool_size>
|
||||||
<mark_cache_size>268435456</mark_cache_size>
|
<mark_cache_size>268435456</mark_cache_size>
|
||||||
<uncompressed_cache_size>0</uncompressed_cache_size>
|
<uncompressed_cache_size>0</uncompressed_cache_size>
|
||||||
|
<merge_tree>
|
||||||
|
<number_of_free_entries_in_pool_to_execute_mutation>2</number_of_free_entries_in_pool_to_execute_mutation>
|
||||||
|
<number_of_free_entries_in_pool_to_lower_max_size_of_merge>2</number_of_free_entries_in_pool_to_lower_max_size_of_merge>
|
||||||
|
<number_of_free_entries_in_pool_to_execute_optimize_entire_partition>2</number_of_free_entries_in_pool_to_execute_optimize_entire_partition>
|
||||||
|
</merge_tree>
|
||||||
</clickhouse>
|
</clickhouse>
|
||||||
|
|||||||
@@ -27,6 +27,7 @@ sidebar: false
|
|||||||
- model 层通过 hooks 写入 CH,去除对 `chwriter` 的直接依赖。
|
- model 层通过 hooks 写入 CH,去除对 `chwriter` 的直接依赖。
|
||||||
- Dashboard 每节点最新指标改为 `LIMIT 1 BY node_id`;新增 metric/openresty 小时预聚合表与读路径优先 rollup。
|
- Dashboard 每节点最新指标改为 `LIMIT 1 BY node_id`;新增 metric/openresty 小时预聚合表与读路径优先 rollup。
|
||||||
- 小规格默认连接池下调;`async_insert_busy_timeout` 调至 2s;`of_node_traffic_hourly` 增加 30 天 TTL,UV 改为峰值窗口估计并修正前端文案。
|
- 小规格默认连接池下调;`async_insert_busy_timeout` 调至 2s;`of_node_traffic_hourly` 增加 30 天 TTL,UV 改为峰值窗口估计并修正前端文案。
|
||||||
|
- Docker ClickHouse:`performance.xml` 下调 merge free-entry 阈值以兼容小 `background_pool`(避免 25.x 启动 Code 36);增加 `listen_host=0.0.0.0` 以便宿主机访问映射端口。
|
||||||
|
|
||||||
## [v3.1.1] - 2026-07-06
|
## [v3.1.1] - 2026-07-06
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,86 @@
|
|||||||
|
//go:build live_ch
|
||||||
|
|
||||||
|
// Copyright 2026 Arctel.net
|
||||||
|
// SPDX-License-Identifier: Apache-2.0
|
||||||
|
|
||||||
|
package chwriter_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/Rain-kl/Wavelet/internal/apps/openflare/chwriter"
|
||||||
|
"github.com/Rain-kl/Wavelet/internal/db"
|
||||||
|
"github.com/Rain-kl/Wavelet/internal/model"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Run with Docker ClickHouse + config.yaml:
|
||||||
|
//
|
||||||
|
// go test -tags live_ch ./internal/apps/openflare/chwriter -run TestLiveAppWritePath -count=1 -timeout 2m
|
||||||
|
func TestLiveAppWritePath(t *testing.T) {
|
||||||
|
if !db.ChConnReady() {
|
||||||
|
t.Skip("ClickHouse connection not ready")
|
||||||
|
}
|
||||||
|
ctx := context.Background()
|
||||||
|
chwriter.Init(ctx)
|
||||||
|
|
||||||
|
now := time.Now().UTC()
|
||||||
|
nodeID := "e2e-app-write-" + now.Format("150405")
|
||||||
|
if err := model.InsertOpenFlareMetricSnapshot(ctx, &model.OpenFlareMetricSnapshot{
|
||||||
|
NodeID: nodeID,
|
||||||
|
CapturedAt: now,
|
||||||
|
CPUUsagePercent: 33.3,
|
||||||
|
MemoryUsedBytes: 111,
|
||||||
|
MemoryTotalBytes: 1000,
|
||||||
|
StorageUsedBytes: 222,
|
||||||
|
StorageTotalBytes: 2000,
|
||||||
|
DiskReadBytes: 10,
|
||||||
|
DiskWriteBytes: 20,
|
||||||
|
NetworkRxBytes: 30,
|
||||||
|
NetworkTxBytes: 40,
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("InsertOpenFlareMetricSnapshot: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
deadline := time.Now().Add(45 * time.Second)
|
||||||
|
var found bool
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
rows, err := model.ListOpenFlareMetricSnapshotsSince(ctx, nodeID, now.Add(-time.Minute), 10)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ListOpenFlareMetricSnapshotsSince: %v", err)
|
||||||
|
}
|
||||||
|
if len(rows) > 0 {
|
||||||
|
found = true
|
||||||
|
t.Logf("found snapshot id=%d cpu=%.1f after flush", rows[0].ID, rows[0].CPUUsagePercent)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
time.Sleep(2 * time.Second)
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Fatal("metric snapshot not visible in ClickHouse after flush wait")
|
||||||
|
}
|
||||||
|
|
||||||
|
latest, err := model.ListOpenFlareLatestMetricSnapshotsSince(ctx, "", now.Add(-time.Hour))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ListOpenFlareLatestMetricSnapshotsSince: %v", err)
|
||||||
|
}
|
||||||
|
var latestOK bool
|
||||||
|
for _, row := range latest {
|
||||||
|
if row != nil && row.NodeID == nodeID {
|
||||||
|
latestOK = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !latestOK {
|
||||||
|
t.Fatalf("latest-per-node query missing node %s (rows=%d)", nodeID, len(latest))
|
||||||
|
}
|
||||||
|
|
||||||
|
stats := chwriter.WriterStats()
|
||||||
|
if len(stats) == 0 {
|
||||||
|
t.Fatal("WriterStats empty after Init")
|
||||||
|
}
|
||||||
|
for _, s := range stats {
|
||||||
|
t.Logf("writer %s running=%v depth=%d drops=%d flush_err=%d", s.Name, s.Running, s.Depth, s.Drops, s.FlushErrors)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -118,9 +118,17 @@ func applyDefaults(c *configModel) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func applyClickHouseDefaults(c *configModel) {
|
func applyClickHouseDefaults(c *configModel) {
|
||||||
|
// Tests disable ClickHouse by default to avoid accidental connections.
|
||||||
|
// Opt in with CLICKHOUSE_ENABLED=true for live integration tests (e.g. -tags live_ch).
|
||||||
if isTest() {
|
if isTest() {
|
||||||
c.ClickHouse.Enabled = false
|
if v, ok := os.LookupEnv("CLICKHOUSE_ENABLED"); !ok {
|
||||||
return
|
c.ClickHouse.Enabled = false
|
||||||
|
return
|
||||||
|
} else if b, err := strconv.ParseBool(v); err != nil || !b {
|
||||||
|
c.ClickHouse.Enabled = false
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// Keep Enabled=true from env and continue applying host/pool defaults.
|
||||||
}
|
}
|
||||||
if !c.ClickHouse.Enabled {
|
if !c.ClickHouse.Enabled {
|
||||||
c.ClickHouse.Enabled = true
|
c.ClickHouse.Enabled = true
|
||||||
|
|||||||
@@ -0,0 +1,69 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/Rain-kl/Wavelet/internal/apps/openflare/chwriter"
|
||||||
|
"github.com/Rain-kl/Wavelet/internal/db"
|
||||||
|
"github.com/Rain-kl/Wavelet/internal/model"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
if !db.ChConnReady() {
|
||||||
|
fmt.Fprintln(os.Stderr, "ChConn not ready — check config.yaml clickhouse.enabled")
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
ctx := context.Background()
|
||||||
|
chwriter.Init(ctx)
|
||||||
|
now := time.Now().UTC()
|
||||||
|
nodeID := "e2e-app-" + now.Format("150405")
|
||||||
|
if err := model.InsertOpenFlareMetricSnapshot(ctx, &model.OpenFlareMetricSnapshot{
|
||||||
|
NodeID: nodeID, CapturedAt: now, CPUUsagePercent: 41.2,
|
||||||
|
MemoryUsedBytes: 123, MemoryTotalBytes: 1000,
|
||||||
|
StorageUsedBytes: 456, StorageTotalBytes: 2000,
|
||||||
|
DiskReadBytes: 11, DiskWriteBytes: 22, NetworkRxBytes: 33, NetworkTxBytes: 44,
|
||||||
|
}); err != nil {
|
||||||
|
fmt.Fprintln(os.Stderr, "insert:", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
fmt.Println("queued", nodeID)
|
||||||
|
deadline := time.Now().Add(45 * time.Second)
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
rows, err := model.ListOpenFlareMetricSnapshotsSince(ctx, nodeID, now.Add(-time.Minute), 5)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintln(os.Stderr, "list:", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
if len(rows) > 0 {
|
||||||
|
fmt.Printf("OK flushed id=%d cpu=%.1f\n", rows[0].ID, rows[0].CPUUsagePercent)
|
||||||
|
latest, err := model.ListOpenFlareLatestMetricSnapshotsSince(ctx, "", now.Add(-time.Hour))
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintln(os.Stderr, "latest:", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
ok := false
|
||||||
|
for _, r := range latest {
|
||||||
|
if r != nil && r.NodeID == nodeID {
|
||||||
|
ok = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !ok {
|
||||||
|
fmt.Fprintln(os.Stderr, "FAIL latest-per-node missing node")
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
fmt.Println("OK latest-per-node includes node")
|
||||||
|
for _, s := range chwriter.WriterStats() {
|
||||||
|
fmt.Printf("writer %s running=%v depth=%d drops=%d flush_err=%d\n",
|
||||||
|
s.Name, s.Running, s.Depth, s.Drops, s.FlushErrors)
|
||||||
|
}
|
||||||
|
_ = chwriter.Stop(ctx)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
time.Sleep(2 * time.Second)
|
||||||
|
}
|
||||||
|
fmt.Fprintln(os.Stderr, "FAIL: not flushed within 45s")
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user