feat(clickhouse): integrate goose migrations and analytics repository

Add a separate ClickHouse OLAP pipeline with goose/clickhouse DDL as the
sole schema source, model/analytics for ORM mapping, and repository/analytics
for reads (ChDB/GORM) and batch writes (ChConn). Refactor risk_control and
admin/logs to use the repository layer instead of inline SQL. Remove the
manual support-files DDL and document the workflow in database-migration skill.
This commit is contained in:
ryan
2026-06-19 11:56:58 +08:00
parent 13e9fead35
commit 00593742c3
22 changed files with 1033 additions and 352 deletions
+77
View File
@@ -0,0 +1,77 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package migrator
import (
"database/sql"
"embed"
"log"
"time"
"github.com/ClickHouse/clickhouse-go/v2"
"github.com/Rain-kl/Wavelet/internal/config"
"github.com/pressly/goose/v3"
)
const (
clickhouseMigrationDir = "goose/clickhouse"
clickhouseGooseVersionTable = "goose_clickhouse_version"
clickhouseMaxExecTime = 60
clickhouseReadTimeoutFactor = 2
)
// clickhouseMigrationFS contains SQL migrations under goose/clickhouse.
//
//go:embed goose/clickhouse/*.sql
var clickhouseMigrationFS embed.FS
// MigrateClickHouse runs goose migrations against ClickHouse when enabled.
func MigrateClickHouse() {
if !config.Config.ClickHouse.Enabled {
return
}
cfg := config.Config.ClickHouse
sqlDB := clickhouse.OpenDB(&clickhouse.Options{
Addr: cfg.Hosts,
Auth: clickhouse.Auth{
Database: cfg.Database,
Username: cfg.Username,
Password: cfg.Password,
},
Settings: clickhouse.Settings{
"max_execution_time": clickhouseMaxExecTime,
},
Compression: &clickhouse.Compression{
Method: clickhouse.CompressionLZ4,
},
DialTimeout: time.Duration(cfg.DialTimeout) * time.Second,
MaxOpenConns: cfg.MaxOpenConn,
MaxIdleConns: cfg.MaxIdleConn,
ConnMaxLifetime: time.Duration(cfg.ConnMaxLifetime) * time.Second,
ReadTimeout: time.Duration(cfg.DialTimeout*clickhouseReadTimeoutFactor) * time.Second,
BlockBufferSize: cfg.BlockBufferSize,
})
goose.SetBaseFS(clickhouseMigrationFS)
if err := goose.SetDialect("clickhouse"); err != nil {
closeClickHouseDB(sqlDB)
log.Fatalf("[ClickHouse] set goose dialect failed: %v\n", err)
}
goose.SetTableName(clickhouseGooseVersionTable)
if err := goose.Up(sqlDB, clickhouseMigrationDir); err != nil {
closeClickHouseDB(sqlDB)
log.Fatalf("[ClickHouse] goose migrate failed: %v\n", err)
}
closeClickHouseDB(sqlDB)
log.Println("[ClickHouse] goose migrate success")
}
func closeClickHouseDB(sqlDB *sql.DB) {
if err := sqlDB.Close(); err != nil {
log.Printf("[ClickHouse] close sql db failed: %v\n", err)
}
}
+51
View File
@@ -0,0 +1,51 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package migrator
import (
"testing"
"github.com/Rain-kl/Wavelet/internal/config"
"github.com/pressly/goose/v3"
)
func TestClickHouseMigrationFilesEmbedded(t *testing.T) {
entries, err := clickhouseMigrationFS.ReadDir(clickhouseMigrationDir)
if err != nil {
t.Fatalf("ReadDir(%q) error = %v", clickhouseMigrationDir, err)
}
if len(entries) == 0 {
t.Fatal("expected embedded ClickHouse migrations, got none")
}
found := false
for _, entry := range entries {
if entry.IsDir() {
continue
}
if entry.Name() == "202606190001_create_user_access_logs.sql" {
found = true
break
}
}
if !found {
t.Fatal("expected 202606190001_create_user_access_logs.sql in embedded migrations")
}
}
func TestClickHouseGooseDialect(t *testing.T) {
if err := goose.SetDialect("clickhouse"); err != nil {
t.Fatalf("SetDialect(clickhouse) error = %v", err)
}
}
func TestMigrateClickHouseSkipsWhenDisabled(t *testing.T) {
previousEnabled := config.Config.ClickHouse.Enabled
config.Config.ClickHouse.Enabled = false
t.Cleanup(func() {
config.Config.ClickHouse.Enabled = previousEnabled
})
MigrateClickHouse()
}
@@ -0,0 +1,21 @@
-- +goose Up
CREATE TABLE IF NOT EXISTS w_user_access_logs
(
id UInt64,
user_id UInt64,
path String,
method String,
ip String,
user_agent String,
headers String,
status Int32,
latency Int64,
created_at DateTime
)
ENGINE = MergeTree()
PARTITION BY toYYYYMM(created_at)
ORDER BY (created_at, ip, user_id)
SETTINGS index_granularity = 8192;
-- +goose Down
DROP TABLE IF EXISTS w_user_access_logs;