mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-11 01:36:37 +08:00
feat(clickhouse): integrate goose migrations and analytics repository
Merge upstream Wavelet patch to manage ClickHouse DDL via goose (goose_clickhouse_version), add GORM ChDB access, and centralize w_user_access_logs reads/writes in internal/repository/analytics. OpenFlare of_node_access_logs DDL moves to goose/clickhouse; remove manual clickhouse_schema startup and support-files SQL duplicates.
This commit is contained in:
@@ -0,0 +1,77 @@
|
||||
// Copyright 2025 linux.do
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package migrator
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"embed"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/ClickHouse/clickhouse-go/v2"
|
||||
"github.com/Rain-kl/Wavelet/internal/config"
|
||||
"github.com/pressly/goose/v3"
|
||||
)
|
||||
|
||||
const (
|
||||
clickhouseMigrationDir = "goose/clickhouse"
|
||||
clickhouseGooseVersionTable = "goose_clickhouse_version"
|
||||
clickhouseMaxExecTime = 60
|
||||
clickhouseReadTimeoutFactor = 2
|
||||
)
|
||||
|
||||
// clickhouseMigrationFS contains SQL migrations under goose/clickhouse.
|
||||
//
|
||||
//go:embed goose/clickhouse/*.sql
|
||||
var clickhouseMigrationFS embed.FS
|
||||
|
||||
// MigrateClickHouse runs goose migrations against ClickHouse when enabled.
|
||||
func MigrateClickHouse() {
|
||||
if !config.Config.ClickHouse.Enabled {
|
||||
return
|
||||
}
|
||||
|
||||
cfg := config.Config.ClickHouse
|
||||
sqlDB := clickhouse.OpenDB(&clickhouse.Options{
|
||||
Addr: cfg.Hosts,
|
||||
Auth: clickhouse.Auth{
|
||||
Database: cfg.Database,
|
||||
Username: cfg.Username,
|
||||
Password: cfg.Password,
|
||||
},
|
||||
Settings: clickhouse.Settings{
|
||||
"max_execution_time": clickhouseMaxExecTime,
|
||||
},
|
||||
Compression: &clickhouse.Compression{
|
||||
Method: clickhouse.CompressionLZ4,
|
||||
},
|
||||
DialTimeout: time.Duration(cfg.DialTimeout) * time.Second,
|
||||
MaxOpenConns: cfg.MaxOpenConn,
|
||||
MaxIdleConns: cfg.MaxIdleConn,
|
||||
ConnMaxLifetime: time.Duration(cfg.ConnMaxLifetime) * time.Second,
|
||||
ReadTimeout: time.Duration(cfg.DialTimeout*clickhouseReadTimeoutFactor) * time.Second,
|
||||
BlockBufferSize: cfg.BlockBufferSize,
|
||||
})
|
||||
|
||||
goose.SetBaseFS(clickhouseMigrationFS)
|
||||
if err := goose.SetDialect("clickhouse"); err != nil {
|
||||
closeClickHouseDB(sqlDB)
|
||||
log.Fatalf("[ClickHouse] set goose dialect failed: %v\n", err)
|
||||
}
|
||||
goose.SetTableName(clickhouseGooseVersionTable)
|
||||
if err := goose.Up(sqlDB, clickhouseMigrationDir); err != nil {
|
||||
closeClickHouseDB(sqlDB)
|
||||
log.Fatalf("[ClickHouse] goose migrate failed: %v\n", err)
|
||||
}
|
||||
closeClickHouseDB(sqlDB)
|
||||
|
||||
log.Println("[ClickHouse] goose migrate success")
|
||||
}
|
||||
|
||||
func closeClickHouseDB(sqlDB *sql.DB) {
|
||||
if err := sqlDB.Close(); err != nil {
|
||||
log.Printf("[ClickHouse] close sql db failed: %v\n", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package migrator
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/config"
|
||||
"github.com/pressly/goose/v3"
|
||||
)
|
||||
|
||||
func TestClickHouseMigrationFilesEmbedded(t *testing.T) {
|
||||
entries, err := clickhouseMigrationFS.ReadDir(clickhouseMigrationDir)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadDir(%q) error = %v", clickhouseMigrationDir, err)
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
t.Fatal("expected embedded ClickHouse migrations, got none")
|
||||
}
|
||||
|
||||
expected := map[string]bool{
|
||||
"202606190001_create_user_access_logs.sql": false,
|
||||
"202606200001_create_node_access_logs.sql": false,
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
if _, ok := expected[entry.Name()]; ok {
|
||||
expected[entry.Name()] = true
|
||||
}
|
||||
}
|
||||
for name, found := range expected {
|
||||
if !found {
|
||||
t.Fatalf("expected %s in embedded migrations", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestClickHouseGooseDialect(t *testing.T) {
|
||||
if err := goose.SetDialect("clickhouse"); err != nil {
|
||||
t.Fatalf("SetDialect(clickhouse) error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrateClickHouseSkipsWhenDisabled(t *testing.T) {
|
||||
previousEnabled := config.Config.ClickHouse.Enabled
|
||||
config.Config.ClickHouse.Enabled = false
|
||||
t.Cleanup(func() {
|
||||
config.Config.ClickHouse.Enabled = previousEnabled
|
||||
})
|
||||
|
||||
MigrateClickHouse()
|
||||
}
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
-- +goose Up
|
||||
CREATE TABLE IF NOT EXISTS w_user_access_logs
|
||||
(
|
||||
id UInt64,
|
||||
user_id UInt64,
|
||||
path String,
|
||||
method String,
|
||||
ip String,
|
||||
user_agent String,
|
||||
headers String,
|
||||
status Int32,
|
||||
latency Int64,
|
||||
created_at DateTime
|
||||
)
|
||||
ENGINE = MergeTree()
|
||||
PARTITION BY toYYYYMM(created_at)
|
||||
ORDER BY (created_at, ip, user_id)
|
||||
SETTINGS index_granularity = 8192;
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE IF EXISTS w_user_access_logs;
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
-- +goose Up
|
||||
CREATE TABLE IF NOT EXISTS of_node_access_logs
|
||||
(
|
||||
id UInt64,
|
||||
node_id String,
|
||||
logged_at DateTime64(3, 'UTC'),
|
||||
remote_addr String,
|
||||
region String,
|
||||
host String,
|
||||
path String,
|
||||
status_code Int32,
|
||||
created_at DateTime64(3, 'UTC')
|
||||
)
|
||||
ENGINE = MergeTree()
|
||||
PARTITION BY toYYYYMM(logged_at)
|
||||
ORDER BY (node_id, logged_at, remote_addr, host, path, status_code)
|
||||
SETTINGS index_granularity = 8192;
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE IF EXISTS of_node_access_logs;
|
||||
Reference in New Issue
Block a user