mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-10 17:26:38 +08:00
feat(clickhouse): integrate goose migrations and analytics repository
Add a separate ClickHouse OLAP pipeline with goose/clickhouse DDL as the sole schema source, model/analytics for ORM mapping, and repository/analytics for reads (ChDB/GORM) and batch writes (ChConn). Refactor risk_control and admin/logs to use the repository layer instead of inline SQL. Remove the manual support-files DDL and document the workflow in database-migration skill.
This commit is contained in:
@@ -0,0 +1,77 @@
|
||||
// Copyright 2025 linux.do
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package migrator
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"embed"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/ClickHouse/clickhouse-go/v2"
|
||||
"github.com/Rain-kl/Wavelet/internal/config"
|
||||
"github.com/pressly/goose/v3"
|
||||
)
|
||||
|
||||
const (
|
||||
clickhouseMigrationDir = "goose/clickhouse"
|
||||
clickhouseGooseVersionTable = "goose_clickhouse_version"
|
||||
clickhouseMaxExecTime = 60
|
||||
clickhouseReadTimeoutFactor = 2
|
||||
)
|
||||
|
||||
// clickhouseMigrationFS contains SQL migrations under goose/clickhouse.
|
||||
//
|
||||
//go:embed goose/clickhouse/*.sql
|
||||
var clickhouseMigrationFS embed.FS
|
||||
|
||||
// MigrateClickHouse runs goose migrations against ClickHouse when enabled.
|
||||
func MigrateClickHouse() {
|
||||
if !config.Config.ClickHouse.Enabled {
|
||||
return
|
||||
}
|
||||
|
||||
cfg := config.Config.ClickHouse
|
||||
sqlDB := clickhouse.OpenDB(&clickhouse.Options{
|
||||
Addr: cfg.Hosts,
|
||||
Auth: clickhouse.Auth{
|
||||
Database: cfg.Database,
|
||||
Username: cfg.Username,
|
||||
Password: cfg.Password,
|
||||
},
|
||||
Settings: clickhouse.Settings{
|
||||
"max_execution_time": clickhouseMaxExecTime,
|
||||
},
|
||||
Compression: &clickhouse.Compression{
|
||||
Method: clickhouse.CompressionLZ4,
|
||||
},
|
||||
DialTimeout: time.Duration(cfg.DialTimeout) * time.Second,
|
||||
MaxOpenConns: cfg.MaxOpenConn,
|
||||
MaxIdleConns: cfg.MaxIdleConn,
|
||||
ConnMaxLifetime: time.Duration(cfg.ConnMaxLifetime) * time.Second,
|
||||
ReadTimeout: time.Duration(cfg.DialTimeout*clickhouseReadTimeoutFactor) * time.Second,
|
||||
BlockBufferSize: cfg.BlockBufferSize,
|
||||
})
|
||||
|
||||
goose.SetBaseFS(clickhouseMigrationFS)
|
||||
if err := goose.SetDialect("clickhouse"); err != nil {
|
||||
closeClickHouseDB(sqlDB)
|
||||
log.Fatalf("[ClickHouse] set goose dialect failed: %v\n", err)
|
||||
}
|
||||
goose.SetTableName(clickhouseGooseVersionTable)
|
||||
if err := goose.Up(sqlDB, clickhouseMigrationDir); err != nil {
|
||||
closeClickHouseDB(sqlDB)
|
||||
log.Fatalf("[ClickHouse] goose migrate failed: %v\n", err)
|
||||
}
|
||||
closeClickHouseDB(sqlDB)
|
||||
|
||||
log.Println("[ClickHouse] goose migrate success")
|
||||
}
|
||||
|
||||
func closeClickHouseDB(sqlDB *sql.DB) {
|
||||
if err := sqlDB.Close(); err != nil {
|
||||
log.Printf("[ClickHouse] close sql db failed: %v\n", err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user