mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-08 08:36:37 +08:00
feat(clickhouse): integrate goose migrations and analytics repository
Add a separate ClickHouse OLAP pipeline with goose/clickhouse DDL as the sole schema source, model/analytics for ORM mapping, and repository/analytics for reads (ChDB/GORM) and batch writes (ChConn). Refactor risk_control and admin/logs to use the repository layer instead of inline SQL. Remove the manual support-files DDL and document the workflow in database-migration skill.
This commit is contained in:
+98
-15
@@ -7,12 +7,20 @@ package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/ClickHouse/clickhouse-go/v2"
|
||||
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
|
||||
"github.com/Rain-kl/Wavelet/internal/config"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
clickhouseDriver "gorm.io/driver/clickhouse"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/plugin/opentelemetry/tracing"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -21,8 +29,10 @@ const (
|
||||
)
|
||||
|
||||
var (
|
||||
// ChConn ClickHouse 连接实例
|
||||
// ChConn ClickHouse 原生连接实例,用于批量写入
|
||||
ChConn driver.Conn
|
||||
|
||||
chDB *gorm.DB
|
||||
)
|
||||
|
||||
func init() {
|
||||
@@ -31,10 +41,55 @@ func init() {
|
||||
}
|
||||
|
||||
cfg := config.Config.ClickHouse
|
||||
var err error
|
||||
opts := buildClickHouseOptions()
|
||||
|
||||
// 配置 ClickHouse 连接
|
||||
ChConn, err = clickhouse.Open(&clickhouse.Options{
|
||||
var err error
|
||||
ChConn, err = clickhouse.Open(opts)
|
||||
if err != nil {
|
||||
log.Fatalf("[ClickHouse] init connection failed: %v\n", err)
|
||||
}
|
||||
|
||||
if err = ChConn.Ping(context.Background()); err != nil {
|
||||
log.Fatalf("[ClickHouse] ping failed: %v\n", err)
|
||||
}
|
||||
|
||||
chDB, err = gorm.Open(clickhouseDriver.New(clickhouseDriver.Config{
|
||||
DSN: buildClickHouseDSN(),
|
||||
}), &gorm.Config{
|
||||
SkipDefaultTransaction: true,
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatalf("[ClickHouse] init gorm connection failed: %v\n", err)
|
||||
}
|
||||
|
||||
if err = chDB.Use(
|
||||
tracing.NewPlugin(
|
||||
tracing.WithoutMetrics(),
|
||||
tracing.WithAttributes(
|
||||
attribute.String("db.instance", cfg.Database),
|
||||
attribute.String("db.system", "ClickHouse"),
|
||||
),
|
||||
),
|
||||
); err != nil {
|
||||
log.Fatalf("[ClickHouse] init trace failed: %v\n", err)
|
||||
}
|
||||
|
||||
sqlDB, err := chDB.DB()
|
||||
if err != nil {
|
||||
log.Fatalf("[ClickHouse] load sql db failed: %v\n", err)
|
||||
}
|
||||
|
||||
sqlDB.SetMaxIdleConns(cfg.MaxIdleConn)
|
||||
sqlDB.SetMaxOpenConns(cfg.MaxOpenConn)
|
||||
sqlDB.SetConnMaxLifetime(time.Duration(cfg.ConnMaxLifetime) * time.Second)
|
||||
|
||||
log.Println("[ClickHouse] connection established successfully")
|
||||
}
|
||||
|
||||
func buildClickHouseOptions() *clickhouse.Options {
|
||||
cfg := config.Config.ClickHouse
|
||||
|
||||
return &clickhouse.Options{
|
||||
Addr: cfg.Hosts,
|
||||
Auth: clickhouse.Auth{
|
||||
Database: cfg.Database,
|
||||
@@ -53,16 +108,44 @@ func init() {
|
||||
ConnMaxLifetime: time.Duration(cfg.ConnMaxLifetime) * time.Second,
|
||||
ReadTimeout: time.Duration(cfg.DialTimeout*clickhouseReadTimeoutFactor) * time.Second,
|
||||
BlockBufferSize: cfg.BlockBufferSize,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
log.Fatalf("[ClickHouse] init connection failed: %v\n", err)
|
||||
}
|
||||
|
||||
// 测试连接
|
||||
if err = ChConn.Ping(context.Background()); err != nil {
|
||||
log.Fatalf("[ClickHouse] ping failed: %v\n", err)
|
||||
}
|
||||
|
||||
log.Println("[ClickHouse] connection established successfully")
|
||||
}
|
||||
|
||||
func buildClickHouseDSN() string {
|
||||
cfg := config.Config.ClickHouse
|
||||
|
||||
chURL := &url.URL{
|
||||
Scheme: "clickhouse",
|
||||
Host: strings.Join(cfg.Hosts, ","),
|
||||
Path: "/" + cfg.Database,
|
||||
}
|
||||
if cfg.Username != "" || cfg.Password != "" {
|
||||
chURL.User = url.UserPassword(cfg.Username, cfg.Password)
|
||||
}
|
||||
|
||||
query := chURL.Query()
|
||||
query.Set("dial_timeout", fmt.Sprintf("%ds", cfg.DialTimeout))
|
||||
query.Set("read_timeout", fmt.Sprintf("%ds", cfg.DialTimeout*clickhouseReadTimeoutFactor))
|
||||
query.Set("max_execution_time", strconv.Itoa(clickhouseMaxExecTime))
|
||||
chURL.RawQuery = query.Encode()
|
||||
|
||||
return chURL.String()
|
||||
}
|
||||
|
||||
// ChDB returns a context-aware GORM ClickHouse instance.
|
||||
func ChDB(ctx context.Context) *gorm.DB {
|
||||
if chDB == nil {
|
||||
return nil
|
||||
}
|
||||
return chDB.WithContext(ctx)
|
||||
}
|
||||
|
||||
// SetChDBForTest sets the package-level ClickHouse GORM instance for testing.
|
||||
func SetChDBForTest(d *gorm.DB) {
|
||||
chDB = d
|
||||
}
|
||||
|
||||
// SetChConnForTest sets the package-level native ClickHouse connection for testing.
|
||||
func SetChConnForTest(c driver.Conn) {
|
||||
ChConn = c
|
||||
}
|
||||
Reference in New Issue
Block a user