tcjlj/OpenFlare
1
0
Fork 0
mirror of https://github.com/Rain-kl/OpenFlare.git synced 2026-10-08 08:36:37 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(clickhouse): integrate goose migrations and analytics repository

Browse Source 
Merge upstream Wavelet patch to manage ClickHouse DDL via goose
(goose_clickhouse_version), add GORM ChDB access, and centralize
w_user_access_logs reads/writes in internal/repository/analytics.
OpenFlare of_node_access_logs DDL moves to goose/clickhouse; remove
manual clickhouse_schema startup and support-files SQL duplicates.
This commit is contained in:
ryan
2026-06-19 12:11:41 +08:00
parent fff26aa343
commit 8a00f53b16
25 changed files with 1095 additions and 421 deletions
Download Patch File Download Diff File Expand all files Collapse all files
openflare-server/internal/db/clickhouse.go
+98 -16
View File
@@ -7,12 +7,20 @@ package db
import (
"context"
"fmt"
"log"
"net/url"
"strconv"
"strings"
"time"
"github.com/ClickHouse/clickhouse-go/v2"
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
"github.com/Rain-kl/Wavelet/internal/config"
"go.opentelemetry.io/otel/attribute"
clickhouseDriver "gorm.io/driver/clickhouse"
"gorm.io/gorm"
"gorm.io/plugin/opentelemetry/tracing"
)
const (
@@ -21,8 +29,10 @@ const (
)
var (
// ChConn ClickHouse 连接实例
// ChConn ClickHouse 原生连接实例，用于批量写入
ChConn driver.Conn
chDB *gorm.DB
)
func init() {
@@ -35,10 +45,55 @@ func init() {
log.Fatalf("[ClickHouse] database name is required (expected: openflare)\n")
}
var err error
opts := buildClickHouseOptions()
// 配置 ClickHouse 连接
ChConn, err = clickhouse.Open(&clickhouse.Options{
var err error
ChConn, err = clickhouse.Open(opts)
if err != nil {
log.Fatalf("[ClickHouse] init connection failed: %v\n", err)
}
if err = ChConn.Ping(context.Background()); err != nil {
log.Fatalf("[ClickHouse] ping failed: %v\n", err)
}
chDB, err = gorm.Open(clickhouseDriver.New(clickhouseDriver.Config{
DSN: buildClickHouseDSN(),
}), &gorm.Config{
SkipDefaultTransaction: true,
})
if err != nil {
log.Fatalf("[ClickHouse] init gorm connection failed: %v\n", err)
}
if err = chDB.Use(
tracing.NewPlugin(
tracing.WithoutMetrics(),
tracing.WithAttributes(
attribute.String("db.instance", cfg.Database),
attribute.String("db.system", "ClickHouse"),
),
),
); err != nil {
log.Fatalf("[ClickHouse] init trace failed: %v\n", err)
}
sqlDB, err := chDB.DB()
if err != nil {
log.Fatalf("[ClickHouse] load sql db failed: %v\n", err)
}
sqlDB.SetMaxIdleConns(cfg.MaxIdleConn)
sqlDB.SetMaxOpenConns(cfg.MaxOpenConn)
sqlDB.SetConnMaxLifetime(time.Duration(cfg.ConnMaxLifetime) * time.Second)
log.Println("[ClickHouse] connection established successfully")
}
func buildClickHouseOptions() *clickhouse.Options {
cfg := config.Config.ClickHouse
return &clickhouse.Options{
Addr: cfg.Hosts,
Auth: clickhouse.Auth{
Database: cfg.Database,
@@ -57,17 +112,44 @@ func init() {
ConnMaxLifetime: time.Duration(cfg.ConnMaxLifetime) * time.Second,
ReadTimeout: time.Duration(cfg.DialTimeout*clickhouseReadTimeoutFactor) * time.Second,
BlockBufferSize: cfg.BlockBufferSize,
})
if err != nil {
log.Fatalf("[ClickHouse] init connection failed: %v\n", err)
}
// 测试连接
if err = ChConn.Ping(context.Background()); err != nil {
log.Fatalf("[ClickHouse] ping failed: %v\n", err)
}
ensureClickHouseSchemaOnStartup()
log.Println("[ClickHouse] connection established successfully")
}
func buildClickHouseDSN() string {
cfg := config.Config.ClickHouse
chURL := &url.URL{
Scheme: "clickhouse",
Host: strings.Join(cfg.Hosts, ","),
Path: "/" + cfg.Database,
}
if cfg.Username != "" || cfg.Password != "" {
chURL.User = url.UserPassword(cfg.Username, cfg.Password)
}
query := chURL.Query()
query.Set("dial_timeout", fmt.Sprintf("%ds", cfg.DialTimeout))
query.Set("read_timeout", fmt.Sprintf("%ds", cfg.DialTimeout*clickhouseReadTimeoutFactor))
query.Set("max_execution_time", strconv.Itoa(clickhouseMaxExecTime))
chURL.RawQuery = query.Encode()
return chURL.String()
}
// ChDB returns a context-aware GORM ClickHouse instance.
func ChDB(ctx context.Context) *gorm.DB {
if chDB == nil {
return nil
}
return chDB.WithContext(ctx)
}
// SetChDBForTest sets the package-level ClickHouse GORM instance for testing.
func SetChDBForTest(d *gorm.DB) {
chDB = d
}
// SetChConnForTest sets the package-level native ClickHouse connection for testing.
func SetChConnForTest(c driver.Conn) {
ChConn = c
}
openflare-server/internal/db/clickhouse_schema.go
-51
View File
@@ -1,51 +0,0 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package db
import (
"context"
"fmt"
"log"
)
const openFlareNodeAccessLogsDDL = `
CREATE TABLE IF NOT EXISTS of_node_access_logs
(
id UInt64,
node_id String,
logged_at DateTime64(3, 'UTC'),
remote_addr String,
region String,
host String,
path String,
status_code Int32,
created_at DateTime64(3, 'UTC')
)
ENGINE = MergeTree()
PARTITION BY toYYYYMM(logged_at)
ORDER BY (node_id, logged_at, remote_addr, host, path, status_code)
SETTINGS index_granularity = 8192`
// EnsureClickHouseSchema creates the openflare database and required tables.
func EnsureClickHouseSchema(ctx context.Context) error {
if ChConn == nil {
return fmt.Errorf("clickhouse connection is not initialized")
}
if err := ChConn.Exec(ctx, "CREATE DATABASE IF NOT EXISTS openflare"); err != nil {
return fmt.Errorf("create database openflare: %w", err)
}
if err := ChConn.Exec(ctx, openFlareNodeAccessLogsDDL); err != nil {
return fmt.Errorf("create table of_node_access_logs: %w", err)
}
return nil
}
func ensureClickHouseSchemaOnStartup() {
ctx := context.Background()
if err := EnsureClickHouseSchema(ctx); err != nil {
log.Fatalf("[ClickHouse] ensure schema failed: %v\n", err)
}
log.Println("[ClickHouse] schema ready (database: openflare)")
}
openflare-server/internal/db/migrator/clickhouse.go
+77
View File
@@ -0,0 +1,77 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package migrator
import (
"database/sql"
"embed"
"log"
"time"
"github.com/ClickHouse/clickhouse-go/v2"
"github.com/Rain-kl/Wavelet/internal/config"
"github.com/pressly/goose/v3"
)
const (
clickhouseMigrationDir = "goose/clickhouse"
clickhouseGooseVersionTable = "goose_clickhouse_version"
clickhouseMaxExecTime = 60
clickhouseReadTimeoutFactor = 2
)
// clickhouseMigrationFS contains SQL migrations under goose/clickhouse.
//
//go:embed goose/clickhouse/*.sql
var clickhouseMigrationFS embed.FS
// MigrateClickHouse runs goose migrations against ClickHouse when enabled.
func MigrateClickHouse() {
if !config.Config.ClickHouse.Enabled {
return
}
cfg := config.Config.ClickHouse
sqlDB := clickhouse.OpenDB(&clickhouse.Options{
Addr: cfg.Hosts,
Auth: clickhouse.Auth{
Database: cfg.Database,
Username: cfg.Username,
Password: cfg.Password,
},
Settings: clickhouse.Settings{
"max_execution_time": clickhouseMaxExecTime,
},
Compression: &clickhouse.Compression{
Method: clickhouse.CompressionLZ4,
},
DialTimeout: time.Duration(cfg.DialTimeout) * time.Second,
MaxOpenConns: cfg.MaxOpenConn,
MaxIdleConns: cfg.MaxIdleConn,
ConnMaxLifetime: time.Duration(cfg.ConnMaxLifetime) * time.Second,
ReadTimeout: time.Duration(cfg.DialTimeout*clickhouseReadTimeoutFactor) * time.Second,
BlockBufferSize: cfg.BlockBufferSize,
})
goose.SetBaseFS(clickhouseMigrationFS)
if err := goose.SetDialect("clickhouse"); err != nil {
closeClickHouseDB(sqlDB)
log.Fatalf("[ClickHouse] set goose dialect failed: %v\n", err)
}
goose.SetTableName(clickhouseGooseVersionTable)
if err := goose.Up(sqlDB, clickhouseMigrationDir); err != nil {
closeClickHouseDB(sqlDB)
log.Fatalf("[ClickHouse] goose migrate failed: %v\n", err)
}
closeClickHouseDB(sqlDB)
log.Println("[ClickHouse] goose migrate success")
}
func closeClickHouseDB(sqlDB *sql.DB) {
if err := sqlDB.Close(); err != nil {
log.Printf("[ClickHouse] close sql db failed: %v\n", err)
}
}
openflare-server/internal/db/migrator/clickhouse_test.go
+55
View File
@@ -0,0 +1,55 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package migrator
import (
"testing"
"github.com/Rain-kl/Wavelet/internal/config"
"github.com/pressly/goose/v3"
)
func TestClickHouseMigrationFilesEmbedded(t *testing.T) {
entries, err := clickhouseMigrationFS.ReadDir(clickhouseMigrationDir)
if err != nil {
t.Fatalf("ReadDir(%q) error = %v", clickhouseMigrationDir, err)
}
if len(entries) == 0 {
t.Fatal("expected embedded ClickHouse migrations, got none")
}
expected := map[string]bool{
"202606190001_create_user_access_logs.sql": false,
"202606200001_create_node_access_logs.sql": false,
}
for _, entry := range entries {
if entry.IsDir() {
continue
}
if _, ok := expected[entry.Name()]; ok {
expected[entry.Name()] = true
}
}
for name, found := range expected {
if !found {
t.Fatalf("expected %s in embedded migrations", name)
}
}
}
func TestClickHouseGooseDialect(t *testing.T) {
if err := goose.SetDialect("clickhouse"); err != nil {
t.Fatalf("SetDialect(clickhouse) error = %v", err)
}
}
func TestMigrateClickHouseSkipsWhenDisabled(t *testing.T) {
previousEnabled := config.Config.ClickHouse.Enabled
config.Config.ClickHouse.Enabled = false
t.Cleanup(func() {
config.Config.ClickHouse.Enabled = previousEnabled
})
MigrateClickHouse()
}
openflare-server/internal/db/migrator/goose/clickhouse/202606190001_create_user_access_logs.sql
+21
View File
@@ -0,0 +1,21 @@
-- +goose Up
CREATE TABLE IF NOT EXISTS w_user_access_logs
(
id UInt64,
user_id UInt64,
path String,
method String,
ip String,
user_agent String,
headers String,
status Int32,
latency Int64,
created_at DateTime
)
ENGINE = MergeTree()
PARTITION BY toYYYYMM(created_at)
ORDER BY (created_at, ip, user_id)
SETTINGS index_granularity = 8192;
-- +goose Down
DROP TABLE IF EXISTS w_user_access_logs;
openflare-server/internal/db/migrator/goose/clickhouse/202606200001_create_node_access_logs.sql
+20
View File
@@ -0,0 +1,20 @@
-- +goose Up
CREATE TABLE IF NOT EXISTS of_node_access_logs
(
id UInt64,
node_id String,
logged_at DateTime64(3, 'UTC'),
remote_addr String,
region String,
host String,
path String,
status_code Int32,
created_at DateTime64(3, 'UTC')
)
ENGINE = MergeTree()
PARTITION BY toYYYYMM(logged_at)
ORDER BY (node_id, logged_at, remote_addr, host, path, status_code)
SETTINGS index_granularity = 8192;
-- +goose Down
DROP TABLE IF EXISTS of_node_access_logs;