feat(clickhouse): integrate goose migrations and analytics repository

Add a separate ClickHouse OLAP pipeline with goose/clickhouse DDL as the
sole schema source, model/analytics for ORM mapping, and repository/analytics
for reads (ChDB/GORM) and batch writes (ChConn). Refactor risk_control and
admin/logs to use the repository layer instead of inline SQL. Remove the
manual support-files DDL and document the workflow in database-migration skill.
This commit is contained in:
ryan
2026-06-19 11:56:58 +08:00
parent 13e9fead35
commit 00593742c3
22 changed files with 1033 additions and 352 deletions
+11 -35
View File
@@ -8,11 +8,12 @@ import (
"time"
"github.com/Rain-kl/Wavelet/internal/config"
"github.com/Rain-kl/Wavelet/internal/db"
"github.com/Rain-kl/Wavelet/internal/model/analytics"
analyticsrepo "github.com/Rain-kl/Wavelet/internal/repository/analytics"
"github.com/Rain-kl/Wavelet/pkg/logger"
)
var logChan chan *UserAccessLog
var logChan chan *analytics.UserAccessLog
const (
defaultQueueSize = 10000
@@ -26,7 +27,7 @@ func InitLogWriter(ctx context.Context) {
return
}
logChan = make(chan *UserAccessLog, defaultQueueSize)
logChan = make(chan *analytics.UserAccessLog, defaultQueueSize)
go startBatchWorker(context.WithoutCancel(ctx))
}
@@ -40,7 +41,7 @@ func IsBufferFull() bool {
}
// QueueAccessLog 异步非阻塞地将日志推入缓冲队列
func QueueAccessLog(logItem *UserAccessLog) {
func QueueAccessLog(logItem *analytics.UserAccessLog) {
if !config.Config.ClickHouse.Enabled || logChan == nil {
return
}
@@ -56,43 +57,18 @@ func startBatchWorker(ctx context.Context) {
ticker := time.NewTicker(flushInterval)
defer ticker.Stop()
var batch []*UserAccessLog
var batch []*analytics.UserAccessLog
flush := func() {
if len(batch) == 0 {
return
}
if db.ChConn == nil {
batch = nil
return
}
b, err := db.ChConn.PrepareBatch(ctx, "INSERT INTO w_user_access_logs (id, user_id, path, method, ip, user_agent, headers, status, latency, created_at)")
if err != nil {
logger.ErrorF(ctx, "[RiskControl] Prepare ClickHouse batch failed: %v", err)
batch = nil
return
items := make([]analytics.UserAccessLog, len(batch))
for i, item := range batch {
items[i] = *item
}
for _, item := range batch {
err = b.Append(
item.ID,
item.UserID,
item.Path,
item.Method,
item.IP,
item.UserAgent,
item.Headers,
item.Status,
item.Latency,
item.CreatedAt,
)
if err != nil {
logger.ErrorF(ctx, "[RiskControl] Append item to ClickHouse batch failed: %v", err)
}
}
if err := b.Send(); err != nil {
if err := analyticsrepo.BatchInsert(ctx, items); err != nil {
logger.ErrorF(ctx, "[RiskControl] Send ClickHouse batch failed: %v", err)
}
batch = nil
@@ -113,4 +89,4 @@ func startBatchWorker(ctx context.Context) {
flush()
}
}
}
}