feat(clickhouse): integrate goose migrations and analytics repository

Add a separate ClickHouse OLAP pipeline with goose/clickhouse DDL as the
sole schema source, model/analytics for ORM mapping, and repository/analytics
for reads (ChDB/GORM) and batch writes (ChConn). Refactor risk_control and
admin/logs to use the repository layer instead of inline SQL. Remove the
manual support-files DDL and document the workflow in database-migration skill.
This commit is contained in:
ryan
2026-06-19 11:56:58 +08:00
parent 13e9fead35
commit 00593742c3
22 changed files with 1033 additions and 352 deletions
+11 -35
View File
@@ -8,11 +8,12 @@ import (
"time"
"github.com/Rain-kl/Wavelet/internal/config"
"github.com/Rain-kl/Wavelet/internal/db"
"github.com/Rain-kl/Wavelet/internal/model/analytics"
analyticsrepo "github.com/Rain-kl/Wavelet/internal/repository/analytics"
"github.com/Rain-kl/Wavelet/pkg/logger"
)
var logChan chan *UserAccessLog
var logChan chan *analytics.UserAccessLog
const (
defaultQueueSize = 10000
@@ -26,7 +27,7 @@ func InitLogWriter(ctx context.Context) {
return
}
logChan = make(chan *UserAccessLog, defaultQueueSize)
logChan = make(chan *analytics.UserAccessLog, defaultQueueSize)
go startBatchWorker(context.WithoutCancel(ctx))
}
@@ -40,7 +41,7 @@ func IsBufferFull() bool {
}
// QueueAccessLog 异步非阻塞地将日志推入缓冲队列
func QueueAccessLog(logItem *UserAccessLog) {
func QueueAccessLog(logItem *analytics.UserAccessLog) {
if !config.Config.ClickHouse.Enabled || logChan == nil {
return
}
@@ -56,43 +57,18 @@ func startBatchWorker(ctx context.Context) {
ticker := time.NewTicker(flushInterval)
defer ticker.Stop()
var batch []*UserAccessLog
var batch []*analytics.UserAccessLog
flush := func() {
if len(batch) == 0 {
return
}
if db.ChConn == nil {
batch = nil
return
}
b, err := db.ChConn.PrepareBatch(ctx, "INSERT INTO w_user_access_logs (id, user_id, path, method, ip, user_agent, headers, status, latency, created_at)")
if err != nil {
logger.ErrorF(ctx, "[RiskControl] Prepare ClickHouse batch failed: %v", err)
batch = nil
return
items := make([]analytics.UserAccessLog, len(batch))
for i, item := range batch {
items[i] = *item
}
for _, item := range batch {
err = b.Append(
item.ID,
item.UserID,
item.Path,
item.Method,
item.IP,
item.UserAgent,
item.Headers,
item.Status,
item.Latency,
item.CreatedAt,
)
if err != nil {
logger.ErrorF(ctx, "[RiskControl] Append item to ClickHouse batch failed: %v", err)
}
}
if err := b.Send(); err != nil {
if err := analyticsrepo.BatchInsert(ctx, items); err != nil {
logger.ErrorF(ctx, "[RiskControl] Send ClickHouse batch failed: %v", err)
}
batch = nil
@@ -113,4 +89,4 @@ func startBatchWorker(ctx context.Context) {
flush()
}
}
}
}
+2 -1
View File
@@ -14,6 +14,7 @@ import (
"github.com/Rain-kl/Wavelet/internal/config"
"github.com/Rain-kl/Wavelet/internal/db/idgen"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/model/analytics"
"github.com/gin-gonic/gin"
)
@@ -68,7 +69,7 @@ func RiskControlMiddleware() gin.HandlerFunc {
status = maxHTTPStatus
}
logItem := &UserAccessLog{
logItem := &analytics.UserAccessLog{
ID: idgen.NextUint64ID(),
UserID: userObj.ID, // 直接从 Context 获取已登录用户ID,避免数据库查询
Path: c.Request.URL.Path,
@@ -13,6 +13,7 @@ import (
"github.com/Rain-kl/Wavelet/internal/apps/oauth"
"github.com/Rain-kl/Wavelet/internal/config"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/model/analytics"
"github.com/Rain-kl/Wavelet/internal/testhelper"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/assert"
@@ -40,7 +41,7 @@ func TestRiskControlMiddleware(t *testing.T) {
t.Run("ClickHouse enabled - Normal Authenticated Request", func(t *testing.T) {
config.Config.ClickHouse.Enabled = true
logChan = make(chan *UserAccessLog, defaultQueueSize)
logChan = make(chan *analytics.UserAccessLog, defaultQueueSize)
defer func() {
config.Config.ClickHouse.Enabled = false
logChan = nil
@@ -84,7 +85,7 @@ func TestRiskControlMiddleware(t *testing.T) {
t.Run("ClickHouse enabled - Unauthenticated Request", func(t *testing.T) {
config.Config.ClickHouse.Enabled = true
logChan = make(chan *UserAccessLog, defaultQueueSize)
logChan = make(chan *analytics.UserAccessLog, defaultQueueSize)
defer func() {
config.Config.ClickHouse.Enabled = false
logChan = nil
@@ -113,7 +114,7 @@ func TestRiskControlMiddleware(t *testing.T) {
t.Run("ClickHouse enabled - Buffer Full Rate Limiting", func(t *testing.T) {
config.Config.ClickHouse.Enabled = true
logChan = make(chan *UserAccessLog, 2) // small capacity for quick fill
logChan = make(chan *analytics.UserAccessLog, 2) // small capacity for quick fill
defer func() {
config.Config.ClickHouse.Enabled = false
logChan = nil
@@ -121,7 +122,7 @@ func TestRiskControlMiddleware(t *testing.T) {
// fill logChan up to cap to simulate buffer full
for len(logChan) < cap(logChan) {
logChan <- &UserAccessLog{}
logChan <- &analytics.UserAccessLog{}
}
r := testhelper.NewTestGinEngine(RiskControlMiddleware())
-22
View File
@@ -1,22 +0,0 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package risk_control
import (
"time"
)
// UserAccessLog 用户访问记录
type UserAccessLog struct {
ID uint64 `json:"id,string"`
UserID uint64 `json:"user_id,string"`
Path string `json:"path"`
Method string `json:"method"`
IP string `json:"ip"`
UserAgent string `json:"user_agent"`
Headers string `json:"headers"`
Status int32 `json:"status"`
Latency int64 `json:"latency"` // 耗时毫秒
CreatedAt time.Time `json:"created_at"`
}