mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-04 23:16:37 +08:00
feat(clickhouse): integrate goose migrations and analytics repository
Add a separate ClickHouse OLAP pipeline with goose/clickhouse DDL as the sole schema source, model/analytics for ORM mapping, and repository/analytics for reads (ChDB/GORM) and batch writes (ChConn). Refactor risk_control and admin/logs to use the repository layer instead of inline SQL. Remove the manual support-files DDL and document the workflow in database-migration skill.
This commit is contained in:
@@ -8,11 +8,12 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/config"
|
||||
"github.com/Rain-kl/Wavelet/internal/db"
|
||||
"github.com/Rain-kl/Wavelet/internal/model/analytics"
|
||||
analyticsrepo "github.com/Rain-kl/Wavelet/internal/repository/analytics"
|
||||
"github.com/Rain-kl/Wavelet/pkg/logger"
|
||||
)
|
||||
|
||||
var logChan chan *UserAccessLog
|
||||
var logChan chan *analytics.UserAccessLog
|
||||
|
||||
const (
|
||||
defaultQueueSize = 10000
|
||||
@@ -26,7 +27,7 @@ func InitLogWriter(ctx context.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
logChan = make(chan *UserAccessLog, defaultQueueSize)
|
||||
logChan = make(chan *analytics.UserAccessLog, defaultQueueSize)
|
||||
go startBatchWorker(context.WithoutCancel(ctx))
|
||||
}
|
||||
|
||||
@@ -40,7 +41,7 @@ func IsBufferFull() bool {
|
||||
}
|
||||
|
||||
// QueueAccessLog 异步非阻塞地将日志推入缓冲队列
|
||||
func QueueAccessLog(logItem *UserAccessLog) {
|
||||
func QueueAccessLog(logItem *analytics.UserAccessLog) {
|
||||
if !config.Config.ClickHouse.Enabled || logChan == nil {
|
||||
return
|
||||
}
|
||||
@@ -56,43 +57,18 @@ func startBatchWorker(ctx context.Context) {
|
||||
ticker := time.NewTicker(flushInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
var batch []*UserAccessLog
|
||||
var batch []*analytics.UserAccessLog
|
||||
|
||||
flush := func() {
|
||||
if len(batch) == 0 {
|
||||
return
|
||||
}
|
||||
if db.ChConn == nil {
|
||||
batch = nil
|
||||
return
|
||||
}
|
||||
|
||||
b, err := db.ChConn.PrepareBatch(ctx, "INSERT INTO w_user_access_logs (id, user_id, path, method, ip, user_agent, headers, status, latency, created_at)")
|
||||
if err != nil {
|
||||
logger.ErrorF(ctx, "[RiskControl] Prepare ClickHouse batch failed: %v", err)
|
||||
batch = nil
|
||||
return
|
||||
items := make([]analytics.UserAccessLog, len(batch))
|
||||
for i, item := range batch {
|
||||
items[i] = *item
|
||||
}
|
||||
|
||||
for _, item := range batch {
|
||||
err = b.Append(
|
||||
item.ID,
|
||||
item.UserID,
|
||||
item.Path,
|
||||
item.Method,
|
||||
item.IP,
|
||||
item.UserAgent,
|
||||
item.Headers,
|
||||
item.Status,
|
||||
item.Latency,
|
||||
item.CreatedAt,
|
||||
)
|
||||
if err != nil {
|
||||
logger.ErrorF(ctx, "[RiskControl] Append item to ClickHouse batch failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := b.Send(); err != nil {
|
||||
if err := analyticsrepo.BatchInsert(ctx, items); err != nil {
|
||||
logger.ErrorF(ctx, "[RiskControl] Send ClickHouse batch failed: %v", err)
|
||||
}
|
||||
batch = nil
|
||||
@@ -113,4 +89,4 @@ func startBatchWorker(ctx context.Context) {
|
||||
flush()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"github.com/Rain-kl/Wavelet/internal/config"
|
||||
"github.com/Rain-kl/Wavelet/internal/db/idgen"
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
"github.com/Rain-kl/Wavelet/internal/model/analytics"
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
@@ -68,7 +69,7 @@ func RiskControlMiddleware() gin.HandlerFunc {
|
||||
status = maxHTTPStatus
|
||||
}
|
||||
|
||||
logItem := &UserAccessLog{
|
||||
logItem := &analytics.UserAccessLog{
|
||||
ID: idgen.NextUint64ID(),
|
||||
UserID: userObj.ID, // 直接从 Context 获取已登录用户ID,避免数据库查询
|
||||
Path: c.Request.URL.Path,
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"github.com/Rain-kl/Wavelet/internal/apps/oauth"
|
||||
"github.com/Rain-kl/Wavelet/internal/config"
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
"github.com/Rain-kl/Wavelet/internal/model/analytics"
|
||||
"github.com/Rain-kl/Wavelet/internal/testhelper"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -40,7 +41,7 @@ func TestRiskControlMiddleware(t *testing.T) {
|
||||
|
||||
t.Run("ClickHouse enabled - Normal Authenticated Request", func(t *testing.T) {
|
||||
config.Config.ClickHouse.Enabled = true
|
||||
logChan = make(chan *UserAccessLog, defaultQueueSize)
|
||||
logChan = make(chan *analytics.UserAccessLog, defaultQueueSize)
|
||||
defer func() {
|
||||
config.Config.ClickHouse.Enabled = false
|
||||
logChan = nil
|
||||
@@ -84,7 +85,7 @@ func TestRiskControlMiddleware(t *testing.T) {
|
||||
|
||||
t.Run("ClickHouse enabled - Unauthenticated Request", func(t *testing.T) {
|
||||
config.Config.ClickHouse.Enabled = true
|
||||
logChan = make(chan *UserAccessLog, defaultQueueSize)
|
||||
logChan = make(chan *analytics.UserAccessLog, defaultQueueSize)
|
||||
defer func() {
|
||||
config.Config.ClickHouse.Enabled = false
|
||||
logChan = nil
|
||||
@@ -113,7 +114,7 @@ func TestRiskControlMiddleware(t *testing.T) {
|
||||
|
||||
t.Run("ClickHouse enabled - Buffer Full Rate Limiting", func(t *testing.T) {
|
||||
config.Config.ClickHouse.Enabled = true
|
||||
logChan = make(chan *UserAccessLog, 2) // small capacity for quick fill
|
||||
logChan = make(chan *analytics.UserAccessLog, 2) // small capacity for quick fill
|
||||
defer func() {
|
||||
config.Config.ClickHouse.Enabled = false
|
||||
logChan = nil
|
||||
@@ -121,7 +122,7 @@ func TestRiskControlMiddleware(t *testing.T) {
|
||||
|
||||
// fill logChan up to cap to simulate buffer full
|
||||
for len(logChan) < cap(logChan) {
|
||||
logChan <- &UserAccessLog{}
|
||||
logChan <- &analytics.UserAccessLog{}
|
||||
}
|
||||
|
||||
r := testhelper.NewTestGinEngine(RiskControlMiddleware())
|
||||
|
||||
@@ -1,22 +0,0 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package risk_control
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// UserAccessLog 用户访问记录
|
||||
type UserAccessLog struct {
|
||||
ID uint64 `json:"id,string"`
|
||||
UserID uint64 `json:"user_id,string"`
|
||||
Path string `json:"path"`
|
||||
Method string `json:"method"`
|
||||
IP string `json:"ip"`
|
||||
UserAgent string `json:"user_agent"`
|
||||
Headers string `json:"headers"`
|
||||
Status int32 `json:"status"`
|
||||
Latency int64 `json:"latency"` // 耗时毫秒
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
Reference in New Issue
Block a user