clickhouse 日志采集

internal/apps/admin/logs/routers.go

@@ -19,10 +19,18 @@ package logs
 
 import (
 	"encoding/json"
+	"fmt"
 	"net/http"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
 
 	"github.com/Rain-kl/Wavelet/internal/apps/admin"
+	"github.com/Rain-kl/Wavelet/internal/config"
+	"github.com/Rain-kl/Wavelet/internal/db"
 	"github.com/Rain-kl/Wavelet/internal/logger"
+	"github.com/Rain-kl/Wavelet/internal/model"
 	"github.com/Rain-kl/Wavelet/internal/util"
 	"github.com/gin-gonic/gin"
 )
@@ -131,3 +139,429 @@ func HandleLogWebSocket(c *gin.Context) {
 		}
 	}
 }
+
+// accessLogItem 访问日志单条数据
+type accessLogItem struct {
+	ID        uint64 `json:"id,string"`
+	UserID    uint64 `json:"user_id,string"`
+	Username  string `json:"username"`
+	Nickname  string `json:"nickname"`
+	Path      string `json:"path"`
+	Method    string `json:"method"`
+	IP        string `json:"ip"`
+	UserAgent string `json:"user_agent"`
+	Headers   string `json:"headers"`
+	Status    int32  `json:"status"`
+	Latency   int64  `json:"latency"`
+	CreatedAt string `json:"created_at"`
+}
+
+// accessLogsResponse 访问日志查询响应
+type accessLogsResponse struct {
+	Total uint64          `json:"total"`
+	List  []accessLogItem `json:"list"`
+}
+
+// GetAccessLogs 获取 ClickHouse 异步采集的访问日志
+// @Summary 获取用户访问日志
+// @Description 分页并按照用户、接口路径、时间范围等维度检索 ClickHouse 用户访问日志列表（需要管理员权限，ClickHouse 未启用时报错）
+// @Tags admin
+// @Produce json
+// @Security SessionCookie
+// @Param page query int false "页码" default(1)
+// @Param page_size query int false "每页条数" default(20)
+// @Param username query string false "用户名模糊搜索"
+// @Param path query string false "接口路径模糊搜索"
+// @Param start_time query string false "起始时间（RFC3339 或 YYYY-MM-DD HH:MM:SS）"
+// @Param end_time query string false "结束时间（RFC3339 或 YYYY-MM-DD HH:MM:SS）"
+// @Success 200 {object} util.ResponseAny{data=logs.accessLogsResponse} "访问日志列表"
+// @Failure 400 {object} util.ResponseAny "ClickHouse 未启用或参数错误"
+// @Failure 401 {object} util.ResponseAny "未登录"
+// @Failure 403 {object} util.ResponseAny "无管理员权限"
+// @Router /api/v1/admin/logs/access [get]
+func GetAccessLogs(c *gin.Context) {
+	// 1. 检查 ClickHouse 是否启用
+	if !config.Config.ClickHouse.Enabled || db.ChConn == nil {
+		c.JSON(http.StatusBadRequest, util.Err("ClickHouse 存储服务未启用，无法检索访问日志"))
+		return
+	}
+
+	// 2. 解析分页参数
+	page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
+	if page < 1 {
+		page = 1
+	}
+	pageSize, _ := strconv.Atoi(c.DefaultQuery("page_size", "20"))
+	if pageSize < 1 {
+		pageSize = 20
+	}
+	if pageSize > 100 {
+		pageSize = 100
+	}
+	offset := (page - 1) * pageSize
+
+	// 3. 按用户名过滤（预查 Postgres 映射 UserID）
+	var userIDs []uint64
+	username := c.Query("username")
+	if username != "" {
+		err := db.DB(c.Request.Context()).Model(&model.User{}).
+			Where("username LIKE ?", "%"+username+"%").
+			Pluck("id", &userIDs).Error
+		if err != nil {
+			c.JSON(http.StatusInternalServerError, util.Err("查询用户信息失败: "+err.Error()))
+			return
+		}
+		// 如果指定了用户名搜索，但在 Postgres 中没匹配到任何用户，则直接返回空结果
+		if len(userIDs) == 0 {
+			c.JSON(http.StatusOK, util.OK(accessLogsResponse{
+				Total: 0,
+				List:  []accessLogItem{},
+			}))
+			return
+		}
+	}
+
+	// 4. 构建 ClickHouse 条件查询子句与参数
+	var conditions []string
+	var args []interface{}
+
+	if len(userIDs) > 0 {
+		placeholders := make([]string, len(userIDs))
+		for i := range userIDs {
+			placeholders[i] = "?"
+			args = append(args, userIDs[i])
+		}
+		conditions = append(conditions, fmt.Sprintf("user_id IN (%s)", strings.Join(placeholders, ",")))
+	}
+
+	if path := c.Query("path"); path != "" {
+		conditions = append(conditions, "path LIKE ?")
+		args = append(args, "%"+path+"%")
+	}
+
+	if startTime := c.Query("start_time"); startTime != "" {
+		if t, err := time.Parse(time.RFC3339, startTime); err == nil {
+			conditions = append(conditions, "created_at >= ?")
+			args = append(args, t)
+		} else if t, err := time.Parse("2006-01-02 15:04:05", startTime); err == nil {
+			conditions = append(conditions, "created_at >= ?")
+			args = append(args, t)
+		}
+	}
+
+	if endTime := c.Query("end_time"); endTime != "" {
+		if t, err := time.Parse(time.RFC3339, endTime); err == nil {
+			conditions = append(conditions, "created_at <= ?")
+			args = append(args, t)
+		} else if t, err := time.Parse("2006-01-02 15:04:05", endTime); err == nil {
+			conditions = append(conditions, "created_at <= ?")
+			args = append(args, t)
+		}
+	}
+
+	whereClause := ""
+	if len(conditions) > 0 {
+		whereClause = "WHERE " + strings.Join(conditions, " AND ")
+	}
+
+	// 5. 查询日志总数
+	var total uint64
+	countQuery := fmt.Sprintf("SELECT count() FROM user_access_logs %s", whereClause)
+	err := db.ChConn.QueryRow(c.Request.Context(), countQuery, args...).Scan(&total)
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, util.Err("查询 ClickHouse 日志统计失败: "+err.Error()))
+		return
+	}
+
+	if total == 0 {
+		c.JSON(http.StatusOK, util.OK(accessLogsResponse{
+			Total: 0,
+			List:  []accessLogItem{},
+		}))
+		return
+	}
+
+	// 6. 分页查询明细数据
+	dataQuery := fmt.Sprintf(`
+		SELECT id, user_id, path, method, ip, user_agent, headers, status, latency, created_at
+		FROM user_access_logs
+		%s
+		ORDER BY created_at DESC, id DESC
+		LIMIT ? OFFSET ?
+	`, whereClause)
+
+	selectArgs := append(args, pageSize, offset)
+	rows, err := db.ChConn.Query(c.Request.Context(), dataQuery, selectArgs...)
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, util.Err("查询 ClickHouse 日志明细失败: "+err.Error()))
+		return
+	}
+	defer rows.Close()
+
+	var list []accessLogItem
+	var fetchUserIDs []uint64
+
+	for rows.Next() {
+		var item accessLogItem
+		var createdAt time.Time
+		err := rows.Scan(
+			&item.ID,
+			&item.UserID,
+			&item.Path,
+			&item.Method,
+			&item.IP,
+			&item.UserAgent,
+			&item.Headers,
+			&item.Status,
+			&item.Latency,
+			&createdAt,
+		)
+		if err != nil {
+			c.JSON(http.StatusInternalServerError, util.Err("读取 ClickHouse 结果失败: "+err.Error()))
+			return
+		}
+		item.CreatedAt = createdAt.Format(time.RFC3339)
+		list = append(list, item)
+		fetchUserIDs = append(fetchUserIDs, item.UserID)
+	}
+
+	// 7. 反查 Postgres 关联 Username 和 Nickname
+	userMap := make(map[uint64]struct {
+		Username string
+		Nickname string
+	})
+
+	if len(fetchUserIDs) > 0 {
+		var users []model.User
+		if err := db.DB(c.Request.Context()).Where("id IN ?", fetchUserIDs).Find(&users).Error; err == nil {
+			for _, u := range users {
+				userMap[u.ID] = struct {
+					Username string
+					Nickname string
+				}{
+					Username: u.Username,
+					Nickname: u.Nickname,
+				}
+			}
+		}
+	}
+
+	for i := range list {
+		if info, ok := userMap[list[i].UserID]; ok {
+			list[i].Username = info.Username
+			list[i].Nickname = info.Nickname
+		}
+	}
+
+	c.JSON(http.StatusOK, util.OK(accessLogsResponse{
+		Total: total,
+		List:  list,
+	}))
+}
+
+// trendItem 趋势图数据点
+type trendItem struct {
+	Date  string `json:"date"`
+	Count uint64 `json:"count"`
+}
+
+// browserItem 浏览器占比排行
+type browserItem struct {
+	Browser string `json:"browser"`
+	Count   uint64 `json:"count"`
+}
+
+// topUserItem 活跃用户数据
+type topUserItem struct {
+	UserID   uint64 `json:"user_id,string"`
+	Username string `json:"username"`
+	Nickname string `json:"nickname"`
+	Count    uint64 `json:"count"`
+}
+
+// logsAnalyticsResponse 访问日志数据分析结果
+type logsAnalyticsResponse struct {
+	Trend    []trendItem   `json:"trend"`
+	Browsers []browserItem `json:"browsers"`
+	TopUsers []topUserItem `json:"top_users"`
+}
+
+// GetLogsAnalytics 获取 ClickHouse 访问日志图表聚合指标
+// @Summary 获取访问日志分析数据
+// @Description 聚合统计最近 7 天的每日访问趋势、浏览器分布以及前 10 名最活跃用户排行（需要管理员权限，ClickHouse 未启用时报错）
+// @Tags admin
+// @Produce json
+// @Security SessionCookie
+// @Success 200 {object} util.ResponseAny{data=logs.logsAnalyticsResponse} "分析统计数据"
+// @Failure 400 {object} util.ResponseAny "ClickHouse 未启用"
+// @Failure 401 {object} util.ResponseAny "未登录"
+// @Failure 403 {object} util.ResponseAny "无管理员权限"
+// @Router /api/v1/admin/logs/analytics [get]
+func GetLogsAnalytics(c *gin.Context) {
+	// 1. 检查 ClickHouse 是否启用
+	if !config.Config.ClickHouse.Enabled || db.ChConn == nil {
+		c.JSON(http.StatusBadRequest, util.Err("ClickHouse 存储服务未启用，无法获取分析数据"))
+		return
+	}
+
+	// 7 天前 00:00:00
+	startTime := time.Now().AddDate(0, 0, -6).Truncate(24 * time.Hour)
+
+	// 2. 查询 7 天访问趋势
+	trendRows, err := db.ChConn.Query(c.Request.Context(), `
+		SELECT toDate(created_at) as date, count() as count
+		FROM user_access_logs
+		WHERE created_at >= ?
+		GROUP BY date
+		ORDER BY date ASC
+	`, startTime)
+
+	trendMap := make(map[string]uint64)
+	// 初始化最近 7 天的数据为 0，防止某天没有访问数据时导致日期断裂
+	for i := 0; i < 7; i++ {
+		dStr := time.Now().AddDate(0, 0, -i).Format("2006-01-02")
+		trendMap[dStr] = 0
+	}
+
+	if err == nil {
+		defer trendRows.Close()
+		for trendRows.Next() {
+			var dt time.Time
+			var cnt uint64
+			if errScan := trendRows.Scan(&dt, &cnt); errScan == nil {
+				dStr := dt.Format("2006-01-02")
+				trendMap[dStr] = cnt
+			}
+		}
+	}
+
+	var trendList []trendItem
+	for i := 6; i >= 0; i-- {
+		dStr := time.Now().AddDate(0, 0, -i).Format("2006-01-02")
+		trendList = append(trendList, trendItem{
+			Date:  dStr,
+			Count: trendMap[dStr],
+		})
+	}
+
+	// 3. 查询浏览器分布排行
+	uaRows, err := db.ChConn.Query(c.Request.Context(), `
+		SELECT user_agent, count() as count
+		FROM user_access_logs
+		WHERE created_at >= ?
+		GROUP BY user_agent
+	`, startTime)
+
+	browserCounts := make(map[string]uint64)
+	if err == nil {
+		defer uaRows.Close()
+		for uaRows.Next() {
+			var ua string
+			var cnt uint64
+			if errScan := uaRows.Scan(&ua, &cnt); errScan == nil {
+				browser := parseBrowserName(ua)
+				browserCounts[browser] += cnt
+			}
+		}
+	}
+
+	var browserList []browserItem
+	for b, cnt := range browserCounts {
+		browserList = append(browserList, browserItem{
+			Browser: b,
+			Count:   cnt,
+		})
+	}
+
+	// 排序：按访问次数降序
+	sort.Slice(browserList, func(i, j int) bool {
+		return browserList[i].Count > browserList[j].Count
+	})
+
+	// 4. 查询活跃用户 Top 10 (user_id > 0 代表已登录用户)
+	userRows, err := db.ChConn.Query(c.Request.Context(), `
+		SELECT user_id, count() as count
+		FROM user_access_logs
+		WHERE created_at >= ? AND user_id > 0
+		GROUP BY user_id
+		ORDER BY count DESC
+		LIMIT 10
+	`, startTime)
+
+	var topUsers []topUserItem
+	var userIDs []uint64
+	userCountMap := make(map[uint64]uint64)
+
+	if err == nil {
+		defer userRows.Close()
+		for userRows.Next() {
+			var uid uint64
+			var cnt uint64
+			if errScan := userRows.Scan(&uid, &cnt); errScan == nil {
+				userIDs = append(userIDs, uid)
+				userCountMap[uid] = cnt
+			}
+		}
+	}
+
+	// 反查 Postgres 补全活跃用户的用户名和昵称
+	userProfileMap := make(map[uint64]struct {
+		Username string
+		Nickname string
+	})
+
+	if len(userIDs) > 0 {
+		var users []model.User
+		if errProfile := db.DB(c.Request.Context()).Where("id IN ?", userIDs).Find(&users).Error; errProfile == nil {
+			for _, u := range users {
+				userProfileMap[u.ID] = struct {
+					Username string
+					Nickname string
+				}{
+					Username: u.Username,
+					Nickname: u.Nickname,
+				}
+			}
+		}
+	}
+
+	for _, uid := range userIDs {
+		profile := userProfileMap[uid]
+		topUsers = append(topUsers, topUserItem{
+			UserID:   uid,
+			Username: profile.Username,
+			Nickname: profile.Nickname,
+			Count:    userCountMap[uid],
+		})
+	}
+
+	c.JSON(http.StatusOK, util.OK(logsAnalyticsResponse{
+		Trend:    trendList,
+		Browsers: browserList,
+		TopUsers: topUsers,
+	}))
+}
+
+// parseBrowserName 简易的 User-Agent 浏览器类型识别
+func parseBrowserName(ua string) string {
+	uaLower := strings.ToLower(ua)
+	if strings.Contains(uaLower, "micromessenger") {
+		return "WeChat"
+	}
+	if strings.Contains(uaLower, "postman") {
+		return "Postman"
+	}
+	if strings.Contains(uaLower, "edg/") || strings.Contains(uaLower, "edge") {
+		return "Edge"
+	}
+	if strings.Contains(uaLower, "firefox") {
+		return "Firefox"
+	}
+	if strings.Contains(uaLower, "chrome") {
+		return "Chrome"
+	}
+	if strings.Contains(uaLower, "safari") {
+		return "Safari"
+	}
+	return "Other"
+}

internal/apps/risk_control/middleware.go

@@ -0,0 +1,91 @@
+/*
+Copyright 2026 Arctel.net
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package risk_control
+
+import (
+	"encoding/json"
+	"net/http"
+	"time"
+
+	"github.com/Rain-kl/Wavelet/internal/apps/oauth"
+	"github.com/Rain-kl/Wavelet/internal/config"
+	"github.com/Rain-kl/Wavelet/internal/db/idgen"
+	"github.com/Rain-kl/Wavelet/internal/model"
+	"github.com/Rain-kl/Wavelet/internal/util"
+	"github.com/gin-gonic/gin"
+)
+
+// RiskControlMiddleware 全局日志采集中间件
+func RiskControlMiddleware() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		// 如果未启用 ClickHouse，直接放行
+		if !config.Config.ClickHouse.Enabled {
+			c.Next()
+			return
+		}
+
+		// 1. 限流背压检测（检测本地缓冲队列是否已满）
+		if IsBufferFull() {
+			c.AbortWithStatusJSON(http.StatusTooManyRequests, util.Err("系统繁忙，请稍后再试"))
+			return
+		}
+
+		start := time.Now()
+
+		// 2. 执行后续请求（穿过业务处理和认证中间件）
+		c.Next()
+
+		// 3. 后置身份检查：仅记录通过认证的请求
+		userObj, exists := util.GetFromContext[*model.User](c, oauth.UserObjKey)
+		if !exists || userObj == nil {
+			return
+		}
+
+		// 4. 计算耗时并异步推送到缓冲队列
+		latency := time.Since(start).Milliseconds()
+
+		var headersStr string
+		if c.Request.Header != nil {
+			// 克隆 Header，避免污染原 HTTP 请求的 Header 对象
+			clonedHeaders := make(http.Header)
+			for k, v := range c.Request.Header {
+				clonedHeaders[k] = v
+			}
+			clonedHeaders.Del("Cookie")
+
+			if headersBytes, err := json.Marshal(clonedHeaders); err == nil {
+				headersStr = string(headersBytes)
+			}
+		}
+
+		logItem := &UserAccessLog{
+			ID:        idgen.NextUint64ID(),
+			UserID:    userObj.ID, // 直接从 Context 获取已登录用户ID，避免数据库查询
+			Path:      c.Request.URL.Path,
+			Method:    c.Request.Method,
+			IP:        c.ClientIP(),
+			UserAgent: c.Request.UserAgent(),
+			Headers:   headersStr,
+			Status:    int32(c.Writer.Status()),
+			Latency:   latency,
+			CreatedAt: time.Now(),
+		}
+
+		// 非阻塞地推入缓存队列
+		QueueAccessLog(logItem)
+	}
+}

internal/apps/risk_control/middleware_test.go

@@ -0,0 +1,159 @@
+/*
+Copyright 2026 Arctel.net
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package risk_control
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/Rain-kl/Wavelet/internal/apps/oauth"
+	"github.com/Rain-kl/Wavelet/internal/config"
+	"github.com/Rain-kl/Wavelet/internal/model"
+	"github.com/Rain-kl/Wavelet/internal/util"
+	"github.com/gin-gonic/gin"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRiskControlMiddleware(t *testing.T) {
+	gin.SetMode(gin.TestMode)
+
+	t.Run("ClickHouse disabled", func(t *testing.T) {
+		config.Config.ClickHouse.Enabled = false
+		defer func() { config.Config.ClickHouse.Enabled = false }()
+
+		r := gin.New()
+		r.Use(RiskControlMiddleware())
+		r.GET("/test", func(c *gin.Context) {
+			c.String(http.StatusOK, "ok")
+		})
+
+		w := httptest.NewRecorder()
+		req, _ := http.NewRequest(http.MethodGet, "/test", nil)
+		r.ServeHTTP(w, req)
+
+		assert.Equal(t, http.StatusOK, w.Code)
+		assert.Equal(t, "ok", w.Body.String())
+	})
+
+	t.Run("ClickHouse enabled - Normal Authenticated Request", func(t *testing.T) {
+		config.Config.ClickHouse.Enabled = true
+		logChan = make(chan *UserAccessLog, defaultQueueSize)
+		defer func() {
+			config.Config.ClickHouse.Enabled = false
+			logChan = nil
+		}()
+
+		r := gin.New()
+		r.Use(func(c *gin.Context) {
+			// Mock authentication middleware placing user in context
+			user := &model.User{ID: 12345}
+			util.SetToContext(c, oauth.UserObjKey, user)
+			c.Next()
+		})
+		r.Use(RiskControlMiddleware())
+		r.GET("/test", func(c *gin.Context) {
+			c.String(http.StatusOK, "ok")
+		})
+
+		w := httptest.NewRecorder()
+		req, _ := http.NewRequest(http.MethodGet, "/test", nil)
+		req.Header.Set("X-Test-Header", "hello")
+		req.Header.Set("Cookie", "session_id=abcdef123456")
+		r.ServeHTTP(w, req)
+
+		assert.Equal(t, http.StatusOK, w.Code)
+		assert.Equal(t, "ok", w.Body.String())
+
+		// Verify log is enqueued
+		select {
+		case logItem := <-logChan:
+			assert.Equal(t, uint64(12345), logItem.UserID)
+			assert.Equal(t, "/test", logItem.Path)
+			assert.Equal(t, http.MethodGet, logItem.Method)
+			assert.Equal(t, int32(http.StatusOK), logItem.Status)
+			assert.NotEmpty(t, logItem.Headers)
+			assert.Contains(t, logItem.Headers, "X-Test-Header")
+			assert.NotContains(t, logItem.Headers, "Cookie")
+		case <-time.After(100 * time.Millisecond):
+			t.Fatal("expected log item in logChan, but got none")
+		}
+	})
+
+	t.Run("ClickHouse enabled - Unauthenticated Request", func(t *testing.T) {
+		config.Config.ClickHouse.Enabled = true
+		logChan = make(chan *UserAccessLog, defaultQueueSize)
+		defer func() {
+			config.Config.ClickHouse.Enabled = false
+			logChan = nil
+		}()
+
+		r := gin.New()
+		r.Use(RiskControlMiddleware())
+		r.GET("/test", func(c *gin.Context) {
+			c.String(http.StatusOK, "ok")
+		})
+
+		w := httptest.NewRecorder()
+		req, _ := http.NewRequest(http.MethodGet, "/test", nil)
+		r.ServeHTTP(w, req)
+
+		assert.Equal(t, http.StatusOK, w.Code)
+		assert.Equal(t, "ok", w.Body.String())
+
+		// Verify no log is enqueued
+		select {
+		case <-logChan:
+			t.Fatal("expected no log item for unauthenticated request")
+		case <-time.After(50 * time.Millisecond):
+			// Success
+		}
+	})
+
+	t.Run("ClickHouse enabled - Buffer Full Rate Limiting", func(t *testing.T) {
+		config.Config.ClickHouse.Enabled = true
+		logChan = make(chan *UserAccessLog, 2) // small capacity for quick fill
+		defer func() {
+			config.Config.ClickHouse.Enabled = false
+			logChan = nil
+		}()
+
+		// fill logChan up to cap to simulate buffer full
+		for len(logChan) < cap(logChan) {
+			logChan <- &UserAccessLog{}
+		}
+
+		r := gin.New()
+		r.Use(RiskControlMiddleware())
+		r.GET("/test", func(c *gin.Context) {
+			c.String(http.StatusOK, "ok")
+		})
+
+		w := httptest.NewRecorder()
+		req, _ := http.NewRequest(http.MethodGet, "/test", nil)
+		r.ServeHTTP(w, req)
+
+		assert.Equal(t, http.StatusTooManyRequests, w.Code)
+
+		var resp map[string]interface{}
+		err := json.Unmarshal(w.Body.Bytes(), &resp)
+		assert.NoError(t, err)
+		assert.Contains(t, resp["error_msg"], "系统繁忙")
+	})
+}

internal/apps/risk_control/model.go

@@ -0,0 +1,148 @@
+/*
+Copyright 2026 Arctel.net
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package risk_control
+
+import (
+	"context"
+	"time"
+
+	"github.com/Rain-kl/Wavelet/internal/config"
+	"github.com/Rain-kl/Wavelet/internal/db"
+	"github.com/Rain-kl/Wavelet/internal/logger"
+)
+
+// UserAccessLog 用户访问记录
+type UserAccessLog struct {
+	ID        uint64    `json:"id,string"`
+	UserID    uint64    `json:"user_id,string"`
+	Path      string    `json:"path"`
+	Method    string    `json:"method"`
+	IP        string    `json:"ip"`
+	UserAgent string    `json:"user_agent"`
+	Headers   string    `json:"headers"`
+	Status    int32     `json:"status"`
+	Latency   int64     `json:"latency"` // 耗时毫秒
+	CreatedAt time.Time `json:"created_at"`
+}
+
+var (
+	logChan chan *UserAccessLog
+)
+
+const (
+	defaultQueueSize = 10000
+	maxBatchSize     = 1000
+	flushInterval    = 1 * time.Second
+)
+
+// InitLogWriter 初始化日志写入通道和后台写入协程
+func InitLogWriter() {
+	if !config.Config.ClickHouse.Enabled {
+		return
+	}
+
+	logChan = make(chan *UserAccessLog, defaultQueueSize)
+	go startBatchWorker()
+}
+
+// IsBufferFull 检查当前本地缓冲队列是否已满
+// 如果没有启用 ClickHouse，默认返回 false，不触发限流
+func IsBufferFull() bool {
+	if !config.Config.ClickHouse.Enabled || logChan == nil {
+		return false
+	}
+	return len(logChan) >= cap(logChan)
+}
+
+// QueueAccessLog 异步非阻塞地将日志推入缓冲队列
+func QueueAccessLog(logItem *UserAccessLog) {
+	if !config.Config.ClickHouse.Enabled || logChan == nil {
+		return
+	}
+
+	select {
+	case logChan <- logItem:
+	default:
+		// 如果在极端并发下仍然写满了，这里做非阻塞丢弃，防止卡死
+		logger.WarnF(context.Background(), "[RiskControl] Log queue full, dropping log item for path: %s", logItem.Path)
+	}
+}
+
+// startBatchWorker 后台批量写入 ClickHouse 的工作协程
+func startBatchWorker() {
+	ticker := time.NewTicker(flushInterval)
+	defer ticker.Stop()
+
+	var batch []*UserAccessLog
+
+	flush := func() {
+		if len(batch) == 0 {
+			return
+		}
+		if db.ChConn == nil {
+			batch = nil
+			return
+		}
+
+		ctx := context.Background()
+		b, err := db.ChConn.PrepareBatch(ctx, "INSERT INTO user_access_logs (id, user_id, path, method, ip, user_agent, headers, status, latency, created_at)")
+		if err != nil {
+			logger.ErrorF(ctx, "[RiskControl] Prepare ClickHouse batch failed: %v", err)
+			batch = nil
+			return
+		}
+
+		for _, item := range batch {
+			err = b.Append(
+				item.ID,
+				item.UserID,
+				item.Path,
+				item.Method,
+				item.IP,
+				item.UserAgent,
+				item.Headers,
+				item.Status,
+				item.Latency,
+				item.CreatedAt,
+			)
+			if err != nil {
+				logger.ErrorF(ctx, "[RiskControl] Append item to ClickHouse batch failed: %v", err)
+			}
+		}
+
+		if err := b.Send(); err != nil {
+			logger.ErrorF(ctx, "[RiskControl] Send ClickHouse batch failed: %v", err)
+		}
+		batch = nil
+	}
+
+	for {
+		select {
+		case item, ok := <-logChan:
+			if !ok {
+				flush()
+				return
+			}
+			batch = append(batch, item)
+			if len(batch) >= maxBatchSize {
+				flush()
+			}
+		case <-ticker.C:
+			flush()
+		}
+	}
+}