feat(openflare): migrate of_node_access_logs to ClickHouse

Move node access log storage from PostgreSQL/SQLite to ClickHouse
(database: openflare). System startup now requires a healthy ClickHouse
connection and auto-initializes the schema. Agent heartbeat writes access
logs via batch insert; goose migration drops the legacy relational table.

openflare-server/internal/apps/admin/logs/routers.go

@@ -16,7 +16,6 @@ import (
 	"time"
 
 	"github.com/Rain-kl/Wavelet/internal/apps/admin"
-	"github.com/Rain-kl/Wavelet/internal/config"
 	"github.com/Rain-kl/Wavelet/internal/db"
 	"github.com/Rain-kl/Wavelet/internal/model"
 	"github.com/Rain-kl/Wavelet/pkg/logger"
@@ -288,9 +287,8 @@ func fetchAccessLogDetails(ctx context.Context, whereClause string, args []inter
 // @Failure 403 {object} response.Any "无管理员权限"
 // @Router /api/v1/admin/logs/access [get]
 func GetAccessLogs(c *gin.Context) {
-	// 1. 检查 ClickHouse 是否启用
-	if !config.Config.ClickHouse.Enabled || db.ChConn == nil {
-		response.AbortWithError(c, http.StatusBadRequest, "ClickHouse 存储服务未启用，无法检索访问日志")
+	if db.ChConn == nil {
+		response.AbortWithError(c, http.StatusInternalServerError, "ClickHouse 未初始化，无法检索访问日志")
 		return
 	}
 
@@ -388,9 +386,8 @@ type logsAnalyticsResponse struct {
 // @Failure 403 {object} response.Any "无管理员权限"
 // @Router /api/v1/admin/logs/analytics [get]
 func GetLogsAnalytics(c *gin.Context) {
-	// 1. 检查 ClickHouse 是否启用
-	if !config.Config.ClickHouse.Enabled || db.ChConn == nil {
-		response.AbortWithError(c, http.StatusBadRequest, "ClickHouse 存储服务未启用，无法获取分析数据")
+	if db.ChConn == nil {
+		response.AbortWithError(c, http.StatusInternalServerError, "ClickHouse 未初始化，无法获取分析数据")
 		return
 	}
 

openflare-server/internal/apps/openflare/agent/observability.go

@@ -124,6 +124,12 @@ func PersistHeartbeatObservability(ctx context.Context, nodeID string, payload N
 		return
 	}
 
+	accessLogRecords, err := buildNodeAccessLogRecords(nodeID, payload.AccessLogs, payload.BufferedObservability, reportedAt)
+	if err != nil {
+		zap.L().Error("build heartbeat access logs failed", zap.String("node_id", nodeID), zap.Error(err))
+		return
+	}
+
 	if err := conn.Transaction(func(tx *gorm.DB) error {
 		if err := persistNodeSystemProfile(tx, nodeID, payload.Profile, reportedAt); err != nil {
 			return err
@@ -140,9 +146,6 @@ func PersistHeartbeatObservability(ctx context.Context, nodeID string, payload N
 		if err := persistNodeTrafficReport(tx, nodeID, payload.TrafficReport, reportedAt); err != nil {
 			return err
 		}
-		if err := persistNodeAccessLogs(tx, nodeID, payload.AccessLogs, reportedAt); err != nil {
-			return err
-		}
 		if payload.HealthEvents != nil {
 			if err := reconcileNodeHealthEvents(tx, nodeID, payload.HealthEvents, reportedAt); err != nil {
 				return err
@@ -151,6 +154,11 @@ func PersistHeartbeatObservability(ctx context.Context, nodeID string, payload N
 		return nil
 	}); err != nil {
 		zap.L().Error("persist heartbeat observability failed", zap.String("node_id", nodeID), zap.Error(err))
+		return
+	}
+
+	if err := persistNodeAccessLogs(ctx, nodeID, accessLogRecords, reportedAt); err != nil {
+		zap.L().Error("persist heartbeat access logs failed", zap.String("node_id", nodeID), zap.Error(err))
 	}
 }
 
@@ -165,9 +173,7 @@ func persistBufferedObservability(tx *gorm.DB, nodeID string, records []Buffered
 		if err := persistNodeTrafficReport(tx, nodeID, record.TrafficReport, reportedAt); err != nil {
 			return err
 		}
-		if err := persistNodeAccessLogs(tx, nodeID, record.AccessLogs, reportedAt); err != nil {
-			return err
-		}
+
 	}
 	return nil
 }
@@ -278,10 +284,15 @@ func persistNodeTrafficReport(tx *gorm.DB, nodeID string, report *NodeTrafficRep
 	return tx.Create(record).Error
 }
 
-func persistNodeAccessLogs(tx *gorm.DB, nodeID string, logs []NodeAccessLog, reportedAt time.Time) error {
-	if len(logs) == 0 {
-		return nil
+func buildNodeAccessLogRecords(nodeID string, direct []NodeAccessLog, buffered []BufferedObservabilityRecord, reportedAt time.Time) ([]*model.OpenFlareAccessLog, error) {
+	total := len(direct)
+	for _, record := range buffered {
+		total += len(record.AccessLogs)
 	}
+	if total == 0 {
+		return nil, nil
+	}
+
 	resolver, err := newAccessLogRegionResolver()
 	if err != nil {
 		slog.Warn("initialize access log geo resolver failed", "node_id", nodeID, "error", err)
@@ -289,31 +300,40 @@ func persistNodeAccessLogs(tx *gorm.DB, nodeID string, logs []NodeAccessLog, rep
 	if resolver != nil {
 		defer resolver.Close()
 	}
-	for _, item := range logs {
-		record := &model.OpenFlareAccessLog{
-			NodeID:     nodeID,
-			LoggedAt:   timeFromUnix(item.LoggedAtUnix, reportedAt),
-			RemoteAddr: strings.TrimSpace(item.RemoteAddr),
-			Region:     "",
-			Host:       strings.TrimSpace(item.Host),
-			Path:       truncateForDatabase(strings.TrimSpace(item.Path), accessLogPathMaxLength),
-			StatusCode: item.StatusCode,
-		}
-		if resolver != nil {
-			record.Region = resolver.Resolve(record.RemoteAddr)
-		}
-		exists, err := accessLogExists(tx, record)
-		if err != nil {
-			return err
-		}
-		if exists {
-			continue
-		}
-		if err := tx.Create(record).Error; err != nil {
-			return err
+
+	records := make([]*model.OpenFlareAccessLog, 0, total)
+	appendLogs := func(logs []NodeAccessLog) {
+		for _, item := range logs {
+			record := &model.OpenFlareAccessLog{
+				NodeID:     nodeID,
+				LoggedAt:   timeFromUnix(item.LoggedAtUnix, reportedAt),
+				RemoteAddr: strings.TrimSpace(item.RemoteAddr),
+				Region:     "",
+				Host:       strings.TrimSpace(item.Host),
+				Path:       truncateForDatabase(strings.TrimSpace(item.Path), accessLogPathMaxLength),
+				StatusCode: item.StatusCode,
+			}
+			if resolver != nil {
+				record.Region = resolver.Resolve(record.RemoteAddr)
+			}
+			records = append(records, record)
 		}
 	}
-	_, err = deleteAccessLogsByNodeBefore(tx, nodeID, reportedAt.Add(-nodeAccessLogRetentionWindow))
+	appendLogs(direct)
+	for _, record := range buffered {
+		appendLogs(record.AccessLogs)
+	}
+	return records, nil
+}
+
+func persistNodeAccessLogs(ctx context.Context, nodeID string, records []*model.OpenFlareAccessLog, reportedAt time.Time) error {
+	if len(records) == 0 {
+		return nil
+	}
+	if err := model.InsertOpenFlareAccessLogsBatch(ctx, records); err != nil {
+		return err
+	}
+	_, err := model.DeleteOpenFlareAccessLogsByNodeBefore(ctx, nodeID, reportedAt.Add(-nodeAccessLogRetentionWindow))
 	return err
 }
 
@@ -434,30 +454,6 @@ func requestReportExists(tx *gorm.DB, nodeID string, windowStartedAt, windowEnde
 	return count > 0, nil
 }
 
-func deleteAccessLogsByNodeBefore(tx *gorm.DB, nodeID string, before time.Time) (int64, error) {
-	result := tx.Where("node_id = ? AND logged_at < ?", nodeID, before).Delete(&model.OpenFlareAccessLog{})
-	return result.RowsAffected, result.Error
-}
-
-func accessLogExists(tx *gorm.DB, record *model.OpenFlareAccessLog) (bool, error) {
-	var count int64
-	if err := tx.Model(&model.OpenFlareAccessLog{}).
-		Where(
-			"node_id = ? AND logged_at = ? AND remote_addr = ? AND host = ? AND path = ? AND status_code = ?",
-			record.NodeID,
-			record.LoggedAt,
-			record.RemoteAddr,
-			record.Host,
-			record.Path,
-			record.StatusCode,
-		).
-		Limit(1).
-		Count(&count).Error; err != nil {
-		return false, err
-	}
-	return count > 0, nil
-}
-
 func normalizeHealthEventType(eventType string) string {
 	eventType = strings.TrimSpace(strings.ToLower(eventType))
 	eventType = strings.ReplaceAll(eventType, " ", "_")

openflare-server/internal/apps/openflare/async_tasks_test.go

@@ -34,21 +34,22 @@ func TestDatabaseAutoCleanupHandlerDeletesRowsWhenEnabled(t *testing.T) {
 		DisableForeignKeyConstraintWhenMigrating: true,
 	})
 	require.NoError(t, err)
-	require.NoError(t, sqliteDB.AutoMigrate(&model.OpenFlareAccessLog{}))
 	db.SetDB(sqliteDB)
+	resetAccessLogStore := model.SetAccessLogStoreForTest(model.NewMemoryAccessLogStore())
 	t.Cleanup(func() {
+		resetAccessLogStore()
 		db.SetDB(nil)
 	})
 
 	now := time.Now().UTC()
-	require.NoError(t, db.DB(context.Background()).Create(&model.OpenFlareAccessLog{
+	require.NoError(t, model.InsertOpenFlareAccessLogsBatch(context.Background(), []*model.OpenFlareAccessLog{{
 		NodeID:     "node-a",
 		LoggedAt:   now.Add(-48 * time.Hour),
 		RemoteAddr: "203.0.113.10",
 		Host:       "example.com",
 		Path:       "/access",
 		StatusCode: 200,
-	}).Error)
+	}}))
 
 	previousEnabled := model.DatabaseAutoCleanupEnabled
 	previousRetentionDays := model.DatabaseAutoCleanupRetentionDays

openflare-server/internal/apps/openflare/dashboard/logics_test.go

@@ -24,7 +24,9 @@ func setupDashboardTestDB(t *testing.T) func() {
 	require.NoError(t, sqliteDB.AutoMigrate(&model.OpenFlareNode{}))
 
 	db.SetDB(sqliteDB)
+	resetAccessLogStore := model.SetAccessLogStoreForTest(model.NewMemoryAccessLogStore())
 	return func() {
+		resetAccessLogStore()
 		db.SetDB(nil)
 	}
 }

openflare-server/internal/apps/openflare/node/logics_test.go

@@ -35,8 +35,10 @@ func setupNodeTestDB(t *testing.T) func() {
 
 	db.SetDB(sqliteDB)
 	option.ResetInitializationForTest()
+	resetAccessLogStore := model.SetAccessLogStoreForTest(model.NewMemoryAccessLogStore())
 
 	return func() {
+		resetAccessLogStore()
 		db.SetDB(nil)
 		option.ResetInitializationForTest()
 	}

openflare-server/internal/apps/openflare/option/logics_test.go

@@ -101,26 +101,28 @@ func TestCleanupDatabaseObservabilityDeletesRows(t *testing.T) {
 	defer cleanup()
 	ctx := context.Background()
 
-	sqliteDB := db.DB(ctx)
-	require.NoError(t, sqliteDB.AutoMigrate(&model.OpenFlareAccessLog{}))
+	resetAccessLogStore := model.SetAccessLogStoreForTest(model.NewMemoryAccessLogStore())
+	defer resetAccessLogStore()
 
 	now := time.Now().UTC()
-	require.NoError(t, sqliteDB.Create(&model.OpenFlareAccessLog{
-		NodeID:     "node-a",
-		LoggedAt:   now.Add(-10 * 24 * time.Hour),
-		RemoteAddr: "203.0.113.1",
-		Host:       "example.com",
-		Path:       "/old",
-		StatusCode: 200,
-	}).Error)
-	require.NoError(t, sqliteDB.Create(&model.OpenFlareAccessLog{
-		NodeID:     "node-a",
-		LoggedAt:   now.Add(-2 * time.Hour),
-		RemoteAddr: "203.0.113.2",
-		Host:       "example.com",
-		Path:       "/recent",
-		StatusCode: 200,
-	}).Error)
+	require.NoError(t, model.InsertOpenFlareAccessLogsBatch(ctx, []*model.OpenFlareAccessLog{
+		{
+			NodeID:     "node-a",
+			LoggedAt:   now.Add(-10 * 24 * time.Hour),
+			RemoteAddr: "203.0.113.1",
+			Host:       "example.com",
+			Path:       "/old",
+			StatusCode: 200,
+		},
+		{
+			NodeID:     "node-a",
+			LoggedAt:   now.Add(-2 * time.Hour),
+			RemoteAddr: "203.0.113.2",
+			Host:       "example.com",
+			Path:       "/recent",
+			StatusCode: 200,
+		},
+	}))
 
 	retention := 7
 	result, err := cleanupDatabaseObservability(ctx, databaseCleanupInput{

openflare-server/internal/apps/openflare/tasks/database_cleanup_test.go

@@ -24,12 +24,13 @@ func setupDatabaseCleanupTestDB(t *testing.T) context.Context {
 	})
 	require.NoError(t, err)
 	require.NoError(t, sqliteDB.AutoMigrate(
-		&model.OpenFlareAccessLog{},
 		&model.OpenFlareMetricSnapshot{},
 		&model.OpenFlareRequestReport{},
 	))
 	db.SetDB(sqliteDB)
+	resetAccessLogStore := model.SetAccessLogStoreForTest(model.NewMemoryAccessLogStore())
 	t.Cleanup(func() {
+		resetAccessLogStore()
 		db.SetDB(nil)
 	})
 	return context.Background()
@@ -69,22 +70,24 @@ func TestCleanupDatabaseObservabilityDeletesAllRowsWhenRetentionMissing(t *testi
 	ctx := setupDatabaseCleanupTestDB(t)
 	now := time.Now().UTC()
 
-	require.NoError(t, db.DB(ctx).Create(&model.OpenFlareAccessLog{
-		NodeID:     "node-a",
-		LoggedAt:   now.Add(-3 * time.Hour),
-		RemoteAddr: "203.0.113.1",
-		Host:       "example.com",
-		Path:       "/one",
-		StatusCode: 200,
-	}).Error)
-	require.NoError(t, db.DB(ctx).Create(&model.OpenFlareAccessLog{
-		NodeID:     "node-a",
-		LoggedAt:   now.Add(-2 * time.Hour),
-		RemoteAddr: "203.0.113.2",
-		Host:       "example.com",
-		Path:       "/two",
-		StatusCode: 502,
-	}).Error)
+	require.NoError(t, model.InsertOpenFlareAccessLogsBatch(ctx, []*model.OpenFlareAccessLog{
+		{
+			NodeID:     "node-a",
+			LoggedAt:   now.Add(-3 * time.Hour),
+			RemoteAddr: "203.0.113.1",
+			Host:       "example.com",
+			Path:       "/one",
+			StatusCode: 200,
+		},
+		{
+			NodeID:     "node-a",
+			LoggedAt:   now.Add(-2 * time.Hour),
+			RemoteAddr: "203.0.113.2",
+			Host:       "example.com",
+			Path:       "/two",
+			StatusCode: 502,
+		},
+	}))
 
 	result, err := CleanupDatabaseObservability(ctx, DatabaseCleanupInput{
 		Target: DatabaseCleanupTargetAccessLogs,
@@ -102,14 +105,14 @@ func TestRunDatabaseAutoCleanupOnceDeletesAllObservabilityTargets(t *testing.T)
 	ctx := setupDatabaseCleanupTestDB(t)
 	now := time.Now().UTC()
 
-	require.NoError(t, db.DB(ctx).Create(&model.OpenFlareAccessLog{
+	require.NoError(t, model.InsertOpenFlareAccessLogsBatch(ctx, []*model.OpenFlareAccessLog{{
 		NodeID:     "node-a",
 		LoggedAt:   now.Add(-48 * time.Hour),
 		RemoteAddr: "203.0.113.10",
 		Host:       "example.com",
 		Path:       "/access",
 		StatusCode: 200,
-	}).Error)
+	}}))
 	require.NoError(t, db.DB(ctx).Create(&model.OpenFlareMetricSnapshot{
 		NodeID:          "node-a",
 		CapturedAt:      now.Add(-48 * time.Hour),

openflare-server/internal/apps/openflare/waf/ip_group_sync_test.go

@@ -29,11 +29,12 @@ func setupIPGroupSyncTestDB(t *testing.T) func() {
 	require.NoError(t, sqliteDB.AutoMigrate(
 		&model.OpenFlareWAFRuleGroup{},
 		&model.OpenFlareWAFIPGroup{},
-		&model.OpenFlareAccessLog{},
 	))
 
 	db.SetDB(sqliteDB)
+	resetAccessLogStore := model.SetAccessLogStoreForTest(model.NewMemoryAccessLogStore())
 	return func() {
+		resetAccessLogStore()
 		db.SetDB(nil)
 	}
 }
@@ -191,18 +192,20 @@ func TestListDueOpenFlareWAFIPGroups(t *testing.T) {
 
 func seedWAFAccessLogs(t *testing.T, ctx context.Context, loggedAt time.Time, remoteAddr string, host string, total int, notFound int) {
 	t.Helper()
+	records := make([]*model.OpenFlareAccessLog, 0, total)
 	for i := 0; i < total; i++ {
 		statusCode := http.StatusOK
 		if i < notFound {
 			statusCode = http.StatusNotFound
 		}
-		require.NoError(t, db.DB(ctx).Create(&model.OpenFlareAccessLog{
+		records = append(records, &model.OpenFlareAccessLog{
 			NodeID:     "node-waf-auto",
 			LoggedAt:   loggedAt.Add(-time.Duration(i%30) * time.Second),
 			RemoteAddr: remoteAddr,
 			Host:       host,
 			Path:       "/probe",
 			StatusCode: statusCode,
-		}).Error)
+		})
 	}
+	require.NoError(t, model.InsertOpenFlareAccessLogsBatch(ctx, records))
 }