fix(log): address CodeRabbit review findings for log database decoupling

This commit is contained in:
ryan
2026-08-08 20:15:36 +08:00
parent 7d71f1e4e1
commit a6fc2b7737
23 changed files with 217 additions and 101 deletions
+1 -1
View File
@@ -1092,7 +1092,7 @@ const docTemplate = `{
} }
}, },
"400": { "400": {
"description": "ClickHouse 未启用或参数错误", "description": "日志存储未启用或参数错误",
"schema": { "schema": {
"$ref": "#/definitions/response.Any" "$ref": "#/definitions/response.Any"
} }
@@ -1532,30 +1532,37 @@ var allowedInfraPersistence = []string{
} }
func TestAppsMustNotImportLogBackendDirectly(t *testing.T) { func TestAppsMustNotImportLogBackendDirectly(t *testing.T) {
out, err := exec.Command("go", "list", "-deps", "./internal/apps/...").Output() t.Chdir("../../..")
out, err := exec.Command("go", "list", "-test", "-f", `{{.ImportPath}} {{join .Imports " "}}`, "./internal/apps/...").Output()
if err != nil { if err != nil {
t.Fatalf("go list: %v", err) t.Fatalf("go list: %v", err)
} }
for _, line := range strings.Split(string(out), "\n") { for _, line := range strings.Split(string(out), "\n") {
pkg := strings.TrimSpace(line) fields := strings.Fields(line)
if pkg == "" { if len(fields) == 0 {
continue continue
} }
for _, forbidden := range forbiddenImports { pkg := fields[0]
if pkg == forbidden { if !strings.HasPrefix(pkg, "github.com/Rain-kl/Wavelet/internal/apps") {
t.Errorf("internal/apps must not import %s", forbidden) continue
}
} }
if strings.HasPrefix(pkg, "github.com/Rain-kl/Wavelet/internal/infra/persistence/") { for _, imp := range fields[1:] {
allowed := false for _, forbidden := range forbiddenImports {
for _, a := range allowedInfraPersistence { if imp == forbidden && !allowedAnalyticsDelegation[pkg] {
if pkg == a || strings.HasPrefix(pkg, a+"/") { t.Errorf("%s must not import forbidden log backend %s", pkg, forbidden)
allowed = true
break
} }
} }
if !allowed { if strings.HasPrefix(imp, "github.com/Rain-kl/Wavelet/internal/infra/persistence/") {
t.Errorf("internal/apps must not import infra/persistence subpackage: %s", pkg) allowed := false
for _, a := range allowedInfraPersistence {
if imp == a || strings.HasPrefix(imp, a+"/") {
allowed = true
break
}
}
if !allowed {
t.Errorf("%s must not import infra/persistence subpackage directly: %s", pkg, imp)
}
} }
} }
} }
@@ -2261,6 +2268,9 @@ func (h *LogDBSwitchHandler) Execute(ctx context.Context, payload []byte) (*task
if err := copyAccessLogs(ctx, src, dst); err != nil { if err := copyAccessLogs(ctx, src, dst); err != nil {
return nil, err return nil, err
} }
if err := copyUserAccessLogs(ctx, src, dst); err != nil {
return nil, err
}
if err := copyObservability(ctx, src, dst); err != nil { if err := copyObservability(ctx, src, dst); err != nil {
return nil, err return nil, err
} }
@@ -2329,10 +2339,13 @@ func buildTargetStore(ctx context.Context, database string) (*logstore.Store, er
} }
func clearTargetLogTables(ctx context.Context, dst *logstore.Store, target string) error { func clearTargetLogTables(ctx context.Context, dst *logstore.Store, target string) error {
// 依次清空 6 张表:AccessLogs.DeleteAll、Observability.DeleteAll*(SQLite/PG 用 DeleteAll;CH 用 TRUNCATE 语义)。 // 依次清空 6 张表:AccessLogs.DeleteAll、UserAccessLogs.DeleteAll、Observability.DeleteAll*(SQLite/PG 用 DeleteAll;CH 用 TRUNCATE 语义)。
if _, err := dst.AccessLogs.DeleteAll(ctx); err != nil { if _, err := dst.AccessLogs.DeleteAll(ctx); err != nil {
return fmt.Errorf("清空目标访问日志失败: %w", err) return fmt.Errorf("清空目标访问日志失败: %w", err)
} }
if _, err := dst.UserAccessLogs.DeleteAll(ctx); err != nil {
return fmt.Errorf("清空目标用户访问日志失败: %w", err)
}
for _, fn := range []func(context.Context) (int64, error){ for _, fn := range []func(context.Context) (int64, error){
dst.Observability.DeleteAllMetricSnapshots, dst.Observability.DeleteAllMetricSnapshots,
dst.Observability.DeleteAllEdgeHealth, dst.Observability.DeleteAllEdgeHealth,
+1 -1
View File
@@ -1085,7 +1085,7 @@
} }
}, },
"400": { "400": {
"description": "ClickHouse 未启用或参数错误", "description": "日志存储未启用或参数错误",
"schema": { "schema": {
"$ref": "#/definitions/response.Any" "$ref": "#/definitions/response.Any"
} }
+1 -1
View File
@@ -4970,7 +4970,7 @@ paths:
$ref: '#/definitions/logs.accessLogsResponse' $ref: '#/definitions/logs.accessLogsResponse'
type: object type: object
"400": "400":
description: ClickHouse 未启用或参数错误 description: 日志存储未启用或参数错误
schema: schema:
$ref: '#/definitions/response.Any' $ref: '#/definitions/response.Any'
"401": "401":
@@ -588,7 +588,11 @@ export function TaskManager() {
} }
disabled={dispatching} disabled={dispatching}
> >
<SelectTrigger className='w-full text-xs' size='sm'> <SelectTrigger
id={`param-${param.name}`}
className='w-full text-xs'
size='sm'
>
<SelectValue placeholder='选择目标日志库...' /> <SelectValue placeholder='选择目标日志库...' />
</SelectTrigger> </SelectTrigger>
<SelectContent> <SelectContent>
+1 -2
View File
@@ -152,8 +152,7 @@ export default function ErrorPagesPage() {
<div className='space-y-1'> <div className='space-y-1'>
<Label className='text-sm font-medium'>仅针对 GET 请求</Label> <Label className='text-sm font-medium'>仅针对 GET 请求</Label>
<p className='text-sm text-muted-foreground'> <p className='text-sm text-muted-foreground'>
开启后仅对 GET 开启后仅对 GET 请求的匹配错误状态码返回自定义错误页;POST/PUT
请求的匹配错误状态码返回自定义错误页;POST/PUT
等其它方法直接透传源站响应。 等其它方法直接透传源站响应。
</p> </p>
</div> </div>
@@ -99,12 +99,17 @@ export function OperationTab({
const updateRetentionMutation = useMutation({ const updateRetentionMutation = useMutation({
mutationFn: async (values: Record<string, string>) => { mutationFn: async (values: Record<string, string>) => {
for (const field of LOG_RETENTION_FIELDS) { for (const field of LOG_RETENTION_FIELDS) {
const raw = (values[field.key] ?? '').trim();
const num = Number(raw);
if (!raw || !Number.isInteger(num) || num < 1) {
throw new Error(`${field.label}必须为大于等于 1 的整数`);
}
const config = businessConfigs[field.key]; const config = businessConfigs[field.key];
if (!config) { if (!config) {
throw new Error(`缺少配置项: ${field.key}`); throw new Error(`缺少配置项: ${field.key}`);
} }
await services.adminSystemConfig.updateSystemConfig(field.key, { await services.adminSystemConfig.updateSystemConfig(field.key, {
value: values[field.key] ?? '90', value: String(num),
description: config.description, description: config.description,
}); });
} }
+3 -1
View File
@@ -237,7 +237,7 @@ func enrichAccessLogsWithUsers(ctx context.Context, list []accessLogItem) {
// @Param start_time query string false "起始时间(RFC3339 或 YYYY-MM-DD HH:MM:SS)" // @Param start_time query string false "起始时间(RFC3339 或 YYYY-MM-DD HH:MM:SS)"
// @Param end_time query string false "结束时间(RFC3339 或 YYYY-MM-DD HH:MM:SS)" // @Param end_time query string false "结束时间(RFC3339 或 YYYY-MM-DD HH:MM:SS)"
// @Success 200 {object} response.Any{data=logs.accessLogsResponse} "访问日志列表" // @Success 200 {object} response.Any{data=logs.accessLogsResponse} "访问日志列表"
// @Failure 400 {object} response.Any "ClickHouse 未启用或参数错误" // @Failure 400 {object} response.Any "日志存储未启用或参数错误"
// @Failure 401 {object} response.Any "未登录" // @Failure 401 {object} response.Any "未登录"
// @Failure 403 {object} response.Any "无管理员权限" // @Failure 403 {object} response.Any "无管理员权限"
// @Router /api/v1/admin/logs/access [get] // @Router /api/v1/admin/logs/access [get]
@@ -245,6 +245,7 @@ func GetAccessLogs(c *gin.Context) {
ctx := c.Request.Context() ctx := c.Request.Context()
store, err := logstore.Active(ctx) store, err := logstore.Active(ctx)
if err != nil { if err != nil {
logger.ErrorF(ctx, "获取日志存储实例失败: %v", err)
response.AbortWithError(c, http.StatusBadRequest, "日志存储未启用,无法检索访问日志") response.AbortWithError(c, http.StatusBadRequest, "日志存储未启用,无法检索访问日志")
return return
} }
@@ -346,6 +347,7 @@ func GetLogsAnalytics(c *gin.Context) {
ctx := c.Request.Context() ctx := c.Request.Context()
store, err := logstore.Active(ctx) store, err := logstore.Active(ctx)
if err != nil { if err != nil {
logger.ErrorF(ctx, "获取日志存储实例失败: %v", err)
response.AbortWithError(c, http.StatusBadRequest, "日志存储未启用,无法获取分析数据") response.AbortWithError(c, http.StatusBadRequest, "日志存储未启用,无法获取分析数据")
return return
} }
+7
View File
@@ -5,6 +5,7 @@ package status
import ( import (
"context" "context"
"errors"
"net/http" "net/http"
"github.com/Rain-kl/Wavelet/internal/apps/openflare/chwriter" "github.com/Rain-kl/Wavelet/internal/apps/openflare/chwriter"
@@ -18,6 +19,7 @@ import (
"github.com/Rain-kl/Wavelet/internal/shared/response" "github.com/Rain-kl/Wavelet/internal/shared/response"
"github.com/Rain-kl/Wavelet/pkg/logger" "github.com/Rain-kl/Wavelet/pkg/logger"
"github.com/gin-gonic/gin" "github.com/gin-gonic/gin"
"gorm.io/gorm"
) )
// 日志库名取值(与 model 配置值、logstore provider 分支保持一致)。 // 日志库名取值(与 model 配置值、logstore provider 分支保持一致)。
@@ -54,6 +56,7 @@ func GetLogDatabaseStatus(c *gin.Context) {
ctx := c.Request.Context() ctx := c.Request.Context()
store, err := logstore.Active(ctx) store, err := logstore.Active(ctx)
if err != nil { if err != nil {
logger.ErrorF(ctx, "获取日志存储实例失败: %v", err)
response.AbortInternal(c, "日志存储初始化失败") response.AbortInternal(c, "日志存储初始化失败")
return return
} }
@@ -61,6 +64,7 @@ func GetLogDatabaseStatus(c *gin.Context) {
// 分支判定复用同一 store 实例的 ActiveDatabase,避免与 Active 解析之间出现 TOCTOU。 // 分支判定复用同一 store 实例的 ActiveDatabase,避免与 Active 解析之间出现 TOCTOU。
activeDB, err := store.Status.ActiveDatabase(ctx) activeDB, err := store.Status.ActiveDatabase(ctx)
if err != nil { if err != nil {
logger.ErrorF(ctx, "获取日志库状态失败: %v", err)
response.AbortInternal(c, "获取日志库状态失败") response.AbortInternal(c, "获取日志库状态失败")
return return
} }
@@ -97,6 +101,9 @@ func GetLogDatabaseStatus(c *gin.Context) {
func retentionOr(ctx context.Context, key string) int { func retentionOr(ctx context.Context, key string) int {
v, err := repository.GetIntByKey(ctx, key) v, err := repository.GetIntByKey(ctx, key)
if err != nil { if err != nil {
if !errors.Is(err, gorm.ErrRecordNotFound) {
logger.ErrorF(ctx, "读取日志保留天数配置失败 key=%s: %v", key, err)
}
return defaultLogRetentionDays return defaultLogRetentionDays
} }
return v return v
@@ -72,6 +72,8 @@ func TestAvailableTargets(t *testing.T) {
// TestGetLogDatabaseStatusSmoke 覆盖 handler 的 CH 激活分支(无需 DB/CH 连接)。 // TestGetLogDatabaseStatusSmoke 覆盖 handler 的 CH 激活分支(无需 DB/CH 连接)。
func TestGetLogDatabaseStatusSmoke(t *testing.T) { func TestGetLogDatabaseStatusSmoke(t *testing.T) {
restoreConfig(t) restoreConfig(t)
config.Config.Database.Enabled = false
config.Config.ClickHouse.Enabled = true
logstore.ResetForTest() logstore.ResetForTest()
t.Cleanup(logstore.ResetForTest) t.Cleanup(logstore.ResetForTest)
+1
View File
@@ -134,6 +134,7 @@ func (h *SystemCleanupHandler) Execute(ctx context.Context, _ []byte) (*task.Tas
summary, err := logstore.CleanupExpired(ctx) summary, err := logstore.CleanupExpired(ctx)
switch { switch {
case err != nil: case err != nil:
logger.ErrorF(ctx, "清理过期日志失败: %v", err)
task.AppendLog(ctx, "清理过期日志失败: %v", err) task.AppendLog(ctx, "清理过期日志失败: %v", err)
case summary.Deleted == 0: case summary.Deleted == 0:
task.AppendLog(ctx, "没有需要清理的过期日志 (保留 %d 天)", summary.RetentionDays) task.AppendLog(ctx, "没有需要清理的过期日志 (保留 %d 天)", summary.RetentionDays)
@@ -1,10 +1,12 @@
-- +goose Up -- +goose Up
-- 日志保留天数配置(business),替换旧的 database_auto_cleanup_* 键。 -- 日志保留天数配置(business),替换旧的 database_auto_cleanup_* 键。
INSERT INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at) INSERT INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
VALUES SELECT k, COALESCE((SELECT value FROM w_system_configs WHERE key = 'database_auto_cleanup_retention_days'), '90'), 'business', 0, descr, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP
('log_retention_days_postgres', '90', 'business', 0, 'PostgreSQL 日志保留天数(访问日志与可观测统一)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP), FROM (VALUES
('log_retention_days_sqlite', '90', 'business', 0, 'SQLite 日志保留天数', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP), ('log_retention_days_postgres', 'PostgreSQL 日志保留天数(访问日志与可观测统一)'),
('log_retention_days_clickhouse','90', 'business', 0, 'ClickHouse 日志保留天数', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) ('log_retention_days_sqlite', 'SQLite 日志保留天数'),
('log_retention_days_clickhouse', 'ClickHouse 日志保留天数')
) AS v(k, descr)
ON CONFLICT (key) DO NOTHING; ON CONFLICT (key) DO NOTHING;
DELETE FROM w_system_configs WHERE key IN ('database_auto_cleanup_enabled', 'database_auto_cleanup_retention_days'); DELETE FROM w_system_configs WHERE key IN ('database_auto_cleanup_enabled', 'database_auto_cleanup_retention_days');
@@ -13,7 +15,8 @@ DELETE FROM w_system_configs WHERE key IN ('database_auto_cleanup_enabled', 'dat
INSERT INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at) INSERT INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
VALUES VALUES
('database_auto_cleanup_enabled', 'true', 'business', 0, '数据库自动清理开关', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP), ('database_auto_cleanup_enabled', 'true', 'business', 0, '数据库自动清理开关', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('database_auto_cleanup_retention_days', '30', 'business', 0, '数据库保留天数', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) ('database_auto_cleanup_retention_days', COALESCE((SELECT value FROM w_system_configs WHERE key = 'log_retention_days_postgres'), '30'), 'business', 0, '数据库保留天数', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (key) DO NOTHING; ON CONFLICT (key) DO NOTHING;
DELETE FROM w_system_configs WHERE key IN ('log_retention_days_postgres', 'log_retention_days_sqlite', 'log_retention_days_clickhouse'); DELETE FROM w_system_configs WHERE key IN ('log_retention_days_postgres', 'log_retention_days_sqlite', 'log_retention_days_clickhouse');
@@ -1,7 +1,18 @@
-- +goose Up -- +goose Up
CREATE TABLE IF NOT EXISTS w_schedules_backup_of_database_auto_cleanup AS
SELECT * FROM w_schedules WHERE task_type = 'of_database_auto_cleanup';
DELETE FROM w_schedules WHERE task_type = 'of_database_auto_cleanup'; DELETE FROM w_schedules WHERE task_type = 'of_database_auto_cleanup';
-- +goose Down -- +goose Down
INSERT INTO w_schedules (id, name, task_type, cron, payload, is_active, created_at, updated_at)
SELECT id, name, task_type, cron, payload, is_active, created_at, updated_at
FROM w_schedules_backup_of_database_auto_cleanup
ON CONFLICT (id) DO NOTHING;
INSERT INTO w_schedules (id, name, task_type, cron, payload, is_active, created_at, updated_at) INSERT INTO w_schedules (id, name, task_type, cron, payload, is_active, created_at, updated_at)
VALUES (102, 'OpenFlare 可观测数据自动清理', 'of_database_auto_cleanup', '0 3 * * *', '{}', TRUE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) VALUES (102, 'OpenFlare 可观测数据自动清理', 'of_database_auto_cleanup', '0 3 * * *', '{}', TRUE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (id) DO NOTHING; ON CONFLICT (id) DO NOTHING;
DROP TABLE IF EXISTS w_schedules_backup_of_database_auto_cleanup;
@@ -1,17 +1,19 @@
-- +goose Up -- +goose Up
-- 日志保留天数配置(business),替换旧的 database_auto_cleanup_* 键。 -- 日志保留天数配置(business),替换旧的 database_auto_cleanup_* 键。
INSERT OR IGNORE INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at) INSERT OR IGNORE INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
VALUES SELECT 'log_retention_days_postgres', COALESCE((SELECT value FROM w_system_configs WHERE key = 'database_auto_cleanup_retention_days'), '90'), 'business', 0, 'PostgreSQL 日志保留天数(访问日志与可观测统一)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP
('log_retention_days_postgres', '90', 'business', 0, 'PostgreSQL 日志保留天数(访问日志与可观测统一)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP), UNION ALL
('log_retention_days_sqlite', '90', 'business', 0, 'SQLite 日志保留天数', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP), SELECT 'log_retention_days_sqlite', COALESCE((SELECT value FROM w_system_configs WHERE key = 'database_auto_cleanup_retention_days'), '90'), 'business', 0, 'SQLite 日志保留天数', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP
('log_retention_days_clickhouse','90', 'business', 0, 'ClickHouse 日志保留天数', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP); UNION ALL
SELECT 'log_retention_days_clickhouse', COALESCE((SELECT value FROM w_system_configs WHERE key = 'database_auto_cleanup_retention_days'), '90'), 'business', 0, 'ClickHouse 日志保留天数', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP;
DELETE FROM w_system_configs WHERE key IN ('database_auto_cleanup_enabled', 'database_auto_cleanup_retention_days'); DELETE FROM w_system_configs WHERE key IN ('database_auto_cleanup_enabled', 'database_auto_cleanup_retention_days');
-- +goose Down -- +goose Down
INSERT OR IGNORE INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at) INSERT OR IGNORE INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
VALUES SELECT 'database_auto_cleanup_enabled', 'true', 'business', 0, '数据库自动清理开关', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP
('database_auto_cleanup_enabled', 'true', 'business', 0, '数据库自动清理开关', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP), UNION ALL
('database_auto_cleanup_retention_days', '30', 'business', 0, '数据库保留天数', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP); SELECT 'database_auto_cleanup_retention_days', COALESCE((SELECT value FROM w_system_configs WHERE key = 'log_retention_days_sqlite'), '30'), 'business', 0, '数据库保留天数', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP;
DELETE FROM w_system_configs WHERE key IN ('log_retention_days_postgres', 'log_retention_days_sqlite', 'log_retention_days_clickhouse'); DELETE FROM w_system_configs WHERE key IN ('log_retention_days_postgres', 'log_retention_days_sqlite', 'log_retention_days_clickhouse');
@@ -1,7 +1,16 @@
-- +goose Up -- +goose Up
CREATE TABLE IF NOT EXISTS w_schedules_backup_of_database_auto_cleanup AS
SELECT * FROM w_schedules WHERE task_type = 'of_database_auto_cleanup';
DELETE FROM w_schedules WHERE task_type = 'of_database_auto_cleanup'; DELETE FROM w_schedules WHERE task_type = 'of_database_auto_cleanup';
-- +goose Down -- +goose Down
INSERT INTO w_schedules (id, name, task_type, cron, payload, is_active, created_at, updated_at) INSERT OR IGNORE INTO w_schedules (id, name, task_type, cron, payload, is_active, created_at, updated_at)
VALUES (102, 'OpenFlare 可观测数据自动清理', 'of_database_auto_cleanup', '0 3 * * *', '{}', 1, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) SELECT id, name, task_type, cron, payload, is_active, created_at, updated_at
ON CONFLICT (id) DO NOTHING; FROM w_schedules_backup_of_database_auto_cleanup;
INSERT OR IGNORE INTO w_schedules (id, name, task_type, cron, payload, is_active, created_at, updated_at)
VALUES (102, 'OpenFlare 可观测数据自动清理', 'of_database_auto_cleanup', '0 3 * * *', '{}', 1, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP);
DROP TABLE IF EXISTS w_schedules_backup_of_database_auto_cleanup;
+10 -7
View File
@@ -109,6 +109,15 @@ func TestValidateAndSeedLogDatabaseUpdatesEmptyMarker(t *testing.T) {
_, _, cleanup := testhelper.SetupTestEnvironment(t) _, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup() defer cleanup()
dbPrev := config.Config.Database.Enabled
chPrev := config.Config.ClickHouse.Enabled
config.Config.Database.Enabled = true
config.Config.ClickHouse.Enabled = false
t.Cleanup(func() {
config.Config.Database.Enabled = dbPrev
config.Config.ClickHouse.Enabled = chPrev
})
ctx := context.Background() ctx := context.Background()
// 标记行已存在但值为空,等同首次启动,应写入默认值(走更新路径)。 // 标记行已存在但值为空,等同首次启动,应写入默认值(走更新路径)。
if err := repository.CreateSystemConfig(ctx, &model.SystemConfig{Key: model.ConfigKeyLogDatabase, Value: "", Type: "system"}); err != nil { if err := repository.CreateSystemConfig(ctx, &model.SystemConfig{Key: model.ConfigKeyLogDatabase, Value: "", Type: "system"}); err != nil {
@@ -125,13 +134,7 @@ func TestValidateAndSeedLogDatabaseUpdatesEmptyMarker(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("GetSystemConfigByKey(%s) error = %v", model.ConfigKeyLogDatabase, err) t.Fatalf("GetSystemConfigByKey(%s) error = %v", model.ConfigKeyLogDatabase, err)
} }
want := "sqlite" want := "postgres"
if config.Config.Database.Enabled {
want = "postgres"
}
if config.Config.ClickHouse.Enabled {
want = "clickhouse"
}
if cfg.Value != want { if cfg.Value != want {
t.Fatalf("log_database = %q, want %q", cfg.Value, want) t.Fatalf("log_database = %q, want %q", cfg.Value, want)
} }
+11 -7
View File
@@ -28,12 +28,13 @@ const defaultLogRetentionDays = 90
// partitionLeadMonths 清理时确保「当前月 + 未来 2 个月」分区持续存在。 // partitionLeadMonths 清理时确保「当前月 + 未来 2 个月」分区持续存在。
const partitionLeadMonths = 2 const partitionLeadMonths = 2
// retentionDaysForActive 按当前激活库读取保留天数(默认 90)。 // retentionDaysForDatabase 按给定日志库读取保留天数(默认 90)。
func retentionDaysForActive(ctx context.Context) int { func retentionDaysForDatabase(ctx context.Context, dbName string) int {
key := model.ConfigKeyLogRetentionDaysPostgres key := model.ConfigKeyLogRetentionDaysPostgres
if dbName, _ := resolveDatabase(ctx); dbName == dbNameSQLite { switch dbName {
case dbNameSQLite:
key = model.ConfigKeyLogRetentionDaysSQLite key = model.ConfigKeyLogRetentionDaysSQLite
} else if dbName == dbNameClickHouse { case dbNameClickHouse:
key = model.ConfigKeyLogRetentionDaysClickHouse key = model.ConfigKeyLogRetentionDaysClickHouse
} }
v, err := getConfig(ctx, key) v, err := getConfig(ctx, key)
@@ -49,14 +50,17 @@ func retentionDaysForActive(ctx context.Context) int {
// CleanupExpired 按当前激活库保留天数清理过期日志(每日由 system_cleanup 调用)。 // CleanupExpired 按当前激活库保留天数清理过期日志(每日由 system_cleanup 调用)。
func CleanupExpired(ctx context.Context) (*CleanupSummary, error) { func CleanupExpired(ctx context.Context) (*CleanupSummary, error) {
dbName, err := resolveDatabase(ctx)
if err != nil {
return nil, fmt.Errorf("resolve active database: %w", err)
}
s, err := Active(ctx) s, err := Active(ctx)
if err != nil { if err != nil {
return nil, err return nil, err
} }
days := retentionDaysForActive(ctx) days := retentionDaysForDatabase(ctx, dbName)
cutoff := time.Now().AddDate(0, 0, -days) cutoff := time.Now().AddDate(0, 0, -days)
summary := &CleanupSummary{RetentionDays: days, Tables: []string{}} summary := &CleanupSummary{ActiveDatabase: dbName, RetentionDays: days, Tables: []string{}}
summary.ActiveDatabase, _ = resolveDatabase(ctx)
// PG 分区表仅在迁移时预建「当前+2 月」分区,此处确保分区持续存在, // PG 分区表仅在迁移时预建「当前+2 月」分区,此处确保分区持续存在,
// 否则跨月后新写入会报 "no partition of relation found"(SQLite/CH 为 no-op)。 // 否则跨月后新写入会报 "no partition of relation found"(SQLite/CH 为 no-op)。
+8 -8
View File
@@ -140,8 +140,8 @@ func TestCleanupExpiredSQLite(t *testing.T) {
} }
} }
// TestRetentionDaysForActive 覆盖保留天数读取:按激活库选 key、非法值回退默认 90。 // TestRetentionDaysForDatabase 覆盖保留天数读取:按激活库选 key、非法值回退默认 90。
func TestRetentionDaysForActive(t *testing.T) { func TestRetentionDaysForDatabase(t *testing.T) {
ResetForTest() ResetForTest()
SetConfigReader(func(_ context.Context, key string) (string, error) { SetConfigReader(func(_ context.Context, key string) (string, error) {
switch key { switch key {
@@ -152,8 +152,8 @@ func TestRetentionDaysForActive(t *testing.T) {
} }
return "", nil return "", nil
}) })
if got := retentionDaysForActive(context.Background()); got != 30 { if got := retentionDaysForDatabase(context.Background(), "sqlite"); got != 30 {
t.Fatalf("retentionDaysForActive = %d, want 30", got) t.Fatalf("retentionDaysForDatabase = %d, want 30", got)
} }
// 非法值(非数字/<=0)回退默认 90。 // 非法值(非数字/<=0)回退默认 90。
@@ -166,16 +166,16 @@ func TestRetentionDaysForActive(t *testing.T) {
} }
return "", nil return "", nil
}) })
if got := retentionDaysForActive(context.Background()); got != 90 { if got := retentionDaysForDatabase(context.Background(), "postgres"); got != 90 {
t.Fatalf("retentionDaysForActive invalid value = %d, want 90", got) t.Fatalf("retentionDaysForDatabase invalid value = %d, want 90", got)
} }
// reader 报错回退默认 90。 // reader 报错回退默认 90。
SetConfigReader(func(_ context.Context, _ string) (string, error) { SetConfigReader(func(_ context.Context, _ string) (string, error) {
return "", fmt.Errorf("boom") return "", fmt.Errorf("boom")
}) })
if got := retentionDaysForActive(context.Background()); got != 90 { if got := retentionDaysForDatabase(context.Background(), "postgres"); got != 90 {
t.Fatalf("retentionDaysForActive reader error = %d, want 90", got) t.Fatalf("retentionDaysForDatabase reader error = %d, want 90", got)
} }
} }
+18 -16
View File
@@ -33,8 +33,7 @@ var allowedInfraPersistence = []string{
func TestAppsMustNotImportLogBackendDirectly(t *testing.T) { func TestAppsMustNotImportLogBackendDirectly(t *testing.T) {
t.Chdir("../../..") // module root,保证 ./internal/apps/... 可解析 t.Chdir("../../..") // module root,保证 ./internal/apps/... 可解析
// -test 同时列出测试二进制依赖,覆盖仅测试文件引入的底层日志实现。 out, err := exec.Command("go", "list", "-test", "-f", `{{.ImportPath}} {{join .Imports " "}}`, "./internal/apps/...").Output()
out, err := exec.Command("go", "list", "-test", "-deps", "-f", `{{.ImportPath}} {{join .Imports " "}}`, "./internal/apps/...").Output()
if err != nil { if err != nil {
t.Fatalf("go list: %v", err) t.Fatalf("go list: %v", err)
} }
@@ -44,23 +43,26 @@ func TestAppsMustNotImportLogBackendDirectly(t *testing.T) {
continue continue
} }
pkg := fields[0] pkg := fields[0]
for _, forbidden := range forbiddenImports { if !strings.HasPrefix(pkg, "github.com/Rain-kl/Wavelet/internal/apps") {
for _, imp := range fields[1:] { continue
if imp == forbidden && !allowedAnalyticsDelegation[pkg] {
t.Errorf("internal/apps must not import %s (via %s)", forbidden, pkg)
}
}
} }
if strings.HasPrefix(pkg, "github.com/Rain-kl/Wavelet/internal/infra/persistence/") { for _, imp := range fields[1:] {
allowed := false for _, forbidden := range forbiddenImports {
for _, a := range allowedInfraPersistence { if imp == forbidden && !allowedAnalyticsDelegation[pkg] {
if pkg == a || strings.HasPrefix(pkg, a+"/") { t.Errorf("%s must not import forbidden log backend %s", pkg, forbidden)
allowed = true
break
} }
} }
if !allowed { if strings.HasPrefix(imp, "github.com/Rain-kl/Wavelet/internal/infra/persistence/") {
t.Errorf("internal/apps must not import infra/persistence subpackage: %s", pkg) allowed := false
for _, a := range allowedInfraPersistence {
if imp == a || strings.HasPrefix(imp, a+"/") {
allowed = true
break
}
}
if !allowed {
t.Errorf("%s must not import infra/persistence subpackage directly: %s", pkg, imp)
}
} }
} }
} }
@@ -1076,7 +1076,7 @@ FROM (
` + counterDelta("disk_write_bytes") + ` AS write_delta ` + counterDelta("disk_write_bytes") + ` AS write_delta
FROM of_node_metric_snapshots FROM of_node_metric_snapshots
WHERE ` + where + ` WHERE ` + where + `
) ) AS deltas
GROUP BY hour_epoch GROUP BY hour_epoch
ORDER BY hour_epoch ASC` ORDER BY hour_epoch ASC`
type row struct { type row struct {
+48 -14
View File
@@ -8,10 +8,12 @@ import (
"errors" "errors"
"fmt" "fmt"
"sync" "sync"
"time"
"github.com/Rain-kl/Wavelet/internal/infra/config" "github.com/Rain-kl/Wavelet/internal/infra/config"
db "github.com/Rain-kl/Wavelet/internal/infra/persistence" db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/model" "github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/pkg/logger"
) )
// logDatabaseKey / logMigrationKey 对应 model.ConfigKeyLogDatabase / ConfigKeyLogDBMigration。 // logDatabaseKey / logMigrationKey 对应 model.ConfigKeyLogDatabase / ConfigKeyLogDBMigration。
@@ -33,12 +35,16 @@ var errConfigReaderNotWired = errors.New("logstore: config reader not wired")
// ConfigReader 读取系统配置字符串值,由 bootstrap 注入(避免 logstore ↔ repository 循环依赖)。 // ConfigReader 读取系统配置字符串值,由 bootstrap 注入(避免 logstore ↔ repository 循环依赖)。
type ConfigReader func(ctx context.Context, key string) (string, error) type ConfigReader func(ctx context.Context, key string) (string, error)
const resolveCacheTTL = 1 * time.Second
var ( var (
configReader ConfigReader configReader ConfigReader
storeMu sync.RWMutex storeMu sync.RWMutex
active *Store active *Store
activeDB string activeDB string
lastResolveDB string
lastResolveTime time.Time
) )
// SetConfigReader 注入系统配置读取函数(bootstrap 调用,测试可注入内存实现)。 // SetConfigReader 注入系统配置读取函数(bootstrap 调用,测试可注入内存实现)。
@@ -124,6 +130,9 @@ func buildStore(ctx context.Context, database string, skipFreeze bool) (*Store,
func Migrating(ctx context.Context) bool { func Migrating(ctx context.Context) bool {
v, err := getConfig(ctx, logMigrationKey) v, err := getConfig(ctx, logMigrationKey)
if err != nil { if err != nil {
if !errors.Is(err, errConfigReaderNotWired) {
logger.ErrorF(ctx, "read log migration config failed: %v", err)
}
return false return false
} }
return v == "migrating" return v == "migrating"
@@ -134,11 +143,21 @@ func Init(ctx context.Context) {
_, _ = Active(ctx) _, _ = Active(ctx)
} }
// InvalidateCache 清空日志库解析缓存(在修改 log_database 配置后显式调用)。
func InvalidateCache() {
storeMu.Lock()
defer storeMu.Unlock()
lastResolveTime = time.Time{}
lastResolveDB = ""
}
// ResetForTest 清空缓存的激活 store 与 config reader,便于测试注入。 // ResetForTest 清空缓存的激活 store 与 config reader,便于测试注入。
func ResetForTest() { func ResetForTest() {
storeMu.Lock() storeMu.Lock()
active = nil active = nil
activeDB = "" activeDB = ""
lastResolveDB = ""
lastResolveTime = time.Time{}
storeMu.Unlock() storeMu.Unlock()
configReader = nil configReader = nil
} }
@@ -151,20 +170,35 @@ func ActiveDatabase(ctx context.Context) (string, error) {
// resolveDatabase 读取 log_database:值缺失或 reader 未装配(首启)时按启动规则 seed; // resolveDatabase 读取 log_database:值缺失或 reader 未装配(首启)时按启动规则 seed;
// 已装配 reader 的真实读取错误直接透出,避免把读失败当首次启动。 // 已装配 reader 的真实读取错误直接透出,避免把读失败当首次启动。
func resolveDatabase(ctx context.Context) (string, error) { func resolveDatabase(ctx context.Context) (string, error) {
storeMu.RLock()
if active != nil && time.Since(lastResolveTime) < resolveCacheTTL {
db := lastResolveDB
storeMu.RUnlock()
return db, nil
}
storeMu.RUnlock()
v, err := getConfig(ctx, logDatabaseKey) v, err := getConfig(ctx, logDatabaseKey)
if err != nil && !errors.Is(err, errConfigReaderNotWired) { if err != nil && !errors.Is(err, errConfigReaderNotWired) {
return "", err return "", err
} }
if v != "" {
return v, nil resolved := v
if resolved == "" {
// 首次启动 seed:CH 启用 → clickhouse;否则随主库。
resolved = dbNameSQLite
if config.Config.Database.Enabled {
resolved = dbNamePostgres
}
if config.Config.ClickHouse.Enabled {
resolved = dbNameClickHouse
}
} }
// 首次启动 seed:CH 启用 → clickhouse;否则随主库。
defaultDB := dbNameSQLite storeMu.Lock()
if config.Config.Database.Enabled { lastResolveDB = resolved
defaultDB = dbNamePostgres lastResolveTime = time.Now()
} storeMu.Unlock()
if config.Config.ClickHouse.Enabled {
defaultDB = dbNameClickHouse return resolved, nil
}
return defaultDB, nil
} }
@@ -18,6 +18,7 @@ import (
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics" analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
analyticsrepo "github.com/Rain-kl/Wavelet/internal/repository/analytics" analyticsrepo "github.com/Rain-kl/Wavelet/internal/repository/analytics"
"github.com/Rain-kl/Wavelet/internal/repository/logstore" "github.com/Rain-kl/Wavelet/internal/repository/logstore"
"github.com/Rain-kl/Wavelet/pkg/logger"
) )
const ( const (
@@ -105,16 +106,20 @@ func ListOpenFlareMetricSnapshotsSince(ctx context.Context, nodeID string, since
// avoiding stale reads of the previous CH store after a migration. // avoiding stale reads of the previous CH store after a migration.
func ListOpenFlareLatestMetricSnapshotsSince(ctx context.Context, nodeID string, since time.Time) ([]*model.OpenFlareMetricSnapshot, error) { func ListOpenFlareLatestMetricSnapshotsSince(ctx context.Context, nodeID string, since time.Time) ([]*model.OpenFlareMetricSnapshot, error) {
active, err := logstore.ActiveDatabase(ctx) active, err := logstore.ActiveDatabase(ctx)
if err == nil && active == logStoreNameClickHouse { if err != nil {
rows, err := analyticsrepo.ListLatestNodeMetricSnapshots(ctx, analyticsrepo.NodeObservabilityFilter{ logger.ErrorF(ctx, "failed to resolve active log database for latest metric snapshots: %v", err)
} else if active == logStoreNameClickHouse {
rows, chErr := analyticsrepo.ListLatestNodeMetricSnapshots(ctx, analyticsrepo.NodeObservabilityFilter{
NodeID: nodeID, NodeID: nodeID,
Since: since, Since: since,
}) })
if err == nil { if chErr == nil {
return fromAnalyticsNodeMetricSnapshots(rows), nil return fromAnalyticsNodeMetricSnapshots(rows), nil
} }
logger.ErrorF(ctx, "clickhouse fast-path ListLatestNodeMetricSnapshots failed: %v", chErr)
return nil, chErr
} }
// Routes through the active log store (CH unavailable or PG/SQLite active). // Routes through the active log store (PG/SQLite active).
all, listErr := ListOpenFlareMetricSnapshotsSince(ctx, nodeID, since, 0) all, listErr := ListOpenFlareMetricSnapshotsSince(ctx, nodeID, since, 0)
if listErr != nil { if listErr != nil {
return nil, listErr return nil, listErr
+15 -5
View File
@@ -401,21 +401,31 @@ func SetupLogStoresForTest(t *testing.T) {
logstore.SetAccessLogHooks(logstore.AccessLogHooks{ logstore.SetAccessLogHooks(logstore.AccessLogHooks{
QueueNodeAccessLogs: func(logs []analyticsmodel.NodeAccessLog) { QueueNodeAccessLogs: func(logs []analyticsmodel.NodeAccessLog) {
_ = store.AccessLogs.BatchInsertNodeAccessLogs(context.Background(), logs) if err := store.AccessLogs.BatchInsertNodeAccessLogs(context.Background(), logs); err != nil {
t.Errorf("batch insert node access logs failed in test hook: %v", err)
}
}, },
}) })
logstore.SetObservabilityHooks(logstore.ObservabilityHooks{ logstore.SetObservabilityHooks(logstore.ObservabilityHooks{
QueueMetricSnapshot: func(record analyticsmodel.NodeMetricSnapshot) { QueueMetricSnapshot: func(record analyticsmodel.NodeMetricSnapshot) {
_ = store.Observability.BatchInsertNodeMetricSnapshots(context.Background(), []analyticsmodel.NodeMetricSnapshot{record}) if err := store.Observability.BatchInsertNodeMetricSnapshots(context.Background(), []analyticsmodel.NodeMetricSnapshot{record}); err != nil {
t.Errorf("batch insert node metric snapshots failed in test hook: %v", err)
}
}, },
QueueEdgeHealth: func(record analyticsmodel.NodeEdgeHealth) { QueueEdgeHealth: func(record analyticsmodel.NodeEdgeHealth) {
_ = store.Observability.BatchInsertNodeEdgeHealth(context.Background(), []analyticsmodel.NodeEdgeHealth{record}) if err := store.Observability.BatchInsertNodeEdgeHealth(context.Background(), []analyticsmodel.NodeEdgeHealth{record}); err != nil {
t.Errorf("batch insert node edge health failed in test hook: %v", err)
}
}, },
QueueNodeObsFrps: func(record analyticsmodel.NodeObsFrps) { QueueNodeObsFrps: func(record analyticsmodel.NodeObsFrps) {
_ = store.Observability.BatchInsertNodeObsFrps(context.Background(), []analyticsmodel.NodeObsFrps{record}) if err := store.Observability.BatchInsertNodeObsFrps(context.Background(), []analyticsmodel.NodeObsFrps{record}); err != nil {
t.Errorf("batch insert node obs frps failed in test hook: %v", err)
}
}, },
QueueNodeObsFrpc: func(record analyticsmodel.NodeObsFrpc) { QueueNodeObsFrpc: func(record analyticsmodel.NodeObsFrpc) {
_ = store.Observability.BatchInsertNodeObsFrpc(context.Background(), []analyticsmodel.NodeObsFrpc{record}) if err := store.Observability.BatchInsertNodeObsFrpc(context.Background(), []analyticsmodel.NodeObsFrpc{record}); err != nil {
t.Errorf("batch insert node obs frpc failed in test hook: %v", err)
}
}, },
}) })