mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-06 23:56:37 +08:00
fix(log): address CodeRabbit review findings for log database decoupling
This commit is contained in:
+1
-1
@@ -1092,7 +1092,7 @@ const docTemplate = `{
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"400": {
|
"400": {
|
||||||
"description": "ClickHouse 未启用或参数错误",
|
"description": "日志存储未启用或参数错误",
|
||||||
"schema": {
|
"schema": {
|
||||||
"$ref": "#/definitions/response.Any"
|
"$ref": "#/definitions/response.Any"
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1532,30 +1532,37 @@ var allowedInfraPersistence = []string{
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestAppsMustNotImportLogBackendDirectly(t *testing.T) {
|
func TestAppsMustNotImportLogBackendDirectly(t *testing.T) {
|
||||||
out, err := exec.Command("go", "list", "-deps", "./internal/apps/...").Output()
|
t.Chdir("../../..")
|
||||||
|
out, err := exec.Command("go", "list", "-test", "-f", `{{.ImportPath}} {{join .Imports " "}}`, "./internal/apps/...").Output()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("go list: %v", err)
|
t.Fatalf("go list: %v", err)
|
||||||
}
|
}
|
||||||
for _, line := range strings.Split(string(out), "\n") {
|
for _, line := range strings.Split(string(out), "\n") {
|
||||||
pkg := strings.TrimSpace(line)
|
fields := strings.Fields(line)
|
||||||
if pkg == "" {
|
if len(fields) == 0 {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
for _, forbidden := range forbiddenImports {
|
pkg := fields[0]
|
||||||
if pkg == forbidden {
|
if !strings.HasPrefix(pkg, "github.com/Rain-kl/Wavelet/internal/apps") {
|
||||||
t.Errorf("internal/apps must not import %s", forbidden)
|
continue
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if strings.HasPrefix(pkg, "github.com/Rain-kl/Wavelet/internal/infra/persistence/") {
|
for _, imp := range fields[1:] {
|
||||||
allowed := false
|
for _, forbidden := range forbiddenImports {
|
||||||
for _, a := range allowedInfraPersistence {
|
if imp == forbidden && !allowedAnalyticsDelegation[pkg] {
|
||||||
if pkg == a || strings.HasPrefix(pkg, a+"/") {
|
t.Errorf("%s must not import forbidden log backend %s", pkg, forbidden)
|
||||||
allowed = true
|
|
||||||
break
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !allowed {
|
if strings.HasPrefix(imp, "github.com/Rain-kl/Wavelet/internal/infra/persistence/") {
|
||||||
t.Errorf("internal/apps must not import infra/persistence subpackage: %s", pkg)
|
allowed := false
|
||||||
|
for _, a := range allowedInfraPersistence {
|
||||||
|
if imp == a || strings.HasPrefix(imp, a+"/") {
|
||||||
|
allowed = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !allowed {
|
||||||
|
t.Errorf("%s must not import infra/persistence subpackage directly: %s", pkg, imp)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -2261,6 +2268,9 @@ func (h *LogDBSwitchHandler) Execute(ctx context.Context, payload []byte) (*task
|
|||||||
if err := copyAccessLogs(ctx, src, dst); err != nil {
|
if err := copyAccessLogs(ctx, src, dst); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
if err := copyUserAccessLogs(ctx, src, dst); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
if err := copyObservability(ctx, src, dst); err != nil {
|
if err := copyObservability(ctx, src, dst); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -2329,10 +2339,13 @@ func buildTargetStore(ctx context.Context, database string) (*logstore.Store, er
|
|||||||
}
|
}
|
||||||
|
|
||||||
func clearTargetLogTables(ctx context.Context, dst *logstore.Store, target string) error {
|
func clearTargetLogTables(ctx context.Context, dst *logstore.Store, target string) error {
|
||||||
// 依次清空 6 张表:AccessLogs.DeleteAll、Observability.DeleteAll*(SQLite/PG 用 DeleteAll;CH 用 TRUNCATE 语义)。
|
// 依次清空 6 张表:AccessLogs.DeleteAll、UserAccessLogs.DeleteAll、Observability.DeleteAll*(SQLite/PG 用 DeleteAll;CH 用 TRUNCATE 语义)。
|
||||||
if _, err := dst.AccessLogs.DeleteAll(ctx); err != nil {
|
if _, err := dst.AccessLogs.DeleteAll(ctx); err != nil {
|
||||||
return fmt.Errorf("清空目标访问日志失败: %w", err)
|
return fmt.Errorf("清空目标访问日志失败: %w", err)
|
||||||
}
|
}
|
||||||
|
if _, err := dst.UserAccessLogs.DeleteAll(ctx); err != nil {
|
||||||
|
return fmt.Errorf("清空目标用户访问日志失败: %w", err)
|
||||||
|
}
|
||||||
for _, fn := range []func(context.Context) (int64, error){
|
for _, fn := range []func(context.Context) (int64, error){
|
||||||
dst.Observability.DeleteAllMetricSnapshots,
|
dst.Observability.DeleteAllMetricSnapshots,
|
||||||
dst.Observability.DeleteAllEdgeHealth,
|
dst.Observability.DeleteAllEdgeHealth,
|
||||||
|
|||||||
+1
-1
@@ -1085,7 +1085,7 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"400": {
|
"400": {
|
||||||
"description": "ClickHouse 未启用或参数错误",
|
"description": "日志存储未启用或参数错误",
|
||||||
"schema": {
|
"schema": {
|
||||||
"$ref": "#/definitions/response.Any"
|
"$ref": "#/definitions/response.Any"
|
||||||
}
|
}
|
||||||
|
|||||||
+1
-1
@@ -4970,7 +4970,7 @@ paths:
|
|||||||
$ref: '#/definitions/logs.accessLogsResponse'
|
$ref: '#/definitions/logs.accessLogsResponse'
|
||||||
type: object
|
type: object
|
||||||
"400":
|
"400":
|
||||||
description: ClickHouse 未启用或参数错误
|
description: 日志存储未启用或参数错误
|
||||||
schema:
|
schema:
|
||||||
$ref: '#/definitions/response.Any'
|
$ref: '#/definitions/response.Any'
|
||||||
"401":
|
"401":
|
||||||
|
|||||||
@@ -588,7 +588,11 @@ export function TaskManager() {
|
|||||||
}
|
}
|
||||||
disabled={dispatching}
|
disabled={dispatching}
|
||||||
>
|
>
|
||||||
<SelectTrigger className='w-full text-xs' size='sm'>
|
<SelectTrigger
|
||||||
|
id={`param-${param.name}`}
|
||||||
|
className='w-full text-xs'
|
||||||
|
size='sm'
|
||||||
|
>
|
||||||
<SelectValue placeholder='选择目标日志库...' />
|
<SelectValue placeholder='选择目标日志库...' />
|
||||||
</SelectTrigger>
|
</SelectTrigger>
|
||||||
<SelectContent>
|
<SelectContent>
|
||||||
|
|||||||
@@ -152,8 +152,7 @@ export default function ErrorPagesPage() {
|
|||||||
<div className='space-y-1'>
|
<div className='space-y-1'>
|
||||||
<Label className='text-sm font-medium'>仅针对 GET 请求</Label>
|
<Label className='text-sm font-medium'>仅针对 GET 请求</Label>
|
||||||
<p className='text-sm text-muted-foreground'>
|
<p className='text-sm text-muted-foreground'>
|
||||||
开启后仅对 GET
|
开启后仅对 GET 请求的匹配错误状态码返回自定义错误页;POST/PUT
|
||||||
请求的匹配错误状态码返回自定义错误页;POST/PUT
|
|
||||||
等其它方法直接透传源站响应。
|
等其它方法直接透传源站响应。
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -99,12 +99,17 @@ export function OperationTab({
|
|||||||
const updateRetentionMutation = useMutation({
|
const updateRetentionMutation = useMutation({
|
||||||
mutationFn: async (values: Record<string, string>) => {
|
mutationFn: async (values: Record<string, string>) => {
|
||||||
for (const field of LOG_RETENTION_FIELDS) {
|
for (const field of LOG_RETENTION_FIELDS) {
|
||||||
|
const raw = (values[field.key] ?? '').trim();
|
||||||
|
const num = Number(raw);
|
||||||
|
if (!raw || !Number.isInteger(num) || num < 1) {
|
||||||
|
throw new Error(`${field.label}必须为大于等于 1 的整数`);
|
||||||
|
}
|
||||||
const config = businessConfigs[field.key];
|
const config = businessConfigs[field.key];
|
||||||
if (!config) {
|
if (!config) {
|
||||||
throw new Error(`缺少配置项: ${field.key}`);
|
throw new Error(`缺少配置项: ${field.key}`);
|
||||||
}
|
}
|
||||||
await services.adminSystemConfig.updateSystemConfig(field.key, {
|
await services.adminSystemConfig.updateSystemConfig(field.key, {
|
||||||
value: values[field.key] ?? '90',
|
value: String(num),
|
||||||
description: config.description,
|
description: config.description,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -237,7 +237,7 @@ func enrichAccessLogsWithUsers(ctx context.Context, list []accessLogItem) {
|
|||||||
// @Param start_time query string false "起始时间(RFC3339 或 YYYY-MM-DD HH:MM:SS)"
|
// @Param start_time query string false "起始时间(RFC3339 或 YYYY-MM-DD HH:MM:SS)"
|
||||||
// @Param end_time query string false "结束时间(RFC3339 或 YYYY-MM-DD HH:MM:SS)"
|
// @Param end_time query string false "结束时间(RFC3339 或 YYYY-MM-DD HH:MM:SS)"
|
||||||
// @Success 200 {object} response.Any{data=logs.accessLogsResponse} "访问日志列表"
|
// @Success 200 {object} response.Any{data=logs.accessLogsResponse} "访问日志列表"
|
||||||
// @Failure 400 {object} response.Any "ClickHouse 未启用或参数错误"
|
// @Failure 400 {object} response.Any "日志存储未启用或参数错误"
|
||||||
// @Failure 401 {object} response.Any "未登录"
|
// @Failure 401 {object} response.Any "未登录"
|
||||||
// @Failure 403 {object} response.Any "无管理员权限"
|
// @Failure 403 {object} response.Any "无管理员权限"
|
||||||
// @Router /api/v1/admin/logs/access [get]
|
// @Router /api/v1/admin/logs/access [get]
|
||||||
@@ -245,6 +245,7 @@ func GetAccessLogs(c *gin.Context) {
|
|||||||
ctx := c.Request.Context()
|
ctx := c.Request.Context()
|
||||||
store, err := logstore.Active(ctx)
|
store, err := logstore.Active(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
logger.ErrorF(ctx, "获取日志存储实例失败: %v", err)
|
||||||
response.AbortWithError(c, http.StatusBadRequest, "日志存储未启用,无法检索访问日志")
|
response.AbortWithError(c, http.StatusBadRequest, "日志存储未启用,无法检索访问日志")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -346,6 +347,7 @@ func GetLogsAnalytics(c *gin.Context) {
|
|||||||
ctx := c.Request.Context()
|
ctx := c.Request.Context()
|
||||||
store, err := logstore.Active(ctx)
|
store, err := logstore.Active(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
logger.ErrorF(ctx, "获取日志存储实例失败: %v", err)
|
||||||
response.AbortWithError(c, http.StatusBadRequest, "日志存储未启用,无法获取分析数据")
|
response.AbortWithError(c, http.StatusBadRequest, "日志存储未启用,无法获取分析数据")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ package status
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
|
||||||
"github.com/Rain-kl/Wavelet/internal/apps/openflare/chwriter"
|
"github.com/Rain-kl/Wavelet/internal/apps/openflare/chwriter"
|
||||||
@@ -18,6 +19,7 @@ import (
|
|||||||
"github.com/Rain-kl/Wavelet/internal/shared/response"
|
"github.com/Rain-kl/Wavelet/internal/shared/response"
|
||||||
"github.com/Rain-kl/Wavelet/pkg/logger"
|
"github.com/Rain-kl/Wavelet/pkg/logger"
|
||||||
"github.com/gin-gonic/gin"
|
"github.com/gin-gonic/gin"
|
||||||
|
"gorm.io/gorm"
|
||||||
)
|
)
|
||||||
|
|
||||||
// 日志库名取值(与 model 配置值、logstore provider 分支保持一致)。
|
// 日志库名取值(与 model 配置值、logstore provider 分支保持一致)。
|
||||||
@@ -54,6 +56,7 @@ func GetLogDatabaseStatus(c *gin.Context) {
|
|||||||
ctx := c.Request.Context()
|
ctx := c.Request.Context()
|
||||||
store, err := logstore.Active(ctx)
|
store, err := logstore.Active(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
logger.ErrorF(ctx, "获取日志存储实例失败: %v", err)
|
||||||
response.AbortInternal(c, "日志存储初始化失败")
|
response.AbortInternal(c, "日志存储初始化失败")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -61,6 +64,7 @@ func GetLogDatabaseStatus(c *gin.Context) {
|
|||||||
// 分支判定复用同一 store 实例的 ActiveDatabase,避免与 Active 解析之间出现 TOCTOU。
|
// 分支判定复用同一 store 实例的 ActiveDatabase,避免与 Active 解析之间出现 TOCTOU。
|
||||||
activeDB, err := store.Status.ActiveDatabase(ctx)
|
activeDB, err := store.Status.ActiveDatabase(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
logger.ErrorF(ctx, "获取日志库状态失败: %v", err)
|
||||||
response.AbortInternal(c, "获取日志库状态失败")
|
response.AbortInternal(c, "获取日志库状态失败")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -97,6 +101,9 @@ func GetLogDatabaseStatus(c *gin.Context) {
|
|||||||
func retentionOr(ctx context.Context, key string) int {
|
func retentionOr(ctx context.Context, key string) int {
|
||||||
v, err := repository.GetIntByKey(ctx, key)
|
v, err := repository.GetIntByKey(ctx, key)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
if !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||||
|
logger.ErrorF(ctx, "读取日志保留天数配置失败 key=%s: %v", key, err)
|
||||||
|
}
|
||||||
return defaultLogRetentionDays
|
return defaultLogRetentionDays
|
||||||
}
|
}
|
||||||
return v
|
return v
|
||||||
|
|||||||
@@ -72,6 +72,8 @@ func TestAvailableTargets(t *testing.T) {
|
|||||||
// TestGetLogDatabaseStatusSmoke 覆盖 handler 的 CH 激活分支(无需 DB/CH 连接)。
|
// TestGetLogDatabaseStatusSmoke 覆盖 handler 的 CH 激活分支(无需 DB/CH 连接)。
|
||||||
func TestGetLogDatabaseStatusSmoke(t *testing.T) {
|
func TestGetLogDatabaseStatusSmoke(t *testing.T) {
|
||||||
restoreConfig(t)
|
restoreConfig(t)
|
||||||
|
config.Config.Database.Enabled = false
|
||||||
|
config.Config.ClickHouse.Enabled = true
|
||||||
logstore.ResetForTest()
|
logstore.ResetForTest()
|
||||||
t.Cleanup(logstore.ResetForTest)
|
t.Cleanup(logstore.ResetForTest)
|
||||||
|
|
||||||
|
|||||||
@@ -134,6 +134,7 @@ func (h *SystemCleanupHandler) Execute(ctx context.Context, _ []byte) (*task.Tas
|
|||||||
summary, err := logstore.CleanupExpired(ctx)
|
summary, err := logstore.CleanupExpired(ctx)
|
||||||
switch {
|
switch {
|
||||||
case err != nil:
|
case err != nil:
|
||||||
|
logger.ErrorF(ctx, "清理过期日志失败: %v", err)
|
||||||
task.AppendLog(ctx, "清理过期日志失败: %v", err)
|
task.AppendLog(ctx, "清理过期日志失败: %v", err)
|
||||||
case summary.Deleted == 0:
|
case summary.Deleted == 0:
|
||||||
task.AppendLog(ctx, "没有需要清理的过期日志 (保留 %d 天)", summary.RetentionDays)
|
task.AppendLog(ctx, "没有需要清理的过期日志 (保留 %d 天)", summary.RetentionDays)
|
||||||
|
|||||||
+8
-5
@@ -1,10 +1,12 @@
|
|||||||
-- +goose Up
|
-- +goose Up
|
||||||
-- 日志保留天数配置(business),替换旧的 database_auto_cleanup_* 键。
|
-- 日志保留天数配置(business),替换旧的 database_auto_cleanup_* 键。
|
||||||
INSERT INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
|
INSERT INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
|
||||||
VALUES
|
SELECT k, COALESCE((SELECT value FROM w_system_configs WHERE key = 'database_auto_cleanup_retention_days'), '90'), 'business', 0, descr, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP
|
||||||
('log_retention_days_postgres', '90', 'business', 0, 'PostgreSQL 日志保留天数(访问日志与可观测统一)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
FROM (VALUES
|
||||||
('log_retention_days_sqlite', '90', 'business', 0, 'SQLite 日志保留天数', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
('log_retention_days_postgres', 'PostgreSQL 日志保留天数(访问日志与可观测统一)'),
|
||||||
('log_retention_days_clickhouse','90', 'business', 0, 'ClickHouse 日志保留天数', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
('log_retention_days_sqlite', 'SQLite 日志保留天数'),
|
||||||
|
('log_retention_days_clickhouse', 'ClickHouse 日志保留天数')
|
||||||
|
) AS v(k, descr)
|
||||||
ON CONFLICT (key) DO NOTHING;
|
ON CONFLICT (key) DO NOTHING;
|
||||||
|
|
||||||
DELETE FROM w_system_configs WHERE key IN ('database_auto_cleanup_enabled', 'database_auto_cleanup_retention_days');
|
DELETE FROM w_system_configs WHERE key IN ('database_auto_cleanup_enabled', 'database_auto_cleanup_retention_days');
|
||||||
@@ -13,7 +15,8 @@ DELETE FROM w_system_configs WHERE key IN ('database_auto_cleanup_enabled', 'dat
|
|||||||
INSERT INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
|
INSERT INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
|
||||||
VALUES
|
VALUES
|
||||||
('database_auto_cleanup_enabled', 'true', 'business', 0, '数据库自动清理开关', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
('database_auto_cleanup_enabled', 'true', 'business', 0, '数据库自动清理开关', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||||
('database_auto_cleanup_retention_days', '30', 'business', 0, '数据库保留天数', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
('database_auto_cleanup_retention_days', COALESCE((SELECT value FROM w_system_configs WHERE key = 'log_retention_days_postgres'), '30'), 'business', 0, '数据库保留天数', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||||
ON CONFLICT (key) DO NOTHING;
|
ON CONFLICT (key) DO NOTHING;
|
||||||
|
|
||||||
DELETE FROM w_system_configs WHERE key IN ('log_retention_days_postgres', 'log_retention_days_sqlite', 'log_retention_days_clickhouse');
|
DELETE FROM w_system_configs WHERE key IN ('log_retention_days_postgres', 'log_retention_days_sqlite', 'log_retention_days_clickhouse');
|
||||||
|
|
||||||
|
|||||||
+11
@@ -1,7 +1,18 @@
|
|||||||
-- +goose Up
|
-- +goose Up
|
||||||
|
CREATE TABLE IF NOT EXISTS w_schedules_backup_of_database_auto_cleanup AS
|
||||||
|
SELECT * FROM w_schedules WHERE task_type = 'of_database_auto_cleanup';
|
||||||
|
|
||||||
DELETE FROM w_schedules WHERE task_type = 'of_database_auto_cleanup';
|
DELETE FROM w_schedules WHERE task_type = 'of_database_auto_cleanup';
|
||||||
|
|
||||||
-- +goose Down
|
-- +goose Down
|
||||||
|
INSERT INTO w_schedules (id, name, task_type, cron, payload, is_active, created_at, updated_at)
|
||||||
|
SELECT id, name, task_type, cron, payload, is_active, created_at, updated_at
|
||||||
|
FROM w_schedules_backup_of_database_auto_cleanup
|
||||||
|
ON CONFLICT (id) DO NOTHING;
|
||||||
|
|
||||||
INSERT INTO w_schedules (id, name, task_type, cron, payload, is_active, created_at, updated_at)
|
INSERT INTO w_schedules (id, name, task_type, cron, payload, is_active, created_at, updated_at)
|
||||||
VALUES (102, 'OpenFlare 可观测数据自动清理', 'of_database_auto_cleanup', '0 3 * * *', '{}', TRUE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
VALUES (102, 'OpenFlare 可观测数据自动清理', 'of_database_auto_cleanup', '0 3 * * *', '{}', TRUE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||||
ON CONFLICT (id) DO NOTHING;
|
ON CONFLICT (id) DO NOTHING;
|
||||||
|
|
||||||
|
DROP TABLE IF EXISTS w_schedules_backup_of_database_auto_cleanup;
|
||||||
|
|
||||||
|
|||||||
+9
-7
@@ -1,17 +1,19 @@
|
|||||||
-- +goose Up
|
-- +goose Up
|
||||||
-- 日志保留天数配置(business),替换旧的 database_auto_cleanup_* 键。
|
-- 日志保留天数配置(business),替换旧的 database_auto_cleanup_* 键。
|
||||||
INSERT OR IGNORE INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
|
INSERT OR IGNORE INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
|
||||||
VALUES
|
SELECT 'log_retention_days_postgres', COALESCE((SELECT value FROM w_system_configs WHERE key = 'database_auto_cleanup_retention_days'), '90'), 'business', 0, 'PostgreSQL 日志保留天数(访问日志与可观测统一)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP
|
||||||
('log_retention_days_postgres', '90', 'business', 0, 'PostgreSQL 日志保留天数(访问日志与可观测统一)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
UNION ALL
|
||||||
('log_retention_days_sqlite', '90', 'business', 0, 'SQLite 日志保留天数', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
SELECT 'log_retention_days_sqlite', COALESCE((SELECT value FROM w_system_configs WHERE key = 'database_auto_cleanup_retention_days'), '90'), 'business', 0, 'SQLite 日志保留天数', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP
|
||||||
('log_retention_days_clickhouse','90', 'business', 0, 'ClickHouse 日志保留天数', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP);
|
UNION ALL
|
||||||
|
SELECT 'log_retention_days_clickhouse', COALESCE((SELECT value FROM w_system_configs WHERE key = 'database_auto_cleanup_retention_days'), '90'), 'business', 0, 'ClickHouse 日志保留天数', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP;
|
||||||
|
|
||||||
DELETE FROM w_system_configs WHERE key IN ('database_auto_cleanup_enabled', 'database_auto_cleanup_retention_days');
|
DELETE FROM w_system_configs WHERE key IN ('database_auto_cleanup_enabled', 'database_auto_cleanup_retention_days');
|
||||||
|
|
||||||
-- +goose Down
|
-- +goose Down
|
||||||
INSERT OR IGNORE INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
|
INSERT OR IGNORE INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
|
||||||
VALUES
|
SELECT 'database_auto_cleanup_enabled', 'true', 'business', 0, '数据库自动清理开关', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP
|
||||||
('database_auto_cleanup_enabled', 'true', 'business', 0, '数据库自动清理开关', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
UNION ALL
|
||||||
('database_auto_cleanup_retention_days', '30', 'business', 0, '数据库保留天数', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP);
|
SELECT 'database_auto_cleanup_retention_days', COALESCE((SELECT value FROM w_system_configs WHERE key = 'log_retention_days_sqlite'), '30'), 'business', 0, '数据库保留天数', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP;
|
||||||
|
|
||||||
DELETE FROM w_system_configs WHERE key IN ('log_retention_days_postgres', 'log_retention_days_sqlite', 'log_retention_days_clickhouse');
|
DELETE FROM w_system_configs WHERE key IN ('log_retention_days_postgres', 'log_retention_days_sqlite', 'log_retention_days_clickhouse');
|
||||||
|
|
||||||
|
|||||||
+12
-3
@@ -1,7 +1,16 @@
|
|||||||
-- +goose Up
|
-- +goose Up
|
||||||
|
CREATE TABLE IF NOT EXISTS w_schedules_backup_of_database_auto_cleanup AS
|
||||||
|
SELECT * FROM w_schedules WHERE task_type = 'of_database_auto_cleanup';
|
||||||
|
|
||||||
DELETE FROM w_schedules WHERE task_type = 'of_database_auto_cleanup';
|
DELETE FROM w_schedules WHERE task_type = 'of_database_auto_cleanup';
|
||||||
|
|
||||||
-- +goose Down
|
-- +goose Down
|
||||||
INSERT INTO w_schedules (id, name, task_type, cron, payload, is_active, created_at, updated_at)
|
INSERT OR IGNORE INTO w_schedules (id, name, task_type, cron, payload, is_active, created_at, updated_at)
|
||||||
VALUES (102, 'OpenFlare 可观测数据自动清理', 'of_database_auto_cleanup', '0 3 * * *', '{}', 1, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
SELECT id, name, task_type, cron, payload, is_active, created_at, updated_at
|
||||||
ON CONFLICT (id) DO NOTHING;
|
FROM w_schedules_backup_of_database_auto_cleanup;
|
||||||
|
|
||||||
|
INSERT OR IGNORE INTO w_schedules (id, name, task_type, cron, payload, is_active, created_at, updated_at)
|
||||||
|
VALUES (102, 'OpenFlare 可观测数据自动清理', 'of_database_auto_cleanup', '0 3 * * *', '{}', 1, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP);
|
||||||
|
|
||||||
|
DROP TABLE IF EXISTS w_schedules_backup_of_database_auto_cleanup;
|
||||||
|
|
||||||
|
|||||||
@@ -109,6 +109,15 @@ func TestValidateAndSeedLogDatabaseUpdatesEmptyMarker(t *testing.T) {
|
|||||||
_, _, cleanup := testhelper.SetupTestEnvironment(t)
|
_, _, cleanup := testhelper.SetupTestEnvironment(t)
|
||||||
defer cleanup()
|
defer cleanup()
|
||||||
|
|
||||||
|
dbPrev := config.Config.Database.Enabled
|
||||||
|
chPrev := config.Config.ClickHouse.Enabled
|
||||||
|
config.Config.Database.Enabled = true
|
||||||
|
config.Config.ClickHouse.Enabled = false
|
||||||
|
t.Cleanup(func() {
|
||||||
|
config.Config.Database.Enabled = dbPrev
|
||||||
|
config.Config.ClickHouse.Enabled = chPrev
|
||||||
|
})
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
// 标记行已存在但值为空,等同首次启动,应写入默认值(走更新路径)。
|
// 标记行已存在但值为空,等同首次启动,应写入默认值(走更新路径)。
|
||||||
if err := repository.CreateSystemConfig(ctx, &model.SystemConfig{Key: model.ConfigKeyLogDatabase, Value: "", Type: "system"}); err != nil {
|
if err := repository.CreateSystemConfig(ctx, &model.SystemConfig{Key: model.ConfigKeyLogDatabase, Value: "", Type: "system"}); err != nil {
|
||||||
@@ -125,13 +134,7 @@ func TestValidateAndSeedLogDatabaseUpdatesEmptyMarker(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("GetSystemConfigByKey(%s) error = %v", model.ConfigKeyLogDatabase, err)
|
t.Fatalf("GetSystemConfigByKey(%s) error = %v", model.ConfigKeyLogDatabase, err)
|
||||||
}
|
}
|
||||||
want := "sqlite"
|
want := "postgres"
|
||||||
if config.Config.Database.Enabled {
|
|
||||||
want = "postgres"
|
|
||||||
}
|
|
||||||
if config.Config.ClickHouse.Enabled {
|
|
||||||
want = "clickhouse"
|
|
||||||
}
|
|
||||||
if cfg.Value != want {
|
if cfg.Value != want {
|
||||||
t.Fatalf("log_database = %q, want %q", cfg.Value, want)
|
t.Fatalf("log_database = %q, want %q", cfg.Value, want)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -28,12 +28,13 @@ const defaultLogRetentionDays = 90
|
|||||||
// partitionLeadMonths 清理时确保「当前月 + 未来 2 个月」分区持续存在。
|
// partitionLeadMonths 清理时确保「当前月 + 未来 2 个月」分区持续存在。
|
||||||
const partitionLeadMonths = 2
|
const partitionLeadMonths = 2
|
||||||
|
|
||||||
// retentionDaysForActive 按当前激活库读取保留天数(默认 90)。
|
// retentionDaysForDatabase 按给定日志库读取保留天数(默认 90)。
|
||||||
func retentionDaysForActive(ctx context.Context) int {
|
func retentionDaysForDatabase(ctx context.Context, dbName string) int {
|
||||||
key := model.ConfigKeyLogRetentionDaysPostgres
|
key := model.ConfigKeyLogRetentionDaysPostgres
|
||||||
if dbName, _ := resolveDatabase(ctx); dbName == dbNameSQLite {
|
switch dbName {
|
||||||
|
case dbNameSQLite:
|
||||||
key = model.ConfigKeyLogRetentionDaysSQLite
|
key = model.ConfigKeyLogRetentionDaysSQLite
|
||||||
} else if dbName == dbNameClickHouse {
|
case dbNameClickHouse:
|
||||||
key = model.ConfigKeyLogRetentionDaysClickHouse
|
key = model.ConfigKeyLogRetentionDaysClickHouse
|
||||||
}
|
}
|
||||||
v, err := getConfig(ctx, key)
|
v, err := getConfig(ctx, key)
|
||||||
@@ -49,14 +50,17 @@ func retentionDaysForActive(ctx context.Context) int {
|
|||||||
|
|
||||||
// CleanupExpired 按当前激活库保留天数清理过期日志(每日由 system_cleanup 调用)。
|
// CleanupExpired 按当前激活库保留天数清理过期日志(每日由 system_cleanup 调用)。
|
||||||
func CleanupExpired(ctx context.Context) (*CleanupSummary, error) {
|
func CleanupExpired(ctx context.Context) (*CleanupSummary, error) {
|
||||||
|
dbName, err := resolveDatabase(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("resolve active database: %w", err)
|
||||||
|
}
|
||||||
s, err := Active(ctx)
|
s, err := Active(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
days := retentionDaysForActive(ctx)
|
days := retentionDaysForDatabase(ctx, dbName)
|
||||||
cutoff := time.Now().AddDate(0, 0, -days)
|
cutoff := time.Now().AddDate(0, 0, -days)
|
||||||
summary := &CleanupSummary{RetentionDays: days, Tables: []string{}}
|
summary := &CleanupSummary{ActiveDatabase: dbName, RetentionDays: days, Tables: []string{}}
|
||||||
summary.ActiveDatabase, _ = resolveDatabase(ctx)
|
|
||||||
|
|
||||||
// PG 分区表仅在迁移时预建「当前+2 月」分区,此处确保分区持续存在,
|
// PG 分区表仅在迁移时预建「当前+2 月」分区,此处确保分区持续存在,
|
||||||
// 否则跨月后新写入会报 "no partition of relation found"(SQLite/CH 为 no-op)。
|
// 否则跨月后新写入会报 "no partition of relation found"(SQLite/CH 为 no-op)。
|
||||||
|
|||||||
@@ -140,8 +140,8 @@ func TestCleanupExpiredSQLite(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestRetentionDaysForActive 覆盖保留天数读取:按激活库选 key、非法值回退默认 90。
|
// TestRetentionDaysForDatabase 覆盖保留天数读取:按激活库选 key、非法值回退默认 90。
|
||||||
func TestRetentionDaysForActive(t *testing.T) {
|
func TestRetentionDaysForDatabase(t *testing.T) {
|
||||||
ResetForTest()
|
ResetForTest()
|
||||||
SetConfigReader(func(_ context.Context, key string) (string, error) {
|
SetConfigReader(func(_ context.Context, key string) (string, error) {
|
||||||
switch key {
|
switch key {
|
||||||
@@ -152,8 +152,8 @@ func TestRetentionDaysForActive(t *testing.T) {
|
|||||||
}
|
}
|
||||||
return "", nil
|
return "", nil
|
||||||
})
|
})
|
||||||
if got := retentionDaysForActive(context.Background()); got != 30 {
|
if got := retentionDaysForDatabase(context.Background(), "sqlite"); got != 30 {
|
||||||
t.Fatalf("retentionDaysForActive = %d, want 30", got)
|
t.Fatalf("retentionDaysForDatabase = %d, want 30", got)
|
||||||
}
|
}
|
||||||
|
|
||||||
// 非法值(非数字/<=0)回退默认 90。
|
// 非法值(非数字/<=0)回退默认 90。
|
||||||
@@ -166,16 +166,16 @@ func TestRetentionDaysForActive(t *testing.T) {
|
|||||||
}
|
}
|
||||||
return "", nil
|
return "", nil
|
||||||
})
|
})
|
||||||
if got := retentionDaysForActive(context.Background()); got != 90 {
|
if got := retentionDaysForDatabase(context.Background(), "postgres"); got != 90 {
|
||||||
t.Fatalf("retentionDaysForActive invalid value = %d, want 90", got)
|
t.Fatalf("retentionDaysForDatabase invalid value = %d, want 90", got)
|
||||||
}
|
}
|
||||||
|
|
||||||
// reader 报错回退默认 90。
|
// reader 报错回退默认 90。
|
||||||
SetConfigReader(func(_ context.Context, _ string) (string, error) {
|
SetConfigReader(func(_ context.Context, _ string) (string, error) {
|
||||||
return "", fmt.Errorf("boom")
|
return "", fmt.Errorf("boom")
|
||||||
})
|
})
|
||||||
if got := retentionDaysForActive(context.Background()); got != 90 {
|
if got := retentionDaysForDatabase(context.Background(), "postgres"); got != 90 {
|
||||||
t.Fatalf("retentionDaysForActive reader error = %d, want 90", got)
|
t.Fatalf("retentionDaysForDatabase reader error = %d, want 90", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -33,8 +33,7 @@ var allowedInfraPersistence = []string{
|
|||||||
|
|
||||||
func TestAppsMustNotImportLogBackendDirectly(t *testing.T) {
|
func TestAppsMustNotImportLogBackendDirectly(t *testing.T) {
|
||||||
t.Chdir("../../..") // module root,保证 ./internal/apps/... 可解析
|
t.Chdir("../../..") // module root,保证 ./internal/apps/... 可解析
|
||||||
// -test 同时列出测试二进制依赖,覆盖仅测试文件引入的底层日志实现。
|
out, err := exec.Command("go", "list", "-test", "-f", `{{.ImportPath}} {{join .Imports " "}}`, "./internal/apps/...").Output()
|
||||||
out, err := exec.Command("go", "list", "-test", "-deps", "-f", `{{.ImportPath}} {{join .Imports " "}}`, "./internal/apps/...").Output()
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("go list: %v", err)
|
t.Fatalf("go list: %v", err)
|
||||||
}
|
}
|
||||||
@@ -44,23 +43,26 @@ func TestAppsMustNotImportLogBackendDirectly(t *testing.T) {
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
pkg := fields[0]
|
pkg := fields[0]
|
||||||
for _, forbidden := range forbiddenImports {
|
if !strings.HasPrefix(pkg, "github.com/Rain-kl/Wavelet/internal/apps") {
|
||||||
for _, imp := range fields[1:] {
|
continue
|
||||||
if imp == forbidden && !allowedAnalyticsDelegation[pkg] {
|
|
||||||
t.Errorf("internal/apps must not import %s (via %s)", forbidden, pkg)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if strings.HasPrefix(pkg, "github.com/Rain-kl/Wavelet/internal/infra/persistence/") {
|
for _, imp := range fields[1:] {
|
||||||
allowed := false
|
for _, forbidden := range forbiddenImports {
|
||||||
for _, a := range allowedInfraPersistence {
|
if imp == forbidden && !allowedAnalyticsDelegation[pkg] {
|
||||||
if pkg == a || strings.HasPrefix(pkg, a+"/") {
|
t.Errorf("%s must not import forbidden log backend %s", pkg, forbidden)
|
||||||
allowed = true
|
|
||||||
break
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !allowed {
|
if strings.HasPrefix(imp, "github.com/Rain-kl/Wavelet/internal/infra/persistence/") {
|
||||||
t.Errorf("internal/apps must not import infra/persistence subpackage: %s", pkg)
|
allowed := false
|
||||||
|
for _, a := range allowedInfraPersistence {
|
||||||
|
if imp == a || strings.HasPrefix(imp, a+"/") {
|
||||||
|
allowed = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !allowed {
|
||||||
|
t.Errorf("%s must not import infra/persistence subpackage directly: %s", pkg, imp)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1076,7 +1076,7 @@ FROM (
|
|||||||
` + counterDelta("disk_write_bytes") + ` AS write_delta
|
` + counterDelta("disk_write_bytes") + ` AS write_delta
|
||||||
FROM of_node_metric_snapshots
|
FROM of_node_metric_snapshots
|
||||||
WHERE ` + where + `
|
WHERE ` + where + `
|
||||||
)
|
) AS deltas
|
||||||
GROUP BY hour_epoch
|
GROUP BY hour_epoch
|
||||||
ORDER BY hour_epoch ASC`
|
ORDER BY hour_epoch ASC`
|
||||||
type row struct {
|
type row struct {
|
||||||
|
|||||||
@@ -8,10 +8,12 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"sync"
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/Rain-kl/Wavelet/internal/infra/config"
|
"github.com/Rain-kl/Wavelet/internal/infra/config"
|
||||||
db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
|
db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
|
||||||
"github.com/Rain-kl/Wavelet/internal/model"
|
"github.com/Rain-kl/Wavelet/internal/model"
|
||||||
|
"github.com/Rain-kl/Wavelet/pkg/logger"
|
||||||
)
|
)
|
||||||
|
|
||||||
// logDatabaseKey / logMigrationKey 对应 model.ConfigKeyLogDatabase / ConfigKeyLogDBMigration。
|
// logDatabaseKey / logMigrationKey 对应 model.ConfigKeyLogDatabase / ConfigKeyLogDBMigration。
|
||||||
@@ -33,12 +35,16 @@ var errConfigReaderNotWired = errors.New("logstore: config reader not wired")
|
|||||||
// ConfigReader 读取系统配置字符串值,由 bootstrap 注入(避免 logstore ↔ repository 循环依赖)。
|
// ConfigReader 读取系统配置字符串值,由 bootstrap 注入(避免 logstore ↔ repository 循环依赖)。
|
||||||
type ConfigReader func(ctx context.Context, key string) (string, error)
|
type ConfigReader func(ctx context.Context, key string) (string, error)
|
||||||
|
|
||||||
|
const resolveCacheTTL = 1 * time.Second
|
||||||
|
|
||||||
var (
|
var (
|
||||||
configReader ConfigReader
|
configReader ConfigReader
|
||||||
|
|
||||||
storeMu sync.RWMutex
|
storeMu sync.RWMutex
|
||||||
active *Store
|
active *Store
|
||||||
activeDB string
|
activeDB string
|
||||||
|
lastResolveDB string
|
||||||
|
lastResolveTime time.Time
|
||||||
)
|
)
|
||||||
|
|
||||||
// SetConfigReader 注入系统配置读取函数(bootstrap 调用,测试可注入内存实现)。
|
// SetConfigReader 注入系统配置读取函数(bootstrap 调用,测试可注入内存实现)。
|
||||||
@@ -124,6 +130,9 @@ func buildStore(ctx context.Context, database string, skipFreeze bool) (*Store,
|
|||||||
func Migrating(ctx context.Context) bool {
|
func Migrating(ctx context.Context) bool {
|
||||||
v, err := getConfig(ctx, logMigrationKey)
|
v, err := getConfig(ctx, logMigrationKey)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
if !errors.Is(err, errConfigReaderNotWired) {
|
||||||
|
logger.ErrorF(ctx, "read log migration config failed: %v", err)
|
||||||
|
}
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
return v == "migrating"
|
return v == "migrating"
|
||||||
@@ -134,11 +143,21 @@ func Init(ctx context.Context) {
|
|||||||
_, _ = Active(ctx)
|
_, _ = Active(ctx)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// InvalidateCache 清空日志库解析缓存(在修改 log_database 配置后显式调用)。
|
||||||
|
func InvalidateCache() {
|
||||||
|
storeMu.Lock()
|
||||||
|
defer storeMu.Unlock()
|
||||||
|
lastResolveTime = time.Time{}
|
||||||
|
lastResolveDB = ""
|
||||||
|
}
|
||||||
|
|
||||||
// ResetForTest 清空缓存的激活 store 与 config reader,便于测试注入。
|
// ResetForTest 清空缓存的激活 store 与 config reader,便于测试注入。
|
||||||
func ResetForTest() {
|
func ResetForTest() {
|
||||||
storeMu.Lock()
|
storeMu.Lock()
|
||||||
active = nil
|
active = nil
|
||||||
activeDB = ""
|
activeDB = ""
|
||||||
|
lastResolveDB = ""
|
||||||
|
lastResolveTime = time.Time{}
|
||||||
storeMu.Unlock()
|
storeMu.Unlock()
|
||||||
configReader = nil
|
configReader = nil
|
||||||
}
|
}
|
||||||
@@ -151,20 +170,35 @@ func ActiveDatabase(ctx context.Context) (string, error) {
|
|||||||
// resolveDatabase 读取 log_database:值缺失或 reader 未装配(首启)时按启动规则 seed;
|
// resolveDatabase 读取 log_database:值缺失或 reader 未装配(首启)时按启动规则 seed;
|
||||||
// 已装配 reader 的真实读取错误直接透出,避免把读失败当首次启动。
|
// 已装配 reader 的真实读取错误直接透出,避免把读失败当首次启动。
|
||||||
func resolveDatabase(ctx context.Context) (string, error) {
|
func resolveDatabase(ctx context.Context) (string, error) {
|
||||||
|
storeMu.RLock()
|
||||||
|
if active != nil && time.Since(lastResolveTime) < resolveCacheTTL {
|
||||||
|
db := lastResolveDB
|
||||||
|
storeMu.RUnlock()
|
||||||
|
return db, nil
|
||||||
|
}
|
||||||
|
storeMu.RUnlock()
|
||||||
|
|
||||||
v, err := getConfig(ctx, logDatabaseKey)
|
v, err := getConfig(ctx, logDatabaseKey)
|
||||||
if err != nil && !errors.Is(err, errConfigReaderNotWired) {
|
if err != nil && !errors.Is(err, errConfigReaderNotWired) {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
if v != "" {
|
|
||||||
return v, nil
|
resolved := v
|
||||||
|
if resolved == "" {
|
||||||
|
// 首次启动 seed:CH 启用 → clickhouse;否则随主库。
|
||||||
|
resolved = dbNameSQLite
|
||||||
|
if config.Config.Database.Enabled {
|
||||||
|
resolved = dbNamePostgres
|
||||||
|
}
|
||||||
|
if config.Config.ClickHouse.Enabled {
|
||||||
|
resolved = dbNameClickHouse
|
||||||
|
}
|
||||||
}
|
}
|
||||||
// 首次启动 seed:CH 启用 → clickhouse;否则随主库。
|
|
||||||
defaultDB := dbNameSQLite
|
storeMu.Lock()
|
||||||
if config.Config.Database.Enabled {
|
lastResolveDB = resolved
|
||||||
defaultDB = dbNamePostgres
|
lastResolveTime = time.Now()
|
||||||
}
|
storeMu.Unlock()
|
||||||
if config.Config.ClickHouse.Enabled {
|
|
||||||
defaultDB = dbNameClickHouse
|
return resolved, nil
|
||||||
}
|
|
||||||
return defaultDB, nil
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ import (
|
|||||||
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
|
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
|
||||||
analyticsrepo "github.com/Rain-kl/Wavelet/internal/repository/analytics"
|
analyticsrepo "github.com/Rain-kl/Wavelet/internal/repository/analytics"
|
||||||
"github.com/Rain-kl/Wavelet/internal/repository/logstore"
|
"github.com/Rain-kl/Wavelet/internal/repository/logstore"
|
||||||
|
"github.com/Rain-kl/Wavelet/pkg/logger"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -105,16 +106,20 @@ func ListOpenFlareMetricSnapshotsSince(ctx context.Context, nodeID string, since
|
|||||||
// avoiding stale reads of the previous CH store after a migration.
|
// avoiding stale reads of the previous CH store after a migration.
|
||||||
func ListOpenFlareLatestMetricSnapshotsSince(ctx context.Context, nodeID string, since time.Time) ([]*model.OpenFlareMetricSnapshot, error) {
|
func ListOpenFlareLatestMetricSnapshotsSince(ctx context.Context, nodeID string, since time.Time) ([]*model.OpenFlareMetricSnapshot, error) {
|
||||||
active, err := logstore.ActiveDatabase(ctx)
|
active, err := logstore.ActiveDatabase(ctx)
|
||||||
if err == nil && active == logStoreNameClickHouse {
|
if err != nil {
|
||||||
rows, err := analyticsrepo.ListLatestNodeMetricSnapshots(ctx, analyticsrepo.NodeObservabilityFilter{
|
logger.ErrorF(ctx, "failed to resolve active log database for latest metric snapshots: %v", err)
|
||||||
|
} else if active == logStoreNameClickHouse {
|
||||||
|
rows, chErr := analyticsrepo.ListLatestNodeMetricSnapshots(ctx, analyticsrepo.NodeObservabilityFilter{
|
||||||
NodeID: nodeID,
|
NodeID: nodeID,
|
||||||
Since: since,
|
Since: since,
|
||||||
})
|
})
|
||||||
if err == nil {
|
if chErr == nil {
|
||||||
return fromAnalyticsNodeMetricSnapshots(rows), nil
|
return fromAnalyticsNodeMetricSnapshots(rows), nil
|
||||||
}
|
}
|
||||||
|
logger.ErrorF(ctx, "clickhouse fast-path ListLatestNodeMetricSnapshots failed: %v", chErr)
|
||||||
|
return nil, chErr
|
||||||
}
|
}
|
||||||
// Routes through the active log store (CH unavailable or PG/SQLite active).
|
// Routes through the active log store (PG/SQLite active).
|
||||||
all, listErr := ListOpenFlareMetricSnapshotsSince(ctx, nodeID, since, 0)
|
all, listErr := ListOpenFlareMetricSnapshotsSince(ctx, nodeID, since, 0)
|
||||||
if listErr != nil {
|
if listErr != nil {
|
||||||
return nil, listErr
|
return nil, listErr
|
||||||
|
|||||||
@@ -401,21 +401,31 @@ func SetupLogStoresForTest(t *testing.T) {
|
|||||||
|
|
||||||
logstore.SetAccessLogHooks(logstore.AccessLogHooks{
|
logstore.SetAccessLogHooks(logstore.AccessLogHooks{
|
||||||
QueueNodeAccessLogs: func(logs []analyticsmodel.NodeAccessLog) {
|
QueueNodeAccessLogs: func(logs []analyticsmodel.NodeAccessLog) {
|
||||||
_ = store.AccessLogs.BatchInsertNodeAccessLogs(context.Background(), logs)
|
if err := store.AccessLogs.BatchInsertNodeAccessLogs(context.Background(), logs); err != nil {
|
||||||
|
t.Errorf("batch insert node access logs failed in test hook: %v", err)
|
||||||
|
}
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
logstore.SetObservabilityHooks(logstore.ObservabilityHooks{
|
logstore.SetObservabilityHooks(logstore.ObservabilityHooks{
|
||||||
QueueMetricSnapshot: func(record analyticsmodel.NodeMetricSnapshot) {
|
QueueMetricSnapshot: func(record analyticsmodel.NodeMetricSnapshot) {
|
||||||
_ = store.Observability.BatchInsertNodeMetricSnapshots(context.Background(), []analyticsmodel.NodeMetricSnapshot{record})
|
if err := store.Observability.BatchInsertNodeMetricSnapshots(context.Background(), []analyticsmodel.NodeMetricSnapshot{record}); err != nil {
|
||||||
|
t.Errorf("batch insert node metric snapshots failed in test hook: %v", err)
|
||||||
|
}
|
||||||
},
|
},
|
||||||
QueueEdgeHealth: func(record analyticsmodel.NodeEdgeHealth) {
|
QueueEdgeHealth: func(record analyticsmodel.NodeEdgeHealth) {
|
||||||
_ = store.Observability.BatchInsertNodeEdgeHealth(context.Background(), []analyticsmodel.NodeEdgeHealth{record})
|
if err := store.Observability.BatchInsertNodeEdgeHealth(context.Background(), []analyticsmodel.NodeEdgeHealth{record}); err != nil {
|
||||||
|
t.Errorf("batch insert node edge health failed in test hook: %v", err)
|
||||||
|
}
|
||||||
},
|
},
|
||||||
QueueNodeObsFrps: func(record analyticsmodel.NodeObsFrps) {
|
QueueNodeObsFrps: func(record analyticsmodel.NodeObsFrps) {
|
||||||
_ = store.Observability.BatchInsertNodeObsFrps(context.Background(), []analyticsmodel.NodeObsFrps{record})
|
if err := store.Observability.BatchInsertNodeObsFrps(context.Background(), []analyticsmodel.NodeObsFrps{record}); err != nil {
|
||||||
|
t.Errorf("batch insert node obs frps failed in test hook: %v", err)
|
||||||
|
}
|
||||||
},
|
},
|
||||||
QueueNodeObsFrpc: func(record analyticsmodel.NodeObsFrpc) {
|
QueueNodeObsFrpc: func(record analyticsmodel.NodeObsFrpc) {
|
||||||
_ = store.Observability.BatchInsertNodeObsFrpc(context.Background(), []analyticsmodel.NodeObsFrpc{record})
|
if err := store.Observability.BatchInsertNodeObsFrpc(context.Background(), []analyticsmodel.NodeObsFrpc{record}); err != nil {
|
||||||
|
t.Errorf("batch insert node obs frpc failed in test hook: %v", err)
|
||||||
|
}
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user