Merge branch 'codex/log-database-decoupling'

# Conflicts:
#	docs/changelog/index.md
#	frontend/app/(main)/error-pages/page.tsx
#	internal/infra/persistence/migrator/migrator_test.go
This commit is contained in:
ryan
2026-08-09 08:37:37 +08:00
95 changed files with 10697 additions and 3744 deletions
-43
View File
@@ -10,7 +10,6 @@ import (
"strings"
"github.com/Rain-kl/Wavelet/internal/apps/openflare/geoip"
oftasks "github.com/Rain-kl/Wavelet/internal/apps/openflare/tasks"
"github.com/Rain-kl/Wavelet/internal/apps/openflare/uptimekuma"
"github.com/Rain-kl/Wavelet/internal/buildinfo"
"github.com/Rain-kl/Wavelet/internal/model"
@@ -50,22 +49,6 @@ type geoIPLookupView struct {
Longitude *float64 `json:"longitude,omitempty"`
}
type databaseCleanupInput struct {
Target string `json:"target"`
RetentionDays *int `json:"retention_days"`
}
type databaseCleanupResult struct {
Target string `json:"target"`
TargetLabel string `json:"target_label"`
DeletedCount int64 `json:"deleted_count"`
EligibleCount int64 `json:"eligible_count,omitempty"`
CleanupMode string `json:"cleanup_mode,omitempty"`
TableTTLDays int `json:"table_ttl_days,omitempty"`
DeleteAll bool `json:"delete_all"`
RetentionDays *int `json:"retention_days,omitempty"`
}
type optionBatchPayload struct {
Options []model.OpenFlareOption `json:"options"`
}
@@ -183,32 +166,6 @@ func lookupGeoIP(_ context.Context, provider, rawIP string) (*geoIPLookupView, e
}, nil
}
func cleanupDatabaseObservability(ctx context.Context, input databaseCleanupInput) (*databaseCleanupResult, error) {
target := strings.TrimSpace(input.Target)
if target == "" {
return nil, errors.New(errInvalidParams)
}
result, err := oftasks.CleanupDatabaseObservability(ctx, oftasks.DatabaseCleanupInput{
Target: target,
RetentionDays: input.RetentionDays,
})
if err != nil {
return nil, err
}
return &databaseCleanupResult{
Target: result.Target,
TargetLabel: result.TargetLabel,
DeletedCount: result.DeletedCount,
EligibleCount: result.EligibleCount,
CleanupMode: result.CleanupMode,
TableTTLDays: result.TableTTLDays,
DeleteAll: result.DeleteAll,
RetentionDays: result.RetentionDays,
}, nil
}
func syncUptimeKuma(ctx context.Context) error {
return uptimekuma.SyncToUptimeKuma(ctx)
}
@@ -6,7 +6,6 @@ package option
import (
"context"
"testing"
"time"
db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/model"
@@ -117,55 +116,3 @@ func TestLookupGeoIPDisabledProvider(t *testing.T) {
assert.Equal(t, "disabled", view.Provider)
assert.Equal(t, "8.8.8.8", view.IP)
}
func TestCleanupDatabaseObservabilityDeletesRows(t *testing.T) {
cleanup := setupOptionTestDB(t)
defer cleanup()
ctx := context.Background()
resetAccessLogStore := repository.SetAccessLogStoreForTest(repository.NewMemoryAccessLogStore())
defer resetAccessLogStore()
now := time.Now().UTC()
require.NoError(t, repository.InsertOpenFlareAccessLogsBatch(ctx, []*model.OpenFlareAccessLog{
{
NodeID: "node-a",
LoggedAt: now.Add(-10 * 24 * time.Hour),
RemoteAddr: "203.0.113.1",
Host: "example.com",
Path: "/old",
StatusCode: 200,
},
{
NodeID: "node-a",
LoggedAt: now.Add(-2 * time.Hour),
RemoteAddr: "203.0.113.2",
Host: "example.com",
Path: "/recent",
StatusCode: 200,
},
}))
// Retention shorter than table TTL (90d for access logs) must be rejected.
shortRetention := 7
_, err := cleanupDatabaseObservability(ctx, databaseCleanupInput{
Target: "node_access_logs",
RetentionDays: &shortRetention,
})
require.Error(t, err)
// Full truncate still hard-deletes all rows.
result, err := cleanupDatabaseObservability(ctx, databaseCleanupInput{
Target: "node_access_logs",
})
require.NoError(t, err)
assert.Equal(t, "node_access_logs", result.Target)
assert.Equal(t, "访问日志", result.TargetLabel)
assert.Equal(t, int64(2), result.DeletedCount)
assert.True(t, result.DeleteAll)
assert.Equal(t, "truncate", result.CleanupMode)
rows, err := repository.ListOpenFlareAccessLogs(ctx, model.OpenFlareAccessLogQuery{Page: 0, PageSize: 10})
require.NoError(t, err)
assert.Empty(t, rows)
}
-37
View File
@@ -4,9 +4,6 @@
package option
import (
"encoding/json"
"errors"
"io"
"net/http"
"github.com/Rain-kl/Wavelet/internal/apps/openflare/apiutil"
@@ -128,33 +125,6 @@ func LookupGeoIPHandler(c *gin.Context) {
c.JSON(http.StatusOK, response.OK(view))
}
// CleanupDatabaseHandler 清理可观测性数据库数据。
// @Summary 清理可观测性数据库
// @Description 按目标与保留天数清理可观测性相关数据表,需要管理员权限
// @Tags openflare-option
// @Accept json
// @Produce json
// @Security SessionCookie
// @Param request body option.databaseCleanupInput false "清理参数"
// @Success 200 {object} response.Any{data=option.databaseCleanupResult} "清理结果"
// @Failure 400 {object} response.Any "参数错误"
// @Failure 401 {object} response.Any "未登录"
// @Failure 404 {object} response.Any "无权限或不存在"
// @Failure 500 {object} response.Any "内部错误"
// @Router /api/v1/d/option/database/cleanup [post]
func CleanupDatabaseHandler(c *gin.Context) {
var input databaseCleanupInput
if err := bindOptionalJSON(c.Request.Body, &input); err != nil {
response.AbortBadRequest(c, errInvalidParams)
return
}
result, err := cleanupDatabaseObservability(c.Request.Context(), input)
if apiutil.AbortBadRequestOnError(c, err) {
return
}
c.JSON(http.StatusOK, response.OK(result))
}
// SyncUptimeKumaHandler 同步 Uptime Kuma 监控。
// @Summary 同步 Uptime Kuma
// @Description 将 OpenFlare 节点同步到 Uptime Kuma,需要管理员权限
@@ -174,10 +144,3 @@ func SyncUptimeKumaHandler(c *gin.Context) {
}
c.JSON(http.StatusOK, response.OK("同步成功"))
}
func bindOptionalJSON(body io.Reader, target any) error {
if err := json.NewDecoder(body).Decode(target); err != nil && !errors.Is(err, io.EOF) {
return err
}
return nil
}
+17 -5
View File
@@ -27,6 +27,17 @@ var (
const optionValueTrue = "true"
// protectedConfigKeyMessage 命中受保护 key 时返回给管理员的业务错误文案。
const protectedConfigKeyMessage = "该配置项由系统任务管理,禁止手动修改"
// protectedConfigKeys 仅允许内部(迁移任务/bootstrap)写入的 key。
var protectedConfigKeys = map[string]bool{
model.ConfigKeyLogDatabase: true,
model.ConfigKeyLogDBMigration: true,
}
func isProtectedConfigKey(key string) bool { return protectedConfigKeys[key] }
func buildOptionValidationState(ctx context.Context, options []model.OpenFlareOption) map[string]string {
// 从 SystemConfig 读取所有业务配置构建状态
configs, err := repository.ListAdminSystemConfigs(ctx, "business")
@@ -53,7 +64,7 @@ func validateOptionWithState(ctx context.Context, option model.OpenFlareOption,
if err := validateGeoIPOption(option.Key, option.Value); err != nil {
return err
}
if err := validateDatabaseCleanupOption(option.Key, option.Value); err != nil {
if err := validateLogRetentionOption(option.Key, option.Value); err != nil {
return err
}
if err := validateAgentOption(option.Key, option.Value); err != nil {
@@ -100,11 +111,9 @@ func validateGeoIPOption(key, value string) error {
return fmt.Errorf("%s 仅支持 disabled、mmdb、ip-api、geojs、ipinfo", key)
}
func validateDatabaseCleanupOption(key, value string) error {
func validateLogRetentionOption(key, value string) error {
switch key {
case model.ConfigKeyDatabaseAutoCleanupEnabled:
return validateBooleanOption(key, value)
case model.ConfigKeyDatabaseAutoCleanupRetentionDays:
case model.ConfigKeyLogRetentionDaysPostgres, model.ConfigKeyLogRetentionDaysSQLite, model.ConfigKeyLogRetentionDaysClickHouse:
intValue, err := strconv.Atoi(value)
if err != nil || intValue < 1 {
return fmt.Errorf("%s 必须为大于等于 1 的整数天", key)
@@ -221,6 +230,9 @@ func validateOptions(ctx context.Context, options []model.OpenFlareOption) error
if strings.TrimSpace(option.Key) == "" {
return errors.New(errInvalidParams)
}
if isProtectedConfigKey(option.Key) {
return errors.New(protectedConfigKeyMessage)
}
if err := validateOptionWithState(ctx, option, state); err != nil {
return err
}