mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-04 15:06:37 +08:00
feat(log): drop empty old-month PG partitions during cleanup
系统垃圾清理任务删除过期日志后,顺带清理旧月份空分区表: PostgreSQL 按月分区的访问日志表(节点/用户)在数据删除后若该月 分区已无数据,则自动删除对应分区表,避免历史分区表无限累积。 - 仅删除「当前月之前」且为空的月份分区,当月/未来月及仍有数据的分区保留 - ClickHouse/SQLite 为 no-op(CH 分区随数据删除自动消失) - 修复既有集成测试 pg_inherits 查询(inhrelid → inhparent) - 新增单元测试与 PG 集成测试
This commit is contained in:
@@ -86,7 +86,7 @@ func TestEnsurePartitionsPostgresInsertAcrossMonths(t *testing.T) {
|
||||
|
||||
var partitionCount int64
|
||||
if err := gdb.Raw(
|
||||
"SELECT count(*) FROM pg_inherits WHERE inhrelid = to_regclass('of_node_access_logs')",
|
||||
"SELECT count(*) FROM pg_inherits WHERE inhparent = to_regclass('of_node_access_logs')",
|
||||
).Scan(&partitionCount).Error; err != nil {
|
||||
t.Fatalf("count partitions: %v", err)
|
||||
}
|
||||
@@ -146,6 +146,105 @@ func TestEnsurePartitionsPostgresInsertAcrossMonths(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestDropEmptyPartitionsPostgres 需要 TEST_POSTGRES_DSN(未设置时跳过):
|
||||
// 验证空分区清理只删除 before 月份之前且无数据的分区:空旧月删除、有数据旧月保留、
|
||||
// 当月/未来月保留;用户访问日志分区同步清理。
|
||||
func TestDropEmptyPartitionsPostgres(t *testing.T) {
|
||||
dsn := strings.TrimSpace(os.Getenv("TEST_POSTGRES_DSN"))
|
||||
if dsn == "" {
|
||||
t.Skip("TEST_POSTGRES_DSN is not set")
|
||||
}
|
||||
|
||||
gdb, err := gorm.Open(postgres.Open(dsn), &gorm.Config{
|
||||
DisableForeignKeyConstraintWhenMigrating: true,
|
||||
Logger: logger.Default.LogMode(logger.Silent),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("open postgres: %v", err)
|
||||
}
|
||||
sqlDB, err := gdb.DB()
|
||||
if err != nil {
|
||||
t.Fatalf("sql db: %v", err)
|
||||
}
|
||||
sqlDB.SetMaxOpenConns(1)
|
||||
|
||||
schema := fmt.Sprintf("logstore_drop_partition_%d", time.Now().UnixNano())
|
||||
if !regexp.MustCompile(`^[a-z0-9_]+$`).MatchString(schema) {
|
||||
t.Fatalf("invalid schema: %s", schema)
|
||||
}
|
||||
if err := gdb.Exec(`CREATE SCHEMA "` + schema + `"`).Error; err != nil {
|
||||
t.Fatalf("create schema: %v", err)
|
||||
}
|
||||
if err := gdb.Exec(`SET search_path TO "` + schema + `"`).Error; err != nil {
|
||||
t.Fatalf("set search_path: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_ = gdb.Exec("SET search_path TO public").Error
|
||||
_ = gdb.Exec(`DROP SCHEMA IF EXISTS "` + schema + `" CASCADE`).Error
|
||||
_ = sqlDB.Close()
|
||||
})
|
||||
|
||||
for _, ddl := range []string{postgresNodeAccessLogsDDL, postgresUserAccessLogsDDL} {
|
||||
if err := gdb.Exec(ddl).Error; err != nil {
|
||||
t.Fatalf("create partitioned table: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
store := newGormStore(gdb)
|
||||
ua := newUserAccessLogGormStore(gdb)
|
||||
|
||||
// 预建 202601..202603 分区,仅 202602 有数据(节点+用户各 1 条),202601/202603 为空。
|
||||
if err := store.EnsurePartitions(ctx,
|
||||
time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC),
|
||||
time.Date(2026, 3, 20, 0, 0, 0, 0, time.UTC)); err != nil {
|
||||
t.Fatalf("EnsurePartitions: %v", err)
|
||||
}
|
||||
if err := store.BatchInsertNodeAccessLogs(ctx, []analyticsmodel.NodeAccessLog{
|
||||
{ID: 1, NodeID: "n1", LoggedAt: time.Date(2026, 2, 10, 0, 0, 0, 0, time.UTC), RemoteAddr: "1.1.1.1"},
|
||||
}); err != nil {
|
||||
t.Fatalf("insert node access log: %v", err)
|
||||
}
|
||||
if err := ua.BatchInsert(ctx, []analyticsmodel.UserAccessLog{
|
||||
{ID: 1, UserID: 101, Path: "/a", CreatedAt: time.Date(2026, 2, 11, 0, 0, 0, 0, time.UTC)},
|
||||
}); err != nil {
|
||||
t.Fatalf("insert user access log: %v", err)
|
||||
}
|
||||
|
||||
// before=2026-03:202601(空)应删,202602(有数据)与 202603(当月)保留。
|
||||
if err := store.DropEmptyPartitions(ctx, time.Date(2026, 3, 15, 0, 0, 0, 0, time.UTC)); err != nil {
|
||||
t.Fatalf("DropEmptyPartitions: %v", err)
|
||||
}
|
||||
|
||||
assertPartitions := func(parent string, want int64) {
|
||||
t.Helper()
|
||||
var n int64
|
||||
if err := gdb.Raw(
|
||||
"SELECT count(*) FROM pg_inherits WHERE inhparent = to_regclass(?)",
|
||||
parent,
|
||||
).Scan(&n).Error; err != nil {
|
||||
t.Fatalf("count partitions of %s: %v", parent, err)
|
||||
}
|
||||
if n != want {
|
||||
t.Fatalf("%s partitions = %d, want %d", parent, n, want)
|
||||
}
|
||||
}
|
||||
assertPartitions("of_node_access_logs", 2)
|
||||
assertPartitions("w_user_access_logs", 2)
|
||||
|
||||
// 数据未受影响。
|
||||
var nodeCount, userCount int64
|
||||
if err := gdb.Model(&analyticsmodel.NodeAccessLog{}).Count(&nodeCount).Error; err != nil {
|
||||
t.Fatalf("count node access logs: %v", err)
|
||||
}
|
||||
if err := gdb.Model(&analyticsmodel.UserAccessLog{}).Count(&userCount).Error; err != nil {
|
||||
t.Fatalf("count user access logs: %v", err)
|
||||
}
|
||||
if nodeCount != 1 || userCount != 1 {
|
||||
t.Fatalf("data counts = (%d, %d), want (1, 1)", nodeCount, userCount)
|
||||
}
|
||||
}
|
||||
|
||||
// postgresNodeAccessLogsDDL 与 goose/postgres/202608080001_create_log_tables.sql 对齐。
|
||||
const postgresNodeAccessLogsDDL = `
|
||||
CREATE TABLE IF NOT EXISTS of_node_access_logs (
|
||||
|
||||
Reference in New Issue
Block a user