perf(log): optimize PG log store queries and expired partition cleanup

- Count/节点访问日志统计改为单次扫描聚合，WAF 按 IP 聚合由三次扫描合并为两次 - IPSummaries 归属地改为取过滤窗口内最新记录（对齐 ClickHouse argMax 口径），子查询带窗口条件，可分区裁剪并命中索引 - 新增 goose 迁移：of_node_access_logs (logged_at DESC, id DESC) 前导索引与 lower(trim(host)) 表达式索引，加速列表排序与主机过滤 - 过期日志清理先按数据校验直接 DROP 完全过期整月分区，再对边界月逐行删除；启动时兜底预建当月及未来 2 个月分区，跨月停机重启后首次写入不再报 "no partition of relation found"
2026-10-06 23:56:37 +08:00 · 2026-08-09 10:20:58 +08:00
parent 08d28c2c8e
commit f530cd4025
12 changed files with 519 additions and 84 deletions
@@ -245,6 +245,220 @@ func TestDropEmptyPartitionsPostgres(t *testing.T) {
 	}
 }

+// TestDropExpiredPartitionsPostgres 需要 TEST_POSTGRES_DSN（未设置时跳过）：
+// 验证直接删除完全早于 cutoff 月份的整月分区：早于 cutoff 月的分区（含其中全部数据）被整表 DROP、
+// 边界月分区保留且数据仍在；重复调用幂等；w_user_access_logs 分区不受影响（无 retention 清理）。
+func TestDropExpiredPartitionsPostgres(t *testing.T) {
+	dsn := strings.TrimSpace(os.Getenv("TEST_POSTGRES_DSN"))
+	if dsn == "" {
+		t.Skip("TEST_POSTGRES_DSN is not set")
+	}
+
+	gdb, err := gorm.Open(postgres.Open(dsn), &gorm.Config{
+		DisableForeignKeyConstraintWhenMigrating: true,
+		Logger:                                   logger.Default.LogMode(logger.Silent),
+	})
+	if err != nil {
+		t.Fatalf("open postgres: %v", err)
+	}
+	sqlDB, err := gdb.DB()
+	if err != nil {
+		t.Fatalf("sql db: %v", err)
+	}
+	sqlDB.SetMaxOpenConns(1)
+
+	schema := fmt.Sprintf("logstore_drop_expired_%d", time.Now().UnixNano())
+	if !regexp.MustCompile(`^[a-z0-9_]+$`).MatchString(schema) {
+		t.Fatalf("invalid schema: %s", schema)
+	}
+	if err := gdb.Exec(`CREATE SCHEMA "` + schema + `"`).Error; err != nil {
+		t.Fatalf("create schema: %v", err)
+	}
+	if err := gdb.Exec(`SET search_path TO "` + schema + `"`).Error; err != nil {
+		t.Fatalf("set search_path: %v", err)
+	}
+	t.Cleanup(func() {
+		_ = gdb.Exec("SET search_path TO public").Error
+		_ = gdb.Exec(`DROP SCHEMA IF EXISTS "` + schema + `" CASCADE`).Error
+		_ = sqlDB.Close()
+	})
+
+	for _, ddl := range []string{postgresNodeAccessLogsDDL, postgresUserAccessLogsDDL} {
+		if err := gdb.Exec(ddl).Error; err != nil {
+			t.Fatalf("create partitioned table: %v", err)
+		}
+	}
+
+	ctx := context.Background()
+	store := newGormStore(gdb)
+	ua := newUserAccessLogGormStore(gdb)
+
+	// 预建 202601..202604 分区；1/3 月有数据、2/4 月为空。
+	if err := store.EnsurePartitions(ctx,
+		time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC),
+		time.Date(2026, 4, 20, 0, 0, 0, 0, time.UTC)); err != nil {
+		t.Fatalf("EnsurePartitions: %v", err)
+	}
+	if err := store.BatchInsertNodeAccessLogs(ctx, []analyticsmodel.NodeAccessLog{
+		{ID: 1, NodeID: "n1", LoggedAt: time.Date(2026, 1, 15, 0, 0, 0, 0, time.UTC), RemoteAddr: "1.1.1.1"},
+		{ID: 2, NodeID: "n1", LoggedAt: time.Date(2026, 1, 20, 0, 0, 0, 0, time.UTC), RemoteAddr: "1.1.1.2"},
+		{ID: 3, NodeID: "n2", LoggedAt: time.Date(2026, 3, 5, 0, 0, 0, 0, time.UTC), RemoteAddr: "3.3.3.3"},
+		{ID: 4, NodeID: "n2", LoggedAt: time.Date(2026, 3, 18, 0, 0, 0, 0, time.UTC), RemoteAddr: "3.3.3.4"},
+	}); err != nil {
+		t.Fatalf("insert node access logs: %v", err)
+	}
+	if err := ua.BatchInsert(ctx, []analyticsmodel.UserAccessLog{
+		{ID: 1, UserID: 101, Path: "/a", CreatedAt: time.Date(2026, 1, 16, 0, 0, 0, 0, time.UTC)},
+	}); err != nil {
+		t.Fatalf("insert user access log: %v", err)
+	}
+
+	// cutoff=2026-03-10：分区月份早于 2026-03 的（202601、202602）整表 DROP；
+	// 202603（边界月，可能含未过期数据）与 202604（未来月）保留。
+	if err := store.DropExpiredPartitions(ctx, time.Date(2026, 3, 10, 0, 0, 0, 0, time.UTC)); err != nil {
+		t.Fatalf("DropExpiredPartitions: %v", err)
+	}
+	// 幂等：重复调用不报错、不额外删除。
+	if err := store.DropExpiredPartitions(ctx, time.Date(2026, 3, 10, 0, 0, 0, 0, time.UTC)); err != nil {
+		t.Fatalf("DropExpiredPartitions idempotent: %v", err)
+	}
+
+	assertPartitions := func(parent string, want int64) {
+		t.Helper()
+		var n int64
+		if err := gdb.Raw(
+			"SELECT count(*) FROM pg_inherits WHERE inhparent = to_regclass(?)",
+			parent,
+		).Scan(&n).Error; err != nil {
+			t.Fatalf("count partitions of %s: %v", parent, err)
+		}
+		if n != want {
+			t.Fatalf("%s partitions = %d, want %d", parent, n, want)
+		}
+	}
+	// of_node_access_logs 只剩边界月+未来月 2 个分区；w_user_access_logs 不受影响（仍 4 个）。
+	assertPartitions("of_node_access_logs", 2)
+	assertPartitions("w_user_access_logs", 4)
+
+	// 202601/202602 分区被整表 DROP：1 月数据随之消失，3 月数据保留。
+	var nodeCount int64
+	if err := gdb.Model(&analyticsmodel.NodeAccessLog{}).Count(&nodeCount).Error; err != nil {
+		t.Fatalf("count node access logs: %v", err)
+	}
+	if nodeCount != 2 {
+		t.Fatalf("node access log count = %d, want 2（仅剩 3 月数据）", nodeCount)
+	}
+}
+
+// TestDropExpiredPartitionsTimezoneSafety 需要 TEST_POSTGRES_DSN（未设置时跳过）：
+// 覆盖本地时区偏移下 DropExpiredPartitions 的时区安全性：cutoff 为 UTC+8 本地时刻
+// （其实刻 = 2026-02-28T21:00Z），名称月份早于 cutoff 月但分区内仍含保留期行的
+// 202602 不得被误删（旧实现按本地月份取 cutoffMonth=2026-03 会整表 DROP 丢数据）；
+// 完全过期的 202601 正常整表 DROP；保留期行仍可查询到。
+func TestDropExpiredPartitionsTimezoneSafety(t *testing.T) {
+	dsn := strings.TrimSpace(os.Getenv("TEST_POSTGRES_DSN"))
+	if dsn == "" {
+		t.Skip("TEST_POSTGRES_DSN is not set")
+	}
+
+	gdb, err := gorm.Open(postgres.Open(dsn), &gorm.Config{
+		DisableForeignKeyConstraintWhenMigrating: true,
+		Logger:                                   logger.Default.LogMode(logger.Silent),
+	})
+	if err != nil {
+		t.Fatalf("open postgres: %v", err)
+	}
+	sqlDB, err := gdb.DB()
+	if err != nil {
+		t.Fatalf("sql db: %v", err)
+	}
+	sqlDB.SetMaxOpenConns(1)
+
+	schema := fmt.Sprintf("logstore_drop_expired_tz_%d", time.Now().UnixNano())
+	if !regexp.MustCompile(`^[a-z0-9_]+$`).MatchString(schema) {
+		t.Fatalf("invalid schema: %s", schema)
+	}
+	if err := gdb.Exec(`CREATE SCHEMA "` + schema + `"`).Error; err != nil {
+		t.Fatalf("create schema: %v", err)
+	}
+	if err := gdb.Exec(`SET search_path TO "` + schema + `"`).Error; err != nil {
+		t.Fatalf("set search_path: %v", err)
+	}
+	t.Cleanup(func() {
+		_ = gdb.Exec("SET search_path TO public").Error
+		_ = gdb.Exec(`DROP SCHEMA IF EXISTS "` + schema + `" CASCADE`).Error
+		_ = sqlDB.Close()
+	})
+
+	for _, ddl := range []string{postgresNodeAccessLogsDDL, postgresUserAccessLogsDDL} {
+		if err := gdb.Exec(ddl).Error; err != nil {
+			t.Fatalf("create partitioned table: %v", err)
+		}
+	}
+
+	ctx := context.Background()
+	store := newGormStore(gdb)
+
+	// 预建 202601..202602 分区。
+	if err := store.EnsurePartitions(ctx,
+		time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC),
+		time.Date(2026, 2, 20, 0, 0, 0, 0, time.UTC)); err != nil {
+		t.Fatalf("EnsurePartitions: %v", err)
+	}
+	// 202601 仅含完全过期行；202602 含一条过期行（2026-02-10）与一条保留期行
+	// （2026-02-28T21:00Z，恰等于 cutoff 其实刻，>= 语义下必须保留）。
+	if err := store.BatchInsertNodeAccessLogs(ctx, []analyticsmodel.NodeAccessLog{
+		{ID: 1, NodeID: "n1", LoggedAt: time.Date(2026, 1, 15, 0, 0, 0, 0, time.UTC), RemoteAddr: "1.1.1.1"},
+		{ID: 2, NodeID: "n1", LoggedAt: time.Date(2026, 2, 10, 0, 0, 0, 0, time.UTC), RemoteAddr: "2.2.2.2"},
+		{ID: 3, NodeID: "n1", LoggedAt: time.Date(2026, 2, 28, 21, 0, 0, 0, time.UTC), RemoteAddr: "3.3.3.3"},
+	}); err != nil {
+		t.Fatalf("insert node access logs: %v", err)
+	}
+
+	// cutoff 为 UTC+8 本地时刻 2026-03-01 05:00，其实刻 = 2026-02-28T21:00Z：
+	// 旧实现按本地月份取 cutoffMonth=2026-03 会把 202602 误判为完全过期整表 DROP。
+	cutoff := time.Date(2026, 3, 1, 5, 0, 0, 0, time.FixedZone("UTC+8", 8*3600))
+	if err := store.DropExpiredPartitions(ctx, cutoff); err != nil {
+		t.Fatalf("DropExpiredPartitions: %v", err)
+	}
+
+	assertPartitions := func(parent string, want int64) {
+		t.Helper()
+		var n int64
+		if err := gdb.Raw(
+			"SELECT count(*) FROM pg_inherits WHERE inhparent = to_regclass(?)",
+			parent,
+		).Scan(&n).Error; err != nil {
+			t.Fatalf("count partitions of %s: %v", parent, err)
+		}
+		if n != want {
+			t.Fatalf("%s partitions = %d, want %d", parent, n, want)
+		}
+	}
+	// 202601 已整表 DROP，202602 保留；w_user_access_logs 不受影响（仍 2 个）。
+	assertPartitions("of_node_access_logs", 1)
+	assertPartitions("w_user_access_logs", 2)
+
+	// 202601 数据随之消失，202602 内保留期行（2026-02-28T21:00Z）仍可查询到。
+	var nodeCount int64
+	if err := gdb.Model(&analyticsmodel.NodeAccessLog{}).Count(&nodeCount).Error; err != nil {
+		t.Fatalf("count node access logs: %v", err)
+	}
+	if nodeCount != 2 {
+		t.Fatalf("node access log count = %d, want 2（仅剩 202602 两行）", nodeCount)
+	}
+	var retained int64
+	if err := gdb.Raw(
+		"SELECT count(*) FROM of_node_access_logs WHERE logged_at >= ?",
+		cutoff.UTC(),
+	).Scan(&retained).Error; err != nil {
+		t.Fatalf("count retained rows: %v", err)
+	}
+	if retained != 1 {
+		t.Fatalf("retained rows (logged_at >= cutoff) = %d, want 1", retained)
+	}
+}
+
 // postgresNodeAccessLogsDDL 与 goose/postgres/202608080001_create_log_tables.sql 对齐。
 const postgresNodeAccessLogsDDL = `
 CREATE TABLE IF NOT EXISTS of_node_access_logs (