mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-11 09:46:37 +08:00
perf(log): optimize PG log store queries and expired partition cleanup
- Count/节点访问日志统计改为单次扫描聚合,WAF 按 IP 聚合由三次扫描合并为两次 - IPSummaries 归属地改为取过滤窗口内最新记录(对齐 ClickHouse argMax 口径), 子查询带窗口条件,可分区裁剪并命中索引 - 新增 goose 迁移:of_node_access_logs (logged_at DESC, id DESC) 前导索引与 lower(trim(host)) 表达式索引,加速列表排序与主机过滤 - 过期日志清理先按数据校验直接 DROP 完全过期整月分区,再对边界月逐行删除; 启动时兜底预建当月及未来 2 个月分区,跨月停机重启后首次写入不再报 "no partition of relation found"
This commit is contained in:
@@ -369,3 +369,35 @@ func TestDropEligiblePartitionNames(t *testing.T) {
|
||||
t.Fatalf("eligible at month boundary = %v, want empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDropExpiredPartitionsSQLiteNoop 验证 SQLite 下 DropExpiredPartitions 为 no-op:
|
||||
// 直接返回 nil、不触碰任何分区 SQL(SQLite 无分区),数据不受影响。
|
||||
func TestDropExpiredPartitionsSQLiteNoop(t *testing.T) {
|
||||
ResetForTest()
|
||||
SetConfigReader(func(_ context.Context, key string) (string, error) {
|
||||
if key == logDatabaseKey {
|
||||
return "sqlite", nil
|
||||
}
|
||||
return "", nil
|
||||
})
|
||||
defer ResetForTest()
|
||||
|
||||
gdb := newCleanupTestDB(t)
|
||||
ctx := context.Background()
|
||||
if err := gdb.Create(&analyticsmodel.NodeAccessLog{ID: 1, NodeID: "n1", LoggedAt: time.Now().AddDate(0, 0, -100).UTC(), RemoteAddr: "1.1.1.1"}).Error; err != nil {
|
||||
t.Fatalf("seed node access log: %v", err)
|
||||
}
|
||||
|
||||
store := newGormStore(gdb)
|
||||
if err := store.DropExpiredPartitions(ctx, time.Now().AddDate(0, 0, -90)); err != nil {
|
||||
t.Fatalf("DropExpiredPartitions on sqlite: %v", err)
|
||||
}
|
||||
|
||||
var n int64
|
||||
if err := gdb.Model(&analyticsmodel.NodeAccessLog{}).Count(&n).Error; err != nil {
|
||||
t.Fatalf("count node access logs: %v", err)
|
||||
}
|
||||
if n != 1 {
|
||||
t.Fatalf("node access log count = %d, want 1(no-op 不应删除任何行)", n)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user