Files
OpenFlare/internal/repository/analytics/access_log.go
T
ryan 7d71f1e4e1 feat(log): decouple log storage from ClickHouse with switchable logstore
- New internal/repository/logstore abstraction: exported domain interfaces
  (AccessLogStore/ObservabilityStore/UserAccessLogStore/StatusStore),
  config-driven provider (Active/Build/Migrating/SetConfigReader), GORM
  implementation for PostgreSQL/SQLite (incl. hourly rollups computed in
  real time, migration listers, PG partition maintenance), and a ClickHouse
  wrapper preserving the native batch path; repository facade delegates to
  logstore; import-lint test enforces apps never import analyticsrepo.
- ClickHouse is now optional: the log DB is either the main DB (postgres
  when database.enabled, else sqlite) or clickhouse; boot validation +
  first-run seed; log_database / log_db_migration are protected keys.
- New user task 切换日志数据库 (of_log_db_switch): freeze log writes,
  drain batch writers, copy all 6 raw log tables by id (preserving IDs)
  with target-partition pre-creation for PG, flip log_database on success,
  clear the freeze flag on failure.
- Per-store retention (log_retention_days_*) with expiry cleanup folded
  into the daily system_cleanup task; legacy database_auto_cleanup_* and
  of_database_auto_cleanup decommissioned.
- goose migrations: 6 log tables in PG (2 monthly-partitioned) + SQLite,
  retention config seeds, schedule cleanup; GET
  /api/v1/admin/status/log-database endpoint; frontend retention settings,
  switch-task UI and status badge; changelog and docs updated.

docs(plan): log database decoupling implementation plan

docs(design): log database decoupling design (ClickHouse optional)
2026-08-08 19:43:01 +08:00

119 lines
3.2 KiB
Go

// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package analytics provides ClickHouse data access for analytics tables.
package analytics
import (
"context"
"fmt"
"time"
db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
)
func userAccessLogConn() error {
if db.ChConn == nil {
return fmt.Errorf("clickhouse native connection is not initialized")
}
return nil
}
// CountAccessLogs returns the number of access logs matching filter.
func CountAccessLogs(ctx context.Context, filter AccessLogFilter) (uint64, error) {
clause, args, ok := buildUserAccessLogFilterClause(filter)
if !ok {
return 0, nil
}
if err := userAccessLogConn(); err != nil {
return 0, err
}
tableName := analyticsmodel.UserAccessLog{}.TableName()
sql := fmt.Sprintf("SELECT count() FROM %s WHERE %s", tableName, clause)
var count uint64
if err := db.ChConn.QueryRow(ctx, sql, args...).Scan(&count); err != nil {
return 0, fmt.Errorf("count access logs: %w", err)
}
return count, nil
}
// DeleteAllUserAccessLogs hard-deletes all user access logs via TRUNCATE.
func DeleteAllUserAccessLogs(ctx context.Context) (int64, error) {
if err := userAccessLogConn(); err != nil {
return 0, err
}
outcome, err := truncateClickHouseTable(ctx, db.ChConn, analyticsmodel.UserAccessLog{}.TableName())
if err != nil {
return 0, err
}
return outcome.DeletedCount, nil
}
// ListAccessLogs returns paginated access logs and the total match count.
func ListAccessLogs(ctx context.Context, filter AccessLogFilter, page, pageSize int) ([]analyticsmodel.UserAccessLog, uint64, error) {
clause, args, ok := buildUserAccessLogFilterClause(filter)
if !ok {
return []analyticsmodel.UserAccessLog{}, 0, nil
}
if err := userAccessLogConn(); err != nil {
return nil, 0, err
}
tableName := analyticsmodel.UserAccessLog{}.TableName()
countSQL := fmt.Sprintf("SELECT count() FROM %s WHERE %s", tableName, clause)
var total uint64
if err := db.ChConn.QueryRow(ctx, countSQL, args...).Scan(&total); err != nil {
return nil, 0, fmt.Errorf("count access logs: %w", err)
}
if total == 0 {
return []analyticsmodel.UserAccessLog{}, 0, nil
}
if page < 1 {
page = 1
}
if pageSize < 1 {
pageSize = 20
}
offset := (page - 1) * pageSize
listSQL := fmt.Sprintf(`
SELECT id, user_id, path, method, ip, user_agent, headers, status, latency, created_at
FROM %s
WHERE %s
ORDER BY created_at DESC, id DESC
LIMIT ? OFFSET ?`, tableName, clause)
listArgs := append(append([]any{}, args...), pageSize, offset)
rows, err := db.ChConn.Query(ctx, listSQL, listArgs...)
if err != nil {
return nil, 0, fmt.Errorf("list access logs: %w", err)
}
defer func() { _ = rows.Close() }()
logs := make([]analyticsmodel.UserAccessLog, 0, pageSize)
for rows.Next() {
var (
item analyticsmodel.UserAccessLog
createdAt time.Time
)
if err := rows.Scan(
&item.ID,
&item.UserID,
&item.Path,
&item.Method,
&item.IP,
&item.UserAgent,
&item.Headers,
&item.Status,
&item.Latency,
&createdAt,
); err != nil {
return nil, 0, fmt.Errorf("scan access log row: %w", err)
}
item.CreatedAt = createdAt
logs = append(logs, item)
}
return logs, total, nil
}