mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-09-29 05:56:38 +08:00
7d71f1e4e1
- New internal/repository/logstore abstraction: exported domain interfaces (AccessLogStore/ObservabilityStore/UserAccessLogStore/StatusStore), config-driven provider (Active/Build/Migrating/SetConfigReader), GORM implementation for PostgreSQL/SQLite (incl. hourly rollups computed in real time, migration listers, PG partition maintenance), and a ClickHouse wrapper preserving the native batch path; repository facade delegates to logstore; import-lint test enforces apps never import analyticsrepo. - ClickHouse is now optional: the log DB is either the main DB (postgres when database.enabled, else sqlite) or clickhouse; boot validation + first-run seed; log_database / log_db_migration are protected keys. - New user task 切换日志数据库 (of_log_db_switch): freeze log writes, drain batch writers, copy all 6 raw log tables by id (preserving IDs) with target-partition pre-creation for PG, flip log_database on success, clear the freeze flag on failure. - Per-store retention (log_retention_days_*) with expiry cleanup folded into the daily system_cleanup task; legacy database_auto_cleanup_* and of_database_auto_cleanup decommissioned. - goose migrations: 6 log tables in PG (2 monthly-partitioned) + SQLite, retention config seeds, schedule cleanup; GET /api/v1/admin/status/log-database endpoint; frontend retention settings, switch-task UI and status badge; changelog and docs updated. docs(plan): log database decoupling implementation plan docs(design): log database decoupling design (ClickHouse optional)
119 lines
3.2 KiB
Go
119 lines
3.2 KiB
Go
// Copyright 2026 Arctel.net
|
|
// SPDX-License-Identifier: Apache-2.0
|
|
|
|
// Package analytics provides ClickHouse data access for analytics tables.
|
|
package analytics
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"time"
|
|
|
|
db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
|
|
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
|
|
)
|
|
|
|
func userAccessLogConn() error {
|
|
if db.ChConn == nil {
|
|
return fmt.Errorf("clickhouse native connection is not initialized")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// CountAccessLogs returns the number of access logs matching filter.
|
|
func CountAccessLogs(ctx context.Context, filter AccessLogFilter) (uint64, error) {
|
|
clause, args, ok := buildUserAccessLogFilterClause(filter)
|
|
if !ok {
|
|
return 0, nil
|
|
}
|
|
if err := userAccessLogConn(); err != nil {
|
|
return 0, err
|
|
}
|
|
tableName := analyticsmodel.UserAccessLog{}.TableName()
|
|
sql := fmt.Sprintf("SELECT count() FROM %s WHERE %s", tableName, clause)
|
|
var count uint64
|
|
if err := db.ChConn.QueryRow(ctx, sql, args...).Scan(&count); err != nil {
|
|
return 0, fmt.Errorf("count access logs: %w", err)
|
|
}
|
|
return count, nil
|
|
}
|
|
|
|
// DeleteAllUserAccessLogs hard-deletes all user access logs via TRUNCATE.
|
|
func DeleteAllUserAccessLogs(ctx context.Context) (int64, error) {
|
|
if err := userAccessLogConn(); err != nil {
|
|
return 0, err
|
|
}
|
|
outcome, err := truncateClickHouseTable(ctx, db.ChConn, analyticsmodel.UserAccessLog{}.TableName())
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
return outcome.DeletedCount, nil
|
|
}
|
|
|
|
// ListAccessLogs returns paginated access logs and the total match count.
|
|
func ListAccessLogs(ctx context.Context, filter AccessLogFilter, page, pageSize int) ([]analyticsmodel.UserAccessLog, uint64, error) {
|
|
clause, args, ok := buildUserAccessLogFilterClause(filter)
|
|
if !ok {
|
|
return []analyticsmodel.UserAccessLog{}, 0, nil
|
|
}
|
|
if err := userAccessLogConn(); err != nil {
|
|
return nil, 0, err
|
|
}
|
|
|
|
tableName := analyticsmodel.UserAccessLog{}.TableName()
|
|
countSQL := fmt.Sprintf("SELECT count() FROM %s WHERE %s", tableName, clause)
|
|
var total uint64
|
|
if err := db.ChConn.QueryRow(ctx, countSQL, args...).Scan(&total); err != nil {
|
|
return nil, 0, fmt.Errorf("count access logs: %w", err)
|
|
}
|
|
if total == 0 {
|
|
return []analyticsmodel.UserAccessLog{}, 0, nil
|
|
}
|
|
|
|
if page < 1 {
|
|
page = 1
|
|
}
|
|
if pageSize < 1 {
|
|
pageSize = 20
|
|
}
|
|
offset := (page - 1) * pageSize
|
|
|
|
listSQL := fmt.Sprintf(`
|
|
SELECT id, user_id, path, method, ip, user_agent, headers, status, latency, created_at
|
|
FROM %s
|
|
WHERE %s
|
|
ORDER BY created_at DESC, id DESC
|
|
LIMIT ? OFFSET ?`, tableName, clause)
|
|
listArgs := append(append([]any{}, args...), pageSize, offset)
|
|
rows, err := db.ChConn.Query(ctx, listSQL, listArgs...)
|
|
if err != nil {
|
|
return nil, 0, fmt.Errorf("list access logs: %w", err)
|
|
}
|
|
defer func() { _ = rows.Close() }()
|
|
|
|
logs := make([]analyticsmodel.UserAccessLog, 0, pageSize)
|
|
for rows.Next() {
|
|
var (
|
|
item analyticsmodel.UserAccessLog
|
|
createdAt time.Time
|
|
)
|
|
if err := rows.Scan(
|
|
&item.ID,
|
|
&item.UserID,
|
|
&item.Path,
|
|
&item.Method,
|
|
&item.IP,
|
|
&item.UserAgent,
|
|
&item.Headers,
|
|
&item.Status,
|
|
&item.Latency,
|
|
&createdAt,
|
|
); err != nil {
|
|
return nil, 0, fmt.Errorf("scan access log row: %w", err)
|
|
}
|
|
item.CreatedAt = createdAt
|
|
logs = append(logs, item)
|
|
}
|
|
return logs, total, nil
|
|
}
|