mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-04 15:06:37 +08:00
refactor(clickhouse): route of_node_access_logs through analytics repository
Move all ClickHouse DML for OpenFlare node access logs into internal/repository/analytics; model layer keeps domain aggregation and in-memory test store via a thin adapter. Aligns with goose-managed DDL and openflare database startup dependency.
This commit is contained in:
@@ -345,68 +345,6 @@ func openFlareAccessLogQueryFromBucket(query OpenFlareAccessLogBucketQuery) Open
|
||||
}
|
||||
}
|
||||
|
||||
func buildOpenFlareAccessLogFilterClause(query OpenFlareAccessLogQuery) (string, []any) {
|
||||
parts := make([]string, 0, 6)
|
||||
args := make([]any, 0, 6)
|
||||
if trimmed := strings.TrimSpace(query.NodeID); trimmed != "" {
|
||||
parts = append(parts, "node_id = ?")
|
||||
args = append(args, trimmed)
|
||||
}
|
||||
if trimmed := strings.TrimSpace(query.RemoteAddr); trimmed != "" {
|
||||
parts = append(parts, "remote_addr LIKE ?")
|
||||
args = append(args, trimmed+"%")
|
||||
}
|
||||
if trimmed := strings.TrimSpace(query.Host); trimmed != "" {
|
||||
parts = append(parts, "host LIKE ?")
|
||||
args = append(args, trimmed+"%")
|
||||
}
|
||||
if trimmed := strings.TrimSpace(query.Path); trimmed != "" {
|
||||
parts = append(parts, "path LIKE ?")
|
||||
args = append(args, trimmed+"%")
|
||||
}
|
||||
if !query.Since.IsZero() {
|
||||
parts = append(parts, "logged_at >= ?")
|
||||
args = append(args, query.Since.UTC())
|
||||
}
|
||||
if !query.Until.IsZero() {
|
||||
parts = append(parts, "logged_at < ?")
|
||||
args = append(args, query.Until.UTC())
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return "1", nil
|
||||
}
|
||||
return strings.Join(parts, " AND "), args
|
||||
}
|
||||
|
||||
func combineOpenFlareAccessLogSQLClauses(left string, right string) string {
|
||||
if strings.TrimSpace(left) == "" || left == "TRUE" || left == "1" {
|
||||
return right
|
||||
}
|
||||
return left + " AND " + right
|
||||
}
|
||||
|
||||
func openFlareAccessLogOrderClause(sortBy string, sortOrder string) string {
|
||||
direction := "DESC"
|
||||
if openFlareAccessLogNormalizeSortOrder(sortOrder) == "asc" {
|
||||
direction = "ASC"
|
||||
}
|
||||
column := "logged_at"
|
||||
switch strings.TrimSpace(sortBy) {
|
||||
case "status_code":
|
||||
column = "status_code"
|
||||
case "remote_addr":
|
||||
column = "remote_addr"
|
||||
case "host":
|
||||
column = "host"
|
||||
case "path":
|
||||
column = "path"
|
||||
}
|
||||
if column == "logged_at" {
|
||||
return column + " " + direction + ", id " + direction
|
||||
}
|
||||
return column + " " + direction + ", logged_at " + direction + ", id " + direction
|
||||
}
|
||||
|
||||
func sortOpenFlareAccessLogBucketIPRows(items []*OpenFlareAccessLogBucketIPRow, sortBy string, sortOrder string) {
|
||||
desc := openFlareAccessLogNormalizeSortOrder(sortOrder) != "asc"
|
||||
sort.Slice(items, func(i, j int) bool {
|
||||
|
||||
@@ -5,19 +5,13 @@ package model
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
|
||||
"github.com/Rain-kl/Wavelet/internal/db"
|
||||
"github.com/Rain-kl/Wavelet/internal/db/idgen"
|
||||
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
|
||||
analyticsrepo "github.com/Rain-kl/Wavelet/internal/repository/analytics"
|
||||
)
|
||||
|
||||
const openFlareAccessLogTable = "of_node_access_logs"
|
||||
|
||||
type accessLogStore interface {
|
||||
InsertBatch(ctx context.Context, records []*OpenFlareAccessLog) error
|
||||
List(ctx context.Context, query OpenFlareAccessLogQuery) ([]*OpenFlareAccessLog, error)
|
||||
@@ -69,397 +63,183 @@ func NewMemoryAccessLogStore() accessLogStore {
|
||||
|
||||
type clickhouseAccessLogStore struct{}
|
||||
|
||||
func (clickhouseAccessLogStore) conn() (driver.Conn, error) {
|
||||
if db.ChConn == nil {
|
||||
return nil, errors.New(errClickHouseNotInitialized)
|
||||
}
|
||||
return db.ChConn, nil
|
||||
}
|
||||
|
||||
func (clickhouseAccessLogStore) InsertBatch(ctx context.Context, records []*OpenFlareAccessLog) error {
|
||||
if len(records) == 0 {
|
||||
return nil
|
||||
}
|
||||
conn, err := clickhouseAccessLogStore{}.conn()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
batch, err := conn.PrepareBatch(ctx, fmt.Sprintf(
|
||||
"INSERT INTO %s (id, node_id, logged_at, remote_addr, region, host, path, status_code, created_at)",
|
||||
openFlareAccessLogTable,
|
||||
))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
logs := make([]analyticsmodel.NodeAccessLog, 0, len(records))
|
||||
for _, record := range records {
|
||||
if record == nil {
|
||||
continue
|
||||
}
|
||||
id := record.ID
|
||||
if id == 0 {
|
||||
id = uint(idgen.NextUint64ID())
|
||||
}
|
||||
createdAt := record.CreatedAt
|
||||
if createdAt.IsZero() {
|
||||
createdAt = now
|
||||
}
|
||||
if err := batch.Append(
|
||||
uint64(id),
|
||||
record.NodeID,
|
||||
record.LoggedAt.UTC(),
|
||||
record.RemoteAddr,
|
||||
record.Region,
|
||||
record.Host,
|
||||
record.Path,
|
||||
int32(record.StatusCode),
|
||||
createdAt.UTC(),
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
logs = append(logs, toAnalyticsNodeAccessLog(record))
|
||||
}
|
||||
return batch.Send()
|
||||
return analyticsrepo.BatchInsertNodeAccessLogs(ctx, logs)
|
||||
}
|
||||
|
||||
func (clickhouseAccessLogStore) List(ctx context.Context, query OpenFlareAccessLogQuery) ([]*OpenFlareAccessLog, error) {
|
||||
conn, err := clickhouseAccessLogStore{}.conn()
|
||||
rows, err := analyticsrepo.ListNodeAccessLogs(ctx, toNodeAccessLogFilter(query))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clause, args := buildOpenFlareAccessLogFilterClause(query)
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT id, node_id, logged_at, remote_addr, region, host, path, status_code, created_at
|
||||
FROM %s
|
||||
WHERE %s
|
||||
ORDER BY %s`, openFlareAccessLogTable, clause, openFlareAccessLogOrderClause(query.SortBy, query.SortOrder))
|
||||
if query.PageSize > 0 {
|
||||
if query.Page < 0 {
|
||||
query.Page = 0
|
||||
}
|
||||
sql += " LIMIT ? OFFSET ?"
|
||||
args = append(args, query.PageSize, query.Page*query.PageSize)
|
||||
}
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
return scanOpenFlareAccessLogRows(rows)
|
||||
}
|
||||
|
||||
func scanOpenFlareAccessLogRows(rows driver.Rows) ([]*OpenFlareAccessLog, error) {
|
||||
var result []*OpenFlareAccessLog
|
||||
for rows.Next() {
|
||||
var (
|
||||
id uint64
|
||||
nodeID string
|
||||
loggedAt time.Time
|
||||
remoteAddr string
|
||||
region string
|
||||
host string
|
||||
path string
|
||||
statusCode int32
|
||||
createdAt time.Time
|
||||
)
|
||||
if err := rows.Scan(&id, &nodeID, &loggedAt, &remoteAddr, ®ion, &host, &path, &statusCode, &createdAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, &OpenFlareAccessLog{
|
||||
ID: uint(id),
|
||||
NodeID: nodeID,
|
||||
LoggedAt: loggedAt.UTC(),
|
||||
RemoteAddr: remoteAddr,
|
||||
Region: region,
|
||||
Host: host,
|
||||
Path: path,
|
||||
StatusCode: int(statusCode),
|
||||
CreatedAt: createdAt.UTC(),
|
||||
})
|
||||
}
|
||||
return result, nil
|
||||
return fromAnalyticsNodeAccessLogs(rows), nil
|
||||
}
|
||||
|
||||
func (clickhouseAccessLogStore) Count(ctx context.Context, query OpenFlareAccessLogQuery) (int64, int64, error) {
|
||||
conn, err := clickhouseAccessLogStore{}.conn()
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
clause, args := buildOpenFlareAccessLogFilterClause(query)
|
||||
var totalRecords int64
|
||||
countSQL := fmt.Sprintf("SELECT count() FROM %s WHERE %s", openFlareAccessLogTable, clause)
|
||||
if err := conn.QueryRow(ctx, countSQL, args...).Scan(&totalRecords); err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
ipSQL := fmt.Sprintf(`
|
||||
SELECT count() FROM (
|
||||
SELECT trim(remote_addr) AS remote_addr
|
||||
FROM %s
|
||||
WHERE %s AND remote_addr != ''
|
||||
GROUP BY trim(remote_addr)
|
||||
)`, openFlareAccessLogTable, clause)
|
||||
var totalIPs int64
|
||||
if err := conn.QueryRow(ctx, ipSQL, args...).Scan(&totalIPs); err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
return totalRecords, totalIPs, nil
|
||||
return analyticsrepo.CountNodeAccessLogs(ctx, toNodeAccessLogFilter(query))
|
||||
}
|
||||
|
||||
func (clickhouseAccessLogStore) RegionCounts(ctx context.Context, nodeID string, since time.Time, limit int) ([]*OpenFlareAccessLogRegionCount, error) {
|
||||
conn, err := clickhouseAccessLogStore{}.conn()
|
||||
rows, err := analyticsrepo.RegionCountsNodeAccessLogs(ctx, nodeID, since, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
filter := OpenFlareAccessLogQuery{NodeID: nodeID, Since: since}
|
||||
clause, args := buildOpenFlareAccessLogFilterClause(filter)
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT trim(region) AS region, count() AS count
|
||||
FROM %s
|
||||
WHERE %s AND trim(region) != ''
|
||||
GROUP BY trim(region)
|
||||
ORDER BY count DESC, region ASC`, openFlareAccessLogTable, clause)
|
||||
if limit > 0 {
|
||||
sql += " LIMIT ?"
|
||||
args = append(args, limit)
|
||||
}
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
var result []*OpenFlareAccessLogRegionCount
|
||||
for rows.Next() {
|
||||
var item OpenFlareAccessLogRegionCount
|
||||
if err := rows.Scan(&item.Region, &item.Count); err != nil {
|
||||
return nil, err
|
||||
result := make([]*OpenFlareAccessLogRegionCount, len(rows))
|
||||
for index, row := range rows {
|
||||
result[index] = &OpenFlareAccessLogRegionCount{
|
||||
Region: row.Region,
|
||||
Count: row.Count,
|
||||
}
|
||||
result = append(result, &item)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (clickhouseAccessLogStore) BucketAggregates(ctx context.Context, filter OpenFlareAccessLogQuery, bucketSeconds int64) ([]openFlareAccessLogBucketAggregateRow, error) {
|
||||
conn, err := clickhouseAccessLogStore{}.conn()
|
||||
rows, err := analyticsrepo.BucketAggregatesNodeAccessLogs(ctx, toNodeAccessLogFilter(filter), bucketSeconds)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clause, args := buildOpenFlareAccessLogFilterClause(filter)
|
||||
bucketExpr := openFlareAccessLogBucketEpochExpr(bucketSeconds)
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT
|
||||
%s AS bucket_epoch,
|
||||
count() AS request_count,
|
||||
countIf(status_code < 400) AS success_count,
|
||||
countIf(status_code >= 400 AND status_code < 500) AS client_error_count,
|
||||
countIf(status_code >= 500) AS server_error_count
|
||||
FROM %s
|
||||
WHERE %s
|
||||
GROUP BY bucket_epoch`, bucketExpr, openFlareAccessLogTable, clause)
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
var result []openFlareAccessLogBucketAggregateRow
|
||||
for rows.Next() {
|
||||
var item openFlareAccessLogBucketAggregateRow
|
||||
if err := rows.Scan(&item.BucketEpoch, &item.RequestCount, &item.SuccessCount, &item.ClientErrorCount, &item.ServerErrorCount); err != nil {
|
||||
return nil, err
|
||||
result := make([]openFlareAccessLogBucketAggregateRow, len(rows))
|
||||
for index, row := range rows {
|
||||
result[index] = openFlareAccessLogBucketAggregateRow{
|
||||
BucketEpoch: row.BucketEpoch,
|
||||
RequestCount: row.RequestCount,
|
||||
SuccessCount: row.SuccessCount,
|
||||
ClientErrorCount: row.ClientErrorCount,
|
||||
ServerErrorCount: row.ServerErrorCount,
|
||||
}
|
||||
result = append(result, item)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (clickhouseAccessLogStore) BucketDimensions(ctx context.Context, filter OpenFlareAccessLogQuery, column string, bucketSeconds int64) ([]openFlareAccessLogBucketDimensionRow, error) {
|
||||
conn, err := clickhouseAccessLogStore{}.conn()
|
||||
rows, err := analyticsrepo.BucketDimensionsNodeAccessLogs(ctx, toNodeAccessLogFilter(filter), column, bucketSeconds)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clause, args := buildOpenFlareAccessLogFilterClause(filter)
|
||||
bucketExpr := openFlareAccessLogBucketEpochExpr(bucketSeconds)
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT
|
||||
%s AS bucket_epoch,
|
||||
trim(%s) AS value
|
||||
FROM %s
|
||||
WHERE %s AND trim(%s) != ''
|
||||
GROUP BY bucket_epoch, trim(%s)`, bucketExpr, column, openFlareAccessLogTable, clause, column, column)
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
var result []openFlareAccessLogBucketDimensionRow
|
||||
for rows.Next() {
|
||||
var item openFlareAccessLogBucketDimensionRow
|
||||
if err := rows.Scan(&item.BucketEpoch, &item.Value); err != nil {
|
||||
return nil, err
|
||||
result := make([]openFlareAccessLogBucketDimensionRow, len(rows))
|
||||
for index, row := range rows {
|
||||
result[index] = openFlareAccessLogBucketDimensionRow{
|
||||
BucketEpoch: row.BucketEpoch,
|
||||
Value: row.Value,
|
||||
}
|
||||
result = append(result, item)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (clickhouseAccessLogStore) IPAggregates(ctx context.Context, filter OpenFlareAccessLogQuery, exactRemoteAddr bool) ([]openFlareAccessLogIPAggregateRow, error) {
|
||||
conn, err := clickhouseAccessLogStore{}.conn()
|
||||
rows, err := analyticsrepo.IPAggregatesNodeAccessLogs(ctx, toNodeAccessLogFilter(filter), exactRemoteAddr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clause, args := buildOpenFlareAccessLogFilterClause(filter)
|
||||
queryClause := clause
|
||||
queryArgs := append([]any{}, args...)
|
||||
if exactRemoteAddr {
|
||||
trimmed := strings.TrimSpace(filter.RemoteAddr)
|
||||
if trimmed == "" {
|
||||
return []openFlareAccessLogIPAggregateRow{}, nil
|
||||
result := make([]openFlareAccessLogIPAggregateRow, len(rows))
|
||||
for index, row := range rows {
|
||||
result[index] = openFlareAccessLogIPAggregateRow{
|
||||
RemoteAddr: row.RemoteAddr,
|
||||
RequestCount: row.RequestCount,
|
||||
SuccessCount: row.SuccessCount,
|
||||
ClientErrorCount: row.ClientErrorCount,
|
||||
ServerErrorCount: row.ServerErrorCount,
|
||||
LastSeenEpoch: row.LastSeenEpoch,
|
||||
}
|
||||
queryClause = combineOpenFlareAccessLogSQLClauses(queryClause, "trim(remote_addr) = ?")
|
||||
queryArgs = append(queryArgs, trimmed)
|
||||
}
|
||||
lastSeenExpr := openFlareAccessLogEpochExpr()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT
|
||||
trim(remote_addr) AS remote_addr,
|
||||
count() AS request_count,
|
||||
countIf(status_code < 400) AS success_count,
|
||||
countIf(status_code >= 400 AND status_code < 500) AS client_error_count,
|
||||
countIf(status_code >= 500) AS server_error_count,
|
||||
max(%s) AS last_seen_epoch
|
||||
FROM %s
|
||||
WHERE %s AND trim(remote_addr) != ''
|
||||
GROUP BY trim(remote_addr)`, lastSeenExpr, openFlareAccessLogTable, queryClause)
|
||||
rows, err := conn.Query(ctx, sql, queryArgs...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
var result []openFlareAccessLogIPAggregateRow
|
||||
for rows.Next() {
|
||||
var item openFlareAccessLogIPAggregateRow
|
||||
if err := rows.Scan(&item.RemoteAddr, &item.RequestCount, &item.SuccessCount, &item.ClientErrorCount, &item.ServerErrorCount, &item.LastSeenEpoch); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, item)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (clickhouseAccessLogStore) IPSummaries(ctx context.Context, filter OpenFlareAccessLogQuery, recentSince time.Time) ([]openFlareAccessLogIPSummaryRow, error) {
|
||||
conn, err := clickhouseAccessLogStore{}.conn()
|
||||
rows, err := analyticsrepo.IPSummariesNodeAccessLogs(ctx, toNodeAccessLogFilter(filter), recentSince)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clause, args := buildOpenFlareAccessLogFilterClause(filter)
|
||||
lastSeenExpr := openFlareAccessLogEpochExpr()
|
||||
recentClause := "0"
|
||||
queryArgs := make([]any, 0, len(args)+1)
|
||||
if !recentSince.IsZero() {
|
||||
recentClause = "if(logged_at >= ?, 1, 0)"
|
||||
queryArgs = append(queryArgs, recentSince)
|
||||
}
|
||||
queryArgs = append(queryArgs, args...)
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT
|
||||
trim(remote_addr) AS remote_addr,
|
||||
count() AS total_requests,
|
||||
sum(%s) AS recent_requests,
|
||||
max(%s) AS last_seen_epoch
|
||||
FROM %s
|
||||
WHERE %s AND trim(remote_addr) != ''
|
||||
GROUP BY trim(remote_addr)`, recentClause, lastSeenExpr, openFlareAccessLogTable, clause)
|
||||
rows, err := conn.Query(ctx, sql, queryArgs...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
var result []openFlareAccessLogIPSummaryRow
|
||||
for rows.Next() {
|
||||
var item openFlareAccessLogIPSummaryRow
|
||||
if err := rows.Scan(&item.RemoteAddr, &item.TotalRequests, &item.RecentRequests, &item.LastSeenEpoch); err != nil {
|
||||
return nil, err
|
||||
result := make([]openFlareAccessLogIPSummaryRow, len(rows))
|
||||
for index, row := range rows {
|
||||
result[index] = openFlareAccessLogIPSummaryRow{
|
||||
RemoteAddr: row.RemoteAddr,
|
||||
TotalRequests: row.TotalRequests,
|
||||
RecentRequests: row.RecentRequests,
|
||||
LastSeenEpoch: row.LastSeenEpoch,
|
||||
}
|
||||
result = append(result, item)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (clickhouseAccessLogStore) IPTrend(ctx context.Context, filter OpenFlareAccessLogQuery, bucketSeconds int64) ([]openFlareAccessLogIPTrendRow, error) {
|
||||
conn, err := clickhouseAccessLogStore{}.conn()
|
||||
rows, err := analyticsrepo.IPTrendNodeAccessLogs(ctx, toNodeAccessLogFilter(filter), bucketSeconds)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clause, args := buildOpenFlareAccessLogFilterClause(filter)
|
||||
bucketExpr := openFlareAccessLogBucketEpochExpr(bucketSeconds)
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT
|
||||
%s AS bucket_epoch,
|
||||
count() AS request_count
|
||||
FROM %s
|
||||
WHERE %s
|
||||
GROUP BY bucket_epoch
|
||||
ORDER BY bucket_epoch ASC`, bucketExpr, openFlareAccessLogTable, clause)
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
var result []openFlareAccessLogIPTrendRow
|
||||
for rows.Next() {
|
||||
var item openFlareAccessLogIPTrendRow
|
||||
if err := rows.Scan(&item.BucketEpoch, &item.RequestCount); err != nil {
|
||||
return nil, err
|
||||
result := make([]openFlareAccessLogIPTrendRow, len(rows))
|
||||
for index, row := range rows {
|
||||
result[index] = openFlareAccessLogIPTrendRow{
|
||||
BucketEpoch: row.BucketEpoch,
|
||||
RequestCount: row.RequestCount,
|
||||
}
|
||||
result = append(result, item)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s clickhouseAccessLogStore) DeleteAll(ctx context.Context) (int64, error) {
|
||||
return s.deleteWithCount(ctx, "SELECT count() FROM "+openFlareAccessLogTable, nil, "ALTER TABLE "+openFlareAccessLogTable+" DELETE WHERE 1")
|
||||
func (clickhouseAccessLogStore) DeleteAll(ctx context.Context) (int64, error) {
|
||||
return analyticsrepo.DeleteAllNodeAccessLogs(ctx)
|
||||
}
|
||||
|
||||
func (s clickhouseAccessLogStore) DeleteBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
return s.deleteWithCount(
|
||||
ctx,
|
||||
fmt.Sprintf("SELECT count() FROM %s WHERE logged_at < ?", openFlareAccessLogTable),
|
||||
[]any{cutoff.UTC()},
|
||||
fmt.Sprintf("ALTER TABLE %s DELETE WHERE logged_at < ?", openFlareAccessLogTable),
|
||||
cutoff.UTC(),
|
||||
)
|
||||
func (clickhouseAccessLogStore) DeleteBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
return analyticsrepo.DeleteNodeAccessLogsBefore(ctx, cutoff)
|
||||
}
|
||||
|
||||
func (s clickhouseAccessLogStore) DeleteByNodeBefore(ctx context.Context, nodeID string, before time.Time) (int64, error) {
|
||||
return s.deleteWithCount(
|
||||
ctx,
|
||||
fmt.Sprintf("SELECT count() FROM %s WHERE node_id = ? AND logged_at < ?", openFlareAccessLogTable),
|
||||
[]any{nodeID, before.UTC()},
|
||||
fmt.Sprintf("ALTER TABLE %s DELETE WHERE node_id = ? AND logged_at < ?", openFlareAccessLogTable),
|
||||
nodeID, before.UTC(),
|
||||
)
|
||||
func (clickhouseAccessLogStore) DeleteByNodeBefore(ctx context.Context, nodeID string, before time.Time) (int64, error) {
|
||||
return analyticsrepo.DeleteNodeAccessLogsByNodeBefore(ctx, nodeID, before)
|
||||
}
|
||||
|
||||
func (clickhouseAccessLogStore) deleteWithCount(ctx context.Context, countSQL string, countArgs []any, deleteSQL string, deleteArgs ...any) (int64, error) {
|
||||
conn, err := clickhouseAccessLogStore{}.conn()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
func toNodeAccessLogFilter(query OpenFlareAccessLogQuery) analyticsrepo.NodeAccessLogFilter {
|
||||
return analyticsrepo.NodeAccessLogFilter{
|
||||
NodeID: query.NodeID,
|
||||
RemoteAddr: query.RemoteAddr,
|
||||
Host: query.Host,
|
||||
Path: query.Path,
|
||||
Since: query.Since,
|
||||
Until: query.Until,
|
||||
Page: query.Page,
|
||||
PageSize: query.PageSize,
|
||||
SortBy: query.SortBy,
|
||||
SortOrder: query.SortOrder,
|
||||
}
|
||||
var count int64
|
||||
if err := conn.QueryRow(ctx, countSQL, countArgs...).Scan(&count); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if count == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
if err := conn.Exec(ctx, deleteSQL, deleteArgs...); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
|
||||
func openFlareAccessLogBucketEpochExpr(bucketSeconds int64) string {
|
||||
return fmt.Sprintf("toInt64(intDiv(toUnixTimestamp(logged_at), %d) * %d)", bucketSeconds, bucketSeconds)
|
||||
func toAnalyticsNodeAccessLog(record *OpenFlareAccessLog) analyticsmodel.NodeAccessLog {
|
||||
return analyticsmodel.NodeAccessLog{
|
||||
ID: uint64(record.ID),
|
||||
NodeID: record.NodeID,
|
||||
LoggedAt: record.LoggedAt,
|
||||
RemoteAddr: record.RemoteAddr,
|
||||
Region: record.Region,
|
||||
Host: record.Host,
|
||||
Path: record.Path,
|
||||
StatusCode: int32(record.StatusCode),
|
||||
CreatedAt: record.CreatedAt,
|
||||
}
|
||||
}
|
||||
|
||||
func openFlareAccessLogEpochExpr() string {
|
||||
return "toInt64(toUnixTimestamp(logged_at))"
|
||||
func fromAnalyticsNodeAccessLogs(rows []analyticsmodel.NodeAccessLog) []*OpenFlareAccessLog {
|
||||
result := make([]*OpenFlareAccessLog, len(rows))
|
||||
for index, row := range rows {
|
||||
result[index] = &OpenFlareAccessLog{
|
||||
ID: uint(row.ID),
|
||||
NodeID: row.NodeID,
|
||||
LoggedAt: row.LoggedAt,
|
||||
RemoteAddr: row.RemoteAddr,
|
||||
Region: row.Region,
|
||||
Host: row.Host,
|
||||
Path: row.Path,
|
||||
StatusCode: int(row.StatusCode),
|
||||
CreatedAt: row.CreatedAt,
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -55,7 +55,8 @@ func (OpenFlareRequestReport) TableName() string {
|
||||
return "of_node_request_reports"
|
||||
}
|
||||
|
||||
// OpenFlareAccessLog stores a single access log row in ClickHouse (database: openflare).
|
||||
// OpenFlareAccessLog stores a single access log row in ClickHouse (database: openflare, table: of_node_access_logs).
|
||||
// ClickHouse DDL is managed by goose; reads/writes go through internal/repository/analytics.
|
||||
type OpenFlareAccessLog struct {
|
||||
ID uint `json:"id" gorm:"primaryKey;autoIncrement"`
|
||||
NodeID string `json:"node_id" gorm:"index;size:64;not null"`
|
||||
|
||||
Reference in New Issue
Block a user