tcjlj/OpenFlare
1
0
Fork 0
mirror of https://github.com/Rain-kl/OpenFlare.git synced 2026-10-04 23:16:37 +08:00
Code Issues Packages Projects Releases Wiki Activity

perf(clickhouse): Phase 2 legacy governance — TTL cleanup, unified pool, MV, ops API

Browse Source 
- Replace retention ALTER DELETE with MATERIALIZE TTL; use TRUNCATE for delete-all
- Remove GORM ClickHouse pool; migrate user access log reads to ChConn
- Drop query-side trim(remote_addr); enable wait_for_async_insert=1
- Add of_node_traffic_hourly MV and dashboard traffic trend fallback
- Add GET /admin/status/clickhouse operational metrics endpoint
This commit is contained in:
ryan
2026-07-02 15:47:38 +08:00
parent 38946d1af5
commit 58624db397
30 changed files with 1351 additions and 436 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/apps/openflare/dashboard/logics.go
+5 -1
View File
@@ -137,13 +137,17 @@ func buildOverviewView(ctx context.Context) (*OverviewView, error) {
if err != nil {
return nil, err
}
trafficTrend := observability.BuildTrafficTrendPoints(now, reports)
if trafficHourly, hourlyErr := model.ListOpenFlareTrafficHourlySince(ctx, "", since); hourlyErr == nil && len(trafficHourly) > 0 {
trafficTrend = observability.BuildTrafficTrendPointsFromHourly(now, trafficHourly)
}
view := &OverviewView{
GeneratedAt: now,
Nodes: make([]NodeHealth, 0, len(nodes)),
Distributions: observability.BuildTrafficDistributions(reports, accessLogRegions, dashboardDistributionLimit),
Trends: observability.NodeTrends{
Traffic24h: observability.BuildTrafficTrendPoints(now, reports),
Traffic24h: trafficTrend,
Capacity24h: observability.BuildCapacityTrendPoints(now, snapshots),
Network24h: observability.BuildNetworkTrendPoints(now, snapshots, openrestySnapshots),
DiskIO24h: observability.BuildDiskIOTrendPoints(now, snapshots),
internal/apps/openflare/observability/analytics.go
+22
View File
@@ -257,6 +257,28 @@ func buildHealthSummary(
return summary
}
// BuildTrafficTrendPointsFromHourly builds 24h traffic trend buckets from hourly rollups.
func BuildTrafficTrendPointsFromHourly(now time.Time, hourly []*model.OpenFlareTrafficHourly) []TrafficTrendPoint {
start := trendWindowStart(now)
points := make([]TrafficTrendPoint, observabilityTrendBuckets)
for index := range points {
points[index].BucketStartedAt = start.Add(time.Duration(index) * time.Hour)
}
for _, row := range hourly {
if row == nil {
continue
}
index, ok := trendBucketIndex(row.Hour, start)
if !ok {
continue
}
points[index].RequestCount += row.RequestCount
points[index].ErrorCount += row.ErrorCount
points[index].UniqueVisitorCount += row.UniqueVisitorCount
}
return points
}
// BuildTrafficTrendPoints builds 24h traffic trend buckets.
func BuildTrafficTrendPoints(now time.Time, reports []*model.OpenFlareRequestReport) []TrafficTrendPoint {
start := trendWindowStart(now)
internal/apps/openflare/observability/analytics_test.go
+17
View File
@@ -10,6 +10,23 @@ import (
"github.com/Rain-kl/Wavelet/internal/model"
)
func TestBuildTrafficTrendPointsFromHourlyBucketsByHour(t *testing.T) {
now := time.Date(2026, 7, 2, 15, 30, 0, 0, time.UTC)
hourly := []*model.OpenFlareTrafficHourly{
{
NodeID: "node-a",
Hour: now.Add(-2 * time.Hour).Truncate(time.Hour),
RequestCount: 12,
ErrorCount: 1,
UniqueVisitorCount: 4,
},
}
points := BuildTrafficTrendPointsFromHourly(now, hourly)
if len(points) != observabilityTrendBuckets {
t.Fatalf("BuildTrafficTrendPointsFromHourly() len = %d, want %d", len(points), observabilityTrendBuckets)
}
}
func TestBuildTrafficTrendPointsBucketsByHour(t *testing.T) {
t.Parallel()
internal/apps/openflare/observability/node_logics.go
+5 -1
View File
@@ -134,6 +134,10 @@ func GetNodeObservability(ctx context.Context, id uint, query NodeQuery) (*NodeV
if err != nil {
return nil, err
}
trafficTrend := BuildTrafficTrendPoints(now, reports)
if trafficHourly, hourlyErr := model.ListOpenFlareTrafficHourlySince(ctx, node.NodeID, now.Add(-24*time.Hour)); hourlyErr == nil && len(trafficHourly) > 0 {
trafficTrend = BuildTrafficTrendPointsFromHourly(now, trafficHourly)
}
view := &NodeView{
NodeID: node.NodeID,
@@ -147,7 +151,7 @@ func GetNodeObservability(ctx context.Context, id uint, query NodeQuery) (*NodeV
Health: buildHealthSummary(latestMetricSnapshot(snapshots), latestTrafficReport(reports), events),
},
Trends: NodeTrends{
Traffic24h: BuildTrafficTrendPoints(now, reports),
Traffic24h: trafficTrend,
Capacity24h: BuildCapacityTrendPoints(now, snapshots),
Network24h: BuildNetworkTrendPoints(now, snapshots, openrestyObs),
DiskIO24h: BuildDiskIOTrendPoints(now, snapshots),
internal/apps/openflare/tasks/database_cleanup.go
+38 -18
View File
@@ -12,6 +12,7 @@ import (
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/repository"
analyticsrepo "github.com/Rain-kl/Wavelet/internal/repository/analytics"
)
const (
@@ -49,6 +50,7 @@ type DatabaseCleanupResult struct {
Target string `json:"target"`
TargetLabel string `json:"target_label"`
DeletedCount int64 `json:"deleted_count"`
CleanupMode string `json:"cleanup_mode,omitempty"`
DeleteAll bool `json:"delete_all"`
RetentionDays *int `json:"retention_days,omitempty"`
Cutoff *time.Time `json:"cutoff,omitempty"`
@@ -79,21 +81,23 @@ func CleanupDatabaseObservability(ctx context.Context, input DatabaseCleanupInpu
}
if input.RetentionDays == nil {
deleted, err := deleteAllObservabilityRows(ctx, target)
deleted, mode, err := deleteAllObservabilityRows(ctx, target)
if err != nil {
return nil, err
}
result.DeletedCount = deleted
result.CleanupMode = mode
return result, nil
}
retentionDays := *input.RetentionDays
cutoff := time.Now().UTC().Add(-time.Duration(retentionDays) * 24 * time.Hour)
deleted, err := deleteObservabilityRowsBefore(ctx, target, cutoff)
deleted, mode, err := deleteObservabilityRowsBefore(ctx, target, cutoff)
if err != nil {
return nil, err
}
result.DeletedCount = deleted
result.CleanupMode = mode
result.RetentionDays = &retentionDays
result.Cutoff = &cutoff
return result, nil
@@ -141,40 +145,56 @@ func RunDatabaseAutoCleanupOnce(ctx context.Context, now time.Time) (*DatabaseAu
}, nil
}
func deleteAllObservabilityRows(ctx context.Context, target string) (int64, error) {
func deleteAllObservabilityRows(ctx context.Context, target string) (int64, string, error) {
var (
deleted int64
err error
)
switch target {
case DatabaseCleanupTargetAccessLogs:
return model.DeleteAllOpenFlareAccessLogs(ctx)
deleted, err = model.DeleteAllOpenFlareAccessLogs(ctx)
case DatabaseCleanupTargetMetricSnapshots:
return model.DeleteAllOpenFlareMetricSnapshots(ctx)
deleted, err = model.DeleteAllOpenFlareMetricSnapshots(ctx)
case DatabaseCleanupTargetRequestReports:
return model.DeleteAllOpenFlareRequestReports(ctx)
deleted, err = model.DeleteAllOpenFlareRequestReports(ctx)
case DatabaseCleanupTargetObsOpenresty:
return model.DeleteAllOpenFlareNodeObservationOpenresty(ctx)
deleted, err = model.DeleteAllOpenFlareNodeObservationOpenresty(ctx)
case DatabaseCleanupTargetObsFrps:
return model.DeleteAllOpenFlareNodeObservationFrps(ctx)
deleted, err = model.DeleteAllOpenFlareNodeObservationFrps(ctx)
case DatabaseCleanupTargetObsFrpc:
return model.DeleteAllOpenFlareNodeObservationFrpc(ctx)
deleted, err = model.DeleteAllOpenFlareNodeObservationFrpc(ctx)
default:
return 0, errors.New("unsupported cleanup target")
return 0, "", errors.New("unsupported cleanup target")
}
if err != nil {
return 0, "", err
}
return deleted, analyticsrepo.CleanupModeTruncate, nil
}
func deleteObservabilityRowsBefore(ctx context.Context, target string, cutoff time.Time) (int64, error) {
func deleteObservabilityRowsBefore(ctx context.Context, target string, cutoff time.Time) (int64, string, error) {
var (
deleted int64
err error
)
switch target {
case DatabaseCleanupTargetAccessLogs:
return model.DeleteOpenFlareAccessLogsBefore(ctx, cutoff)
deleted, err = model.DeleteOpenFlareAccessLogsBefore(ctx, cutoff)
case DatabaseCleanupTargetMetricSnapshots:
return model.DeleteOpenFlareMetricSnapshotsBefore(ctx, cutoff)
deleted, err = model.DeleteOpenFlareMetricSnapshotsBefore(ctx, cutoff)
case DatabaseCleanupTargetRequestReports:
return model.DeleteOpenFlareRequestReportsBefore(ctx, cutoff)
deleted, err = model.DeleteOpenFlareRequestReportsBefore(ctx, cutoff)
case DatabaseCleanupTargetObsOpenresty:
return model.DeleteOpenFlareNodeObservationOpenrestyBefore(ctx, cutoff)
deleted, err = model.DeleteOpenFlareNodeObservationOpenrestyBefore(ctx, cutoff)
case DatabaseCleanupTargetObsFrps:
return model.DeleteOpenFlareNodeObservationFrpsBefore(ctx, cutoff)
deleted, err = model.DeleteOpenFlareNodeObservationFrpsBefore(ctx, cutoff)
case DatabaseCleanupTargetObsFrpc:
return model.DeleteOpenFlareNodeObservationFrpcBefore(ctx, cutoff)
deleted, err = model.DeleteOpenFlareNodeObservationFrpcBefore(ctx, cutoff)
default:
return 0, errors.New("unsupported cleanup target")
return 0, "", errors.New("unsupported cleanup target")
}
if err != nil {
return 0, "", err
}
return deleted, analyticsrepo.CleanupModeTTLMaterialize, nil
}