tcjlj/OpenFlare
1
0
Fork 0
mirror of https://github.com/Rain-kl/OpenFlare.git synced 2026-10-01 06:36:38 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(clickhouse): harden R/W path P0–P3 (cleanup, durability, rollups)

Browse Source 
Honest TTL cleanup semantics; enqueue-safe dedup with flush retry and writer
metrics; model insert hooks; latest-per-node and hourly metric/openresty
rollups; small-host pool/async defaults, traffic hourly TTL, and UV labeling.
This commit is contained in:
ryan
2026-07-10 10:34:04 +08:00
parent 9b3555c569
commit 160e63558f
37 changed files with 1779 additions and 269 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/model/openflare_access_log_store.go
+27 -2
View File
@@ -8,11 +8,34 @@ import (
"sync"
"time"
"github.com/Rain-kl/Wavelet/internal/apps/openflare/chwriter"
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
analyticsrepo "github.com/Rain-kl/Wavelet/internal/repository/analytics"
)
// AccessLogInsertHooks queues node access logs for async ClickHouse write.
// Wired from openflare/chwriter.Init so model never imports the apps layer.
type AccessLogInsertHooks struct {
QueueNodeAccessLogs func(logs []analyticsmodel.NodeAccessLog)
}
var (
accessLogInsertHooksMu sync.RWMutex
accessLogInsertHooks AccessLogInsertHooks
)
// SetAccessLogInsertHooks registers async queue callbacks for access log inserts.
func SetAccessLogInsertHooks(hooks AccessLogInsertHooks) {
accessLogInsertHooksMu.Lock()
accessLogInsertHooks = hooks
accessLogInsertHooksMu.Unlock()
}
func currentAccessLogInsertHooks() AccessLogInsertHooks {
accessLogInsertHooksMu.RLock()
defer accessLogInsertHooksMu.RUnlock()
return accessLogInsertHooks
}
type accessLogStore interface {
InsertBatch(ctx context.Context, records []*OpenFlareAccessLog) error
List(ctx context.Context, query OpenFlareAccessLogQuery) ([]*OpenFlareAccessLog, error)
@@ -75,7 +98,9 @@ func (clickhouseAccessLogStore) InsertBatch(_ context.Context, records []*OpenFl
}
logs = append(logs, toAnalyticsNodeAccessLog(record))
}
chwriter.QueueNodeAccessLogs(logs)
if hook := currentAccessLogInsertHooks().QueueNodeAccessLogs; hook != nil {
hook(logs)
}
return nil
}
internal/model/openflare_insert_hooks_test.go
+75
View File
@@ -0,0 +1,75 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package model
import (
"context"
"testing"
"time"
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
)
// Hook setters are process-global; keep these tests serial.
func TestObservabilityInsertHooksAreInvoked(t *testing.T) {
var gotSnapshot analyticsmodel.NodeMetricSnapshot
SetObservabilityInsertHooks(ObservabilityInsertHooks{
QueueMetricSnapshot: func(s analyticsmodel.NodeMetricSnapshot) {
gotSnapshot = s
},
})
t.Cleanup(func() {
SetObservabilityInsertHooks(ObservabilityInsertHooks{})
})
record := &OpenFlareMetricSnapshot{
NodeID: "node-1",
CapturedAt: time.Unix(100, 0).UTC(),
}
if err := (clickhouseObservabilityStore{}).InsertMetricSnapshot(context.Background(), record); err != nil {
t.Fatalf("InsertMetricSnapshot error = %v", err)
}
if gotSnapshot.NodeID != "node-1" {
t.Fatalf("hook node id = %q, want node-1", gotSnapshot.NodeID)
}
}
func TestAccessLogInsertHooksAreInvoked(t *testing.T) {
var got []analyticsmodel.NodeAccessLog
SetAccessLogInsertHooks(AccessLogInsertHooks{
QueueNodeAccessLogs: func(logs []analyticsmodel.NodeAccessLog) {
got = append([]analyticsmodel.NodeAccessLog(nil), logs...)
},
})
t.Cleanup(func() {
SetAccessLogInsertHooks(AccessLogInsertHooks{})
})
records := []*OpenFlareAccessLog{
{NodeID: "n1", Path: "/a"},
{NodeID: "n1", Path: "/b"},
}
if err := (clickhouseAccessLogStore{}).InsertBatch(context.Background(), records); err != nil {
t.Fatalf("InsertBatch error = %v", err)
}
if len(got) != 2 {
t.Fatalf("hook logs = %d, want 2", len(got))
}
if got[0].Path != "/a" || got[1].Path != "/b" {
t.Fatalf("hook paths = %q/%q, want /a /b", got[0].Path, got[1].Path)
}
}
func TestInsertHooksNoopWhenUnset(t *testing.T) {
SetObservabilityInsertHooks(ObservabilityInsertHooks{})
SetAccessLogInsertHooks(AccessLogInsertHooks{})
if err := (clickhouseObservabilityStore{}).InsertMetricSnapshot(context.Background(), &OpenFlareMetricSnapshot{NodeID: "x"}); err != nil {
t.Fatalf("InsertMetricSnapshot with nil hook error = %v", err)
}
if err := (clickhouseAccessLogStore{}).InsertBatch(context.Background(), []*OpenFlareAccessLog{{NodeID: "x"}}); err != nil {
t.Fatalf("InsertBatch with nil hook error = %v", err)
}
}
internal/model/openflare_observability.go
+71
View File
@@ -333,11 +333,82 @@ func ListOpenFlareMetricSnapshotsSince(ctx context.Context, nodeID string, since
return currentObservabilityStore().ListMetricSnapshots(ctx, nodeID, since, limit)
}
// ListOpenFlareLatestMetricSnapshotsSince returns the latest metric snapshot per node.
// Prefer ClickHouse LIMIT 1 BY; on CH unavailability fall back to store list + reduce.
func ListOpenFlareLatestMetricSnapshotsSince(ctx context.Context, nodeID string, since time.Time) ([]*OpenFlareMetricSnapshot, error) {
rows, err := analyticsrepo.ListLatestNodeMetricSnapshots(ctx, analyticsrepo.NodeObservabilityFilter{
NodeID: nodeID,
Since: since,
})
if err == nil {
return fromAnalyticsNodeMetricSnapshots(rows), nil
}
// Fallback for unit tests (memory store) and environments without ClickHouse.
all, listErr := ListOpenFlareMetricSnapshotsSince(ctx, nodeID, since, 0)
if listErr != nil {
return nil, err
}
return openFlareLatestMetricSnapshots(all), nil
}
// ListOpenFlareRequestReportsSince returns request reports since the given time.
func ListOpenFlareRequestReportsSince(ctx context.Context, nodeID string, since time.Time, limit int) ([]*OpenFlareRequestReport, error) {
return currentObservabilityStore().ListRequestReports(ctx, nodeID, since, limit)
}
// ListOpenFlareLatestRequestReportsSince returns the latest request report per node.
// Prefer ClickHouse LIMIT 1 BY; on CH unavailability fall back to store list + reduce.
func ListOpenFlareLatestRequestReportsSince(ctx context.Context, nodeID string, since time.Time) ([]*OpenFlareRequestReport, error) {
rows, err := analyticsrepo.ListLatestNodeRequestReports(ctx, analyticsrepo.NodeObservabilityFilter{
NodeID: nodeID,
Since: since,
})
if err == nil {
return fromAnalyticsNodeRequestReports(rows), nil
}
all, listErr := ListOpenFlareRequestReportsSince(ctx, nodeID, since, 0)
if listErr != nil {
return nil, err
}
return openFlareLatestRequestReports(all), nil
}
func openFlareLatestMetricSnapshots(snapshots []*OpenFlareMetricSnapshot) []*OpenFlareMetricSnapshot {
latestByNode := make(map[string]*OpenFlareMetricSnapshot, len(snapshots))
for _, snapshot := range snapshots {
if snapshot == nil || snapshot.NodeID == "" {
continue
}
if existing, ok := latestByNode[snapshot.NodeID]; ok && !snapshot.CapturedAt.After(existing.CapturedAt) {
continue
}
latestByNode[snapshot.NodeID] = snapshot
}
result := make([]*OpenFlareMetricSnapshot, 0, len(latestByNode))
for _, snapshot := range latestByNode {
result = append(result, snapshot)
}
return result
}
func openFlareLatestRequestReports(reports []*OpenFlareRequestReport) []*OpenFlareRequestReport {
latestByNode := make(map[string]*OpenFlareRequestReport, len(reports))
for _, report := range reports {
if report == nil || report.NodeID == "" {
continue
}
if existing, ok := latestByNode[report.NodeID]; ok && !report.WindowEndedAt.After(existing.WindowEndedAt) {
continue
}
latestByNode[report.NodeID] = report
}
result := make([]*OpenFlareRequestReport, 0, len(latestByNode))
for _, report := range latestByNode {
result = append(result, report)
}
return result
}
// OpenFlareTrafficHourly is an hourly traffic rollup row.
type OpenFlareTrafficHourly struct {
NodeID string `json:"node_id"`
internal/model/openflare_observability_store.go
+43 -6
View File
@@ -9,11 +9,38 @@ import (
"sync"
"time"
"github.com/Rain-kl/Wavelet/internal/apps/openflare/chwriter"
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
analyticsrepo "github.com/Rain-kl/Wavelet/internal/repository/analytics"
)
// ObservabilityInsertHooks queues observability rows for async ClickHouse write.
// Wired from openflare/chwriter.Init so model never imports the apps layer.
type ObservabilityInsertHooks struct {
QueueMetricSnapshot func(analyticsmodel.NodeMetricSnapshot)
QueueRequestReport func(analyticsmodel.NodeRequestReport)
QueueOpenrestyObservation func(analyticsmodel.NodeObsOpenresty)
QueueFrpsObservation func(analyticsmodel.NodeObsFrps)
QueueFrpcObservation func(analyticsmodel.NodeObsFrpc)
}
var (
observabilityInsertHooksMu sync.RWMutex
observabilityInsertHooks ObservabilityInsertHooks
)
// SetObservabilityInsertHooks registers async queue callbacks for observability inserts.
func SetObservabilityInsertHooks(hooks ObservabilityInsertHooks) {
observabilityInsertHooksMu.Lock()
observabilityInsertHooks = hooks
observabilityInsertHooksMu.Unlock()
}
func currentObservabilityInsertHooks() ObservabilityInsertHooks {
observabilityInsertHooksMu.RLock()
defer observabilityInsertHooksMu.RUnlock()
return observabilityInsertHooks
}
type observabilityStore interface {
InsertMetricSnapshot(ctx context.Context, record *OpenFlareMetricSnapshot) error
ListMetricSnapshots(ctx context.Context, nodeID string, since time.Time, limit int) ([]*OpenFlareMetricSnapshot, error)
@@ -79,7 +106,9 @@ func (clickhouseObservabilityStore) InsertMetricSnapshot(_ context.Context, reco
if record == nil {
return nil
}
chwriter.QueueMetricSnapshot(toAnalyticsNodeMetricSnapshot(record))
if hook := currentObservabilityInsertHooks().QueueMetricSnapshot; hook != nil {
hook(toAnalyticsNodeMetricSnapshot(record))
}
return nil
}
@@ -103,7 +132,9 @@ func (clickhouseObservabilityStore) InsertRequestReport(_ context.Context, recor
if record == nil {
return nil
}
chwriter.QueueRequestReport(toAnalyticsNodeRequestReport(record))
if hook := currentObservabilityInsertHooks().QueueRequestReport; hook != nil {
hook(toAnalyticsNodeRequestReport(record))
}
return nil
}
@@ -127,7 +158,9 @@ func (clickhouseObservabilityStore) InsertNodeObservationOpenresty(_ context.Con
if record == nil {
return nil
}
chwriter.QueueOpenrestyObservation(toAnalyticsNodeObsOpenresty(record))
if hook := currentObservabilityInsertHooks().QueueOpenrestyObservation; hook != nil {
hook(toAnalyticsNodeObsOpenresty(record))
}
return nil
}
@@ -151,7 +184,9 @@ func (clickhouseObservabilityStore) InsertNodeObservationFrps(_ context.Context,
if record == nil {
return nil
}
chwriter.QueueFrpsObservation(toAnalyticsNodeObsFrps(record))
if hook := currentObservabilityInsertHooks().QueueFrpsObservation; hook != nil {
hook(toAnalyticsNodeObsFrps(record))
}
return nil
}
@@ -175,7 +210,9 @@ func (clickhouseObservabilityStore) InsertNodeObservationFrpc(_ context.Context,
if record == nil {
return nil
}
chwriter.QueueFrpcObservation(toAnalyticsNodeObsFrpc(record))
if hook := currentObservabilityInsertHooks().QueueFrpcObservation; hook != nil {
hook(toAnalyticsNodeObsFrpc(record))
}
return nil
}