tcjlj/OpenFlare
1
0
Fork 0
mirror of https://github.com/Rain-kl/OpenFlare.git synced 2026-10-08 08:36:37 +08:00
Code Issues Packages Projects Releases Wiki Activity

[功能] 实现节点访问日志的分片支持，优化日志查询和管理逻辑

Browse Source
This commit is contained in:
ryan
2026-03-19 16:19:46 +08:00
parent 891cb7b9c1
commit dd49b2777d
13 changed files with 709 additions and 152 deletions
Download Patch File Download Diff File Expand all files Collapse all files
openflare_server/service/access_log_test.go
+49 -14
View File
@@ -2,6 +2,7 @@ package service
import (
"openflare/model"
"strings"
"testing"
"time"
)
@@ -60,9 +61,7 @@ func TestListAccessLogsIncludesSummaryTotals(t *testing.T) {
StatusCode: 200,
},
}
if err := model.DB.Create(&logs).Error; err != nil {
t.Fatalf("failed to seed access logs: %v", err)
}
seedNodeAccessLogs(t, logs)
result, err := ListAccessLogs(AccessLogQuery{Page: 0, PageSize: 2})
if err != nil {
@@ -121,9 +120,7 @@ func TestListAccessLogsUsesDefaultPageSize(t *testing.T) {
StatusCode: 200,
})
}
if err := model.DB.Create(&logs).Error; err != nil {
t.Fatalf("failed to seed access logs: %v", err)
}
seedNodeAccessLogs(t, logs)
result, err := ListAccessLogs(AccessLogQuery{})
if err != nil {
@@ -143,7 +140,7 @@ func TestListAccessLogsUsesDefaultPageSize(t *testing.T) {
func TestListFoldedAccessLogsAndIPSummaries(t *testing.T) {
setupServiceTestDB(t)
now := time.Now().UTC()
now := time.Date(2026, 3, 19, 8, 12, 30, 0, time.UTC)
if err := model.DB.Create(&model.Node{
NodeID: "node-folded",
Name: "edge-folded",
@@ -176,9 +173,7 @@ func TestListFoldedAccessLogsAndIPSummaries(t *testing.T) {
StatusCode: 404,
},
}
if err := model.DB.Create(&logs).Error; err != nil {
t.Fatalf("failed to seed access logs: %v", err)
}
seedNodeAccessLogs(t, logs)
folded, err := ListFoldedAccessLogs(AccessLogQuery{
NodeID: "node-folded",
@@ -221,7 +216,7 @@ func TestCleanupAccessLogsDeletesExpiredData(t *testing.T) {
setupServiceTestDB(t)
now := time.Now().UTC()
if err := model.DB.Create([]*model.NodeAccessLog{
seedNodeAccessLogs(t, []*model.NodeAccessLog{
{
NodeID: "node-cleanup",
LoggedAt: now.Add(-10 * 24 * time.Hour),
@@ -238,9 +233,7 @@ func TestCleanupAccessLogsDeletesExpiredData(t *testing.T) {
Path: "/recent",
StatusCode: 200,
},
}).Error; err != nil {
t.Fatalf("failed to seed cleanup logs: %v", err)
}
})
result, err := CleanupAccessLogs(AccessLogCleanupInput{RetentionDays: 7})
if err != nil {
@@ -258,3 +251,45 @@ func TestCleanupAccessLogsDeletesExpiredData(t *testing.T) {
t.Fatalf("unexpected remaining logs after cleanup: %+v", remaining.Items)
}
}
func TestPersistNodeAccessLogsTruncatesLongPath(t *testing.T) {
setupServiceTestDB(t)
longPath := "/" + strings.Repeat("a", 140)
reportedAt := time.Now().UTC()
if err := persistNodeAccessLogs(model.DB, "node-truncate", []AgentNodeAccessLog{
{
LoggedAtUnix: reportedAt.Unix(),
RemoteAddr: "203.0.113.10",
Host: "truncate.example.com",
Path: longPath,
StatusCode: 200,
},
}, reportedAt); err != nil {
t.Fatalf("persistNodeAccessLogs failed: %v", err)
}
logs, err := model.ListNodeAccessLogs(model.NodeAccessLogQuery{
NodeID: "node-truncate",
Page: 0,
PageSize: 10,
})
if err != nil {
t.Fatalf("ListNodeAccessLogs failed: %v", err)
}
if len(logs) != 1 {
t.Fatalf("expected one stored log, got %+v", logs)
}
if got := len([]rune(logs[0].Path)); got != nodeAccessLogPathMaxLength {
t.Fatalf("expected truncated path length %d, got %d (%q)", nodeAccessLogPathMaxLength, got, logs[0].Path)
}
}
func seedNodeAccessLogs(t *testing.T, logs []*model.NodeAccessLog) {
t.Helper()
for _, item := range logs {
if err := model.DB.Create(item).Error; err != nil {
t.Fatalf("failed to seed access log: %v", err)
}
}
}
openflare_server/service/node_update_test.go
+10 -6
View File
@@ -908,7 +908,7 @@ func TestListAccessLogsUsesPagination(t *testing.T) {
}
now := time.Now().UTC()
if err := model.DB.Create([]*model.NodeAccessLog{
for _, item := range []*model.NodeAccessLog{
{
NodeID: node.NodeID,
LoggedAt: now.Add(-10 * time.Second),
@@ -936,8 +936,10 @@ func TestListAccessLogsUsesPagination(t *testing.T) {
Path: "/three",
StatusCode: 502,
},
}).Error; err != nil {
t.Fatalf("failed to seed access logs: %v", err)
} {
if err := model.DB.Create(item).Error; err != nil {
t.Fatalf("failed to seed access logs: %v", err)
}
}
pageOne, err := ListAccessLogs(AccessLogQuery{
@@ -1404,7 +1406,7 @@ func TestGetDashboardOverview(t *testing.T) {
}).Insert(); err != nil {
t.Fatalf("failed to insert node b traffic report: %v", err)
}
if err := model.DB.Create([]*model.NodeAccessLog{
for _, item := range []*model.NodeAccessLog{
{
NodeID: "node-dashboard-a",
LoggedAt: now.Add(-30 * time.Minute),
@@ -1432,8 +1434,10 @@ func TestGetDashboardOverview(t *testing.T) {
Path: "/edge",
StatusCode: 502,
},
}).Error; err != nil {
t.Fatalf("failed to seed dashboard access logs: %v", err)
} {
if err := model.DB.Create(item).Error; err != nil {
t.Fatalf("failed to seed dashboard access logs: %v", err)
}
}
if err := model.DB.Create(&model.NodeHealthEvent{
openflare_server/service/observability.go
+5 -4
View File
@@ -18,6 +18,7 @@ const (
NodeHealthSeverityWarning = "warning"
NodeHealthSeverityCritical = "critical"
nodeAccessLogRetentionWindow = nodeAccessLogRetentionDays * 24 * time.Hour
nodeAccessLogPathMaxLength = 100
)
type AgentNodeSystemProfile struct {
@@ -177,7 +178,7 @@ func persistNodeMetricSnapshot(tx *gorm.DB, nodeID string, snapshot *AgentNodeMe
OpenrestyConnections: snapshot.OpenrestyConnections,
RawJSON: marshalJSON(snapshot),
}
return tx.Where("node_id = ? AND captured_at = ?", nodeID, record.CapturedAt).Assign(record).FirstOrCreate(record).Error
return tx.Where("node_id = ? AND captured_at = ?", nodeID, record.CapturedAt).FirstOrCreate(record).Error
}
func persistNodeTrafficReport(tx *gorm.DB, nodeID string, report *AgentNodeTrafficReport, reportedAt time.Time) error {
@@ -199,7 +200,7 @@ func persistNodeTrafficReport(tx *gorm.DB, nodeID string, report *AgentNodeTraff
SourceCountriesJSON: marshalJSON(report.SourceCountries),
RawJSON: marshalJSON(report),
}
return tx.Where("node_id = ? AND window_started_at = ? AND window_ended_at = ?", nodeID, record.WindowStartedAt, record.WindowEndedAt).Assign(record).FirstOrCreate(record).Error
return tx.Where("node_id = ? AND window_started_at = ? AND window_ended_at = ?", nodeID, record.WindowStartedAt, record.WindowEndedAt).FirstOrCreate(record).Error
}
func persistNodeAccessLogs(tx *gorm.DB, nodeID string, logs []AgentNodeAccessLog, reportedAt time.Time) error {
@@ -220,7 +221,7 @@ func persistNodeAccessLogs(tx *gorm.DB, nodeID string, logs []AgentNodeAccessLog
RemoteAddr: strings.TrimSpace(item.RemoteAddr),
Region: "",
Host: strings.TrimSpace(item.Host),
Path: strings.TrimSpace(item.Path),
Path: truncateForDatabase(strings.TrimSpace(item.Path), nodeAccessLogPathMaxLength),
StatusCode: item.StatusCode,
RawJSON: marshalJSON(item),
}
@@ -235,7 +236,7 @@ func persistNodeAccessLogs(tx *gorm.DB, nodeID string, logs []AgentNodeAccessLog
record.Host,
record.Path,
record.StatusCode,
).Assign(record).FirstOrCreate(record).Error; err != nil {
).FirstOrCreate(record).Error; err != nil {
return err
}
}