tcjlj/OpenFlare
1
0
Fork 0
mirror of https://github.com/Rain-kl/OpenFlare.git synced 2026-10-09 00:56:37 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: add access log functionality and related components

Browse Source 
- Introduced NodeAccessLog model and corresponding database migrations.
- Implemented access log retrieval in the service layer.
- Created API endpoint for accessing logs with appropriate security measures.
- Developed frontend components for displaying access logs, including filtering and summary statistics.
- Updated observability buffer to include access logs and ensure proper merging and retention.
- Enhanced traffic report and observability tests to validate new access log features.
- Updated documentation to reflect changes in access log handling and data retention policies.
This commit is contained in:
ryan
2026-03-14 18:21:48 +08:00
parent e25b41fd75
commit b7d38590ba
25 changed files with 669 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
atsf_agent/internal/observability/traffic.go
+61 -8
View File
@@ -21,10 +21,11 @@ type accessLogRecord struct {
Timestamp string `json:"ts"`
Host string `json:"host"`
RemoteAddr string `json:"remote_addr"`
Path string `json:"path"`
Status int `json:"status"`
}
var combinedAccessLogPattern = regexp.MustCompile(`^(\S+)\s+\S+\s+\S+\s+\[([^\]]+)\]\s+"[^"]*"\s+(\d{3})\s+\S+`)
var combinedAccessLogPattern = regexp.MustCompile(`^(\S+)\s+\S+\s+\S+\s+\[([^\]]+)\]\s+"(?:\S+)\s+(\S+)(?:\s+[^"]*)?"\s+(\d{3})\s+\S+`)
type trafficAggregate struct {
windowStartedAt time.Time
@@ -34,16 +35,37 @@ type trafficAggregate struct {
statusCodes map[string]int64
topDomains map[string]int64
visitors map[string]struct{}
logs []protocol.NodeAccessLog
}
func BuildTrafficReport(cfg *config.Config, stateStore *state.Store, managed *managedOpenRestyMetrics) *protocol.NodeTrafficReport {
if managed != nil && managed.TrafficReport != nil {
return managed.TrafficReport
}
report, _ := BuildTrafficObservability(cfg, stateStore, managed)
return report
}
func BuildTrafficObservability(cfg *config.Config, stateStore *state.Store, managed *managedOpenRestyMetrics) (*protocol.NodeTrafficReport, []protocol.NodeAccessLog) {
if cfg == nil || stateStore == nil {
return nil
if managed != nil && managed.TrafficReport != nil {
return managed.TrafficReport, nil
}
return nil, nil
}
aggregate := readAccessLogDelta(cfg, stateStore)
accessLogs := []protocol.NodeAccessLog{}
if aggregate != nil {
accessLogs = aggregate.accessLogs()
}
if managed != nil && managed.TrafficReport != nil {
return managed.TrafficReport, accessLogs
}
if aggregate == nil {
return nil, accessLogs
}
return aggregate.report(), accessLogs
}
func readAccessLogDelta(cfg *config.Config, stateStore *state.Store) *trafficAggregate {
snapshot, err := stateStore.Load()
if err != nil {
return nil
@@ -97,7 +119,7 @@ func BuildTrafficReport(cfg *config.Config, stateStore *state.Store, managed *ma
snapshot.AccessLogOffset = currentOffset
_ = stateStore.Save(snapshot)
return aggregate.report()
return aggregate
}
func managedAccessLogPath(cfg *config.Config) string {
@@ -146,12 +168,20 @@ func (aggregate *trafficAggregate) consume(line []byte) {
if remoteAddr := strings.TrimSpace(record.RemoteAddr); remoteAddr != "" {
aggregate.visitors[remoteAddr] = struct{}{}
}
aggregate.logs = append(aggregate.logs, protocol.NodeAccessLog{
LoggedAtUnix: record.Timestamp.Unix(),
RemoteAddr: strings.TrimSpace(record.RemoteAddr),
Host: strings.TrimSpace(record.Host),
Path: normalizeAccessLogPath(record.Path),
StatusCode: record.Status,
})
}
type parsedAccessLogRecord struct {
Timestamp time.Time
Host string
RemoteAddr string
Path string
Status int
}
@@ -176,26 +206,28 @@ func parseJSONAccessLogRecord(raw string) (parsedAccessLogRecord, bool) {
Timestamp: timestamp,
Host: strings.TrimSpace(record.Host),
RemoteAddr: strings.TrimSpace(record.RemoteAddr),
Path: normalizeAccessLogPath(record.Path),
Status: record.Status,
}, true
}
func parseCombinedAccessLogRecord(raw string) (parsedAccessLogRecord, bool) {
matches := combinedAccessLogPattern.FindStringSubmatch(raw)
if len(matches) != 4 {
if len(matches) != 5 {
return parsedAccessLogRecord{}, false
}
timestamp, err := parseAccessLogTime(matches[2])
if err != nil {
return parsedAccessLogRecord{}, false
}
status, err := strconv.Atoi(matches[3])
status, err := strconv.Atoi(matches[4])
if err != nil {
return parsedAccessLogRecord{}, false
}
return parsedAccessLogRecord{
Timestamp: timestamp,
RemoteAddr: strings.TrimSpace(matches[1]),
Path: normalizeAccessLogPath(matches[3]),
Status: status,
}, true
}
@@ -217,6 +249,13 @@ func (aggregate *trafficAggregate) report() *protocol.NodeTrafficReport {
}
}
func (aggregate *trafficAggregate) accessLogs() []protocol.NodeAccessLog {
if aggregate == nil || len(aggregate.logs) == 0 {
return []protocol.NodeAccessLog{}
}
return append([]protocol.NodeAccessLog(nil), aggregate.logs...)
}
func parseAccessLogTime(value string) (time.Time, error) {
trimmed := strings.TrimSpace(value)
if trimmed == "" {
@@ -258,6 +297,20 @@ type trafficCountItem struct {
value int64
}
func normalizeAccessLogPath(value string) string {
trimmed := strings.TrimSpace(value)
if trimmed == "" {
return ""
}
if strings.HasPrefix(trimmed, "http://") || strings.HasPrefix(trimmed, "https://") {
return trimmed
}
if strings.HasPrefix(trimmed, "/") {
return trimmed
}
return "/" + trimmed
}
func topCounts(values map[string]int64, limit int) map[string]int64 {
return cloneTrafficCounts(values, limit)
}
atsf_agent/internal/observability/traffic_test.go
+32 -4
View File
@@ -18,9 +18,9 @@ func TestBuildTrafficReportAggregatesManagedAccessLog(t *testing.T) {
}
logPath := filepath.Join(filepath.Dir(routeConfigPath), "atsflare_access.log")
content := []byte(
"{\"ts\":\"2026-03-14T08:00:00Z\",\"host\":\"app.example.com\",\"remote_addr\":\"10.0.0.1\",\"status\":200}\n" +
"{\"ts\":\"2026-03-14T08:00:05Z\",\"host\":\"app.example.com\",\"remote_addr\":\"10.0.0.2\",\"status\":503}\n" +
"{\"ts\":\"2026-03-14T08:00:08Z\",\"host\":\"api.example.com\",\"remote_addr\":\"10.0.0.1\",\"status\":200}\n",
"{\"ts\":\"2026-03-14T08:00:00Z\",\"host\":\"app.example.com\",\"path\":\"/\",\"remote_addr\":\"10.0.0.1\",\"status\":200}\n" +
"{\"ts\":\"2026-03-14T08:00:05Z\",\"host\":\"app.example.com\",\"path\":\"/healthz\",\"remote_addr\":\"10.0.0.2\",\"status\":503}\n" +
"{\"ts\":\"2026-03-14T08:00:08Z\",\"host\":\"api.example.com\",\"path\":\"/api\",\"remote_addr\":\"10.0.0.1\",\"status\":200}\n",
)
if err := os.WriteFile(logPath, content, 0o644); err != nil {
t.Fatalf("WriteFile failed: %v", err)
@@ -62,7 +62,7 @@ func TestBuildTrafficReportResetsOffsetAfterTruncate(t *testing.T) {
t.Fatalf("MkdirAll failed: %v", err)
}
logPath := filepath.Join(filepath.Dir(routeConfigPath), "atsflare_access.log")
if err := os.WriteFile(logPath, []byte("{\"ts\":\"2026-03-14T09:00:00Z\",\"host\":\"app.example.com\",\"remote_addr\":\"10.0.0.3\",\"status\":200}\n"), 0o644); err != nil {
if err := os.WriteFile(logPath, []byte("{\"ts\":\"2026-03-14T09:00:00Z\",\"host\":\"app.example.com\",\"path\":\"/\",\"remote_addr\":\"10.0.0.3\",\"status\":200}\n"), 0o644); err != nil {
t.Fatalf("WriteFile failed: %v", err)
}
@@ -77,6 +77,34 @@ func TestBuildTrafficReportResetsOffsetAfterTruncate(t *testing.T) {
}
}
func TestBuildTrafficObservabilityReturnsAccessLogs(t *testing.T) {
tempDir := t.TempDir()
routeConfigPath := filepath.Join(tempDir, "conf.d", "atsflare_routes.conf")
if err := os.MkdirAll(filepath.Dir(routeConfigPath), 0o755); err != nil {
t.Fatalf("MkdirAll failed: %v", err)
}
logPath := filepath.Join(filepath.Dir(routeConfigPath), "atsflare_access.log")
content := []byte(
"{\"ts\":\"2026-03-14T08:00:00Z\",\"host\":\"app.example.com\",\"path\":\"/login\",\"remote_addr\":\"10.0.0.1\",\"status\":200}\n" +
"{\"ts\":\"2026-03-14T08:00:05Z\",\"host\":\"api.example.com\",\"path\":\"/v1/ping\",\"remote_addr\":\"10.0.0.2\",\"status\":502}\n",
)
if err := os.WriteFile(logPath, content, 0o644); err != nil {
t.Fatalf("WriteFile failed: %v", err)
}
stateStore := state.NewStore(filepath.Join(tempDir, "state.json"))
report, accessLogs := BuildTrafficObservability(&config.Config{RouteConfigPath: routeConfigPath}, stateStore, nil)
if report == nil || report.RequestCount != 2 {
t.Fatalf("expected traffic report, got %+v", report)
}
if len(accessLogs) != 2 {
t.Fatalf("expected access logs, got %+v", accessLogs)
}
if accessLogs[0].Path != "/login" || accessLogs[1].Path != "/v1/ping" {
t.Fatalf("unexpected access log paths: %+v", accessLogs)
}
}
func TestBuildTrafficReportParsesCombinedAccessLog(t *testing.T) {
tempDir := t.TempDir()
routeConfigPath := filepath.Join(tempDir, "conf.d", "atsflare_routes.conf")