mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-07 16:16:37 +08:00
refactor(repo): consolidate openflare-server to root and move subprojects to internal/apps
- Merge all files inside openflare-server to the repository root directory. - Relocate agent, relay, and flared subprojects from internal/ to internal/apps/. - Combine docker-compose files and update build context paths to root. - Update GitHub workflows and Dockerfiles to refer to new directories and package names. - Rewrite Go package imports across all files. - Resolve database renew test race condition and clean up docs.
This commit is contained in:
ryan
@@ -0,0 +1,150 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package tasks
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
)
|
||||
|
||||
const (
|
||||
// DatabaseCleanupTargetAccessLogs is the API cleanup target for access logs.
|
||||
DatabaseCleanupTargetAccessLogs = "node_access_logs"
|
||||
// DatabaseCleanupTargetMetricSnapshots is the API cleanup target for metric snapshots.
|
||||
DatabaseCleanupTargetMetricSnapshots = "node_metric_snapshots"
|
||||
// DatabaseCleanupTargetRequestReports is the API cleanup target for request reports.
|
||||
DatabaseCleanupTargetRequestReports = "node_request_reports"
|
||||
)
|
||||
|
||||
var databaseCleanupTargets = map[string]string{
|
||||
DatabaseCleanupTargetAccessLogs: "访问日志",
|
||||
DatabaseCleanupTargetMetricSnapshots: "性能快照",
|
||||
DatabaseCleanupTargetRequestReports: "请求聚合",
|
||||
}
|
||||
|
||||
// DatabaseCleanupInput describes a manual observability cleanup request.
|
||||
type DatabaseCleanupInput struct {
|
||||
Target string `json:"target"`
|
||||
RetentionDays *int `json:"retention_days"`
|
||||
}
|
||||
|
||||
// DatabaseCleanupResult summarizes a manual observability cleanup run.
|
||||
type DatabaseCleanupResult struct {
|
||||
Target string `json:"target"`
|
||||
TargetLabel string `json:"target_label"`
|
||||
DeletedCount int64 `json:"deleted_count"`
|
||||
DeleteAll bool `json:"delete_all"`
|
||||
RetentionDays *int `json:"retention_days,omitempty"`
|
||||
Cutoff *time.Time `json:"cutoff,omitempty"`
|
||||
}
|
||||
|
||||
// DatabaseAutoCleanupSummary summarizes a scheduled auto-cleanup run.
|
||||
type DatabaseAutoCleanupSummary struct {
|
||||
RetentionDays int `json:"retention_days"`
|
||||
ExecutedAt time.Time `json:"executed_at"`
|
||||
Results []DatabaseCleanupResult `json:"results"`
|
||||
}
|
||||
|
||||
// CleanupDatabaseObservability deletes observability rows for the given target.
|
||||
func CleanupDatabaseObservability(ctx context.Context, input DatabaseCleanupInput) (*DatabaseCleanupResult, error) {
|
||||
target := strings.TrimSpace(input.Target)
|
||||
targetLabel, ok := databaseCleanupTargets[target]
|
||||
if !ok {
|
||||
return nil, errors.New("unsupported cleanup target")
|
||||
}
|
||||
if input.RetentionDays != nil && *input.RetentionDays <= 0 {
|
||||
return nil, errors.New("retention_days 必须为大于 0 的整数")
|
||||
}
|
||||
|
||||
result := &DatabaseCleanupResult{
|
||||
Target: target,
|
||||
TargetLabel: targetLabel,
|
||||
DeleteAll: input.RetentionDays == nil,
|
||||
}
|
||||
|
||||
if input.RetentionDays == nil {
|
||||
deleted, err := deleteAllObservabilityRows(ctx, target)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result.DeletedCount = deleted
|
||||
return result, nil
|
||||
}
|
||||
|
||||
retentionDays := *input.RetentionDays
|
||||
cutoff := time.Now().UTC().Add(-time.Duration(retentionDays) * 24 * time.Hour)
|
||||
deleted, err := deleteObservabilityRowsBefore(ctx, target, cutoff)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result.DeletedCount = deleted
|
||||
result.RetentionDays = &retentionDays
|
||||
result.Cutoff = &cutoff
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// RunDatabaseAutoCleanupOnce runs retention-based cleanup for all observability targets.
|
||||
func RunDatabaseAutoCleanupOnce(now time.Time) (*DatabaseAutoCleanupSummary, error) {
|
||||
if !model.DatabaseAutoCleanupEnabled {
|
||||
return nil, nil
|
||||
}
|
||||
if model.DatabaseAutoCleanupRetentionDays < 1 {
|
||||
return nil, fmt.Errorf("database auto cleanup retention_days must be at least 1")
|
||||
}
|
||||
|
||||
retentionDays := model.DatabaseAutoCleanupRetentionDays
|
||||
ctx := context.Background()
|
||||
results := make([]DatabaseCleanupResult, 0, len(databaseCleanupTargets))
|
||||
for _, target := range []string{
|
||||
DatabaseCleanupTargetAccessLogs,
|
||||
DatabaseCleanupTargetMetricSnapshots,
|
||||
DatabaseCleanupTargetRequestReports,
|
||||
} {
|
||||
result, err := CleanupDatabaseObservability(ctx, DatabaseCleanupInput{
|
||||
Target: target,
|
||||
RetentionDays: &retentionDays,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
results = append(results, *result)
|
||||
}
|
||||
|
||||
return &DatabaseAutoCleanupSummary{
|
||||
RetentionDays: retentionDays,
|
||||
ExecutedAt: now.UTC(),
|
||||
Results: results,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func deleteAllObservabilityRows(ctx context.Context, target string) (int64, error) {
|
||||
switch target {
|
||||
case DatabaseCleanupTargetAccessLogs:
|
||||
return model.DeleteAllOpenFlareAccessLogs(ctx)
|
||||
case DatabaseCleanupTargetMetricSnapshots:
|
||||
return model.DeleteAllOpenFlareMetricSnapshots(ctx)
|
||||
case DatabaseCleanupTargetRequestReports:
|
||||
return model.DeleteAllOpenFlareRequestReports(ctx)
|
||||
default:
|
||||
return 0, errors.New("unsupported cleanup target")
|
||||
}
|
||||
}
|
||||
|
||||
func deleteObservabilityRowsBefore(ctx context.Context, target string, cutoff time.Time) (int64, error) {
|
||||
switch target {
|
||||
case DatabaseCleanupTargetAccessLogs:
|
||||
return model.DeleteOpenFlareAccessLogsBefore(ctx, cutoff)
|
||||
case DatabaseCleanupTargetMetricSnapshots:
|
||||
return model.DeleteOpenFlareMetricSnapshotsBefore(ctx, cutoff)
|
||||
case DatabaseCleanupTargetRequestReports:
|
||||
return model.DeleteOpenFlareRequestReportsBefore(ctx, cutoff)
|
||||
default:
|
||||
return 0, errors.New("unsupported cleanup target")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package tasks
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/db"
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
"github.com/glebarez/sqlite"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func setupDatabaseCleanupTestDB(t *testing.T) context.Context {
|
||||
t.Helper()
|
||||
|
||||
sqliteDB, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{
|
||||
DisableForeignKeyConstraintWhenMigrating: true,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, sqliteDB.AutoMigrate(
|
||||
&model.OpenFlareMetricSnapshot{},
|
||||
&model.OpenFlareRequestReport{},
|
||||
))
|
||||
db.SetDB(sqliteDB)
|
||||
resetAccessLogStore := model.SetAccessLogStoreForTest(model.NewMemoryAccessLogStore())
|
||||
t.Cleanup(func() {
|
||||
resetAccessLogStore()
|
||||
db.SetDB(nil)
|
||||
})
|
||||
return context.Background()
|
||||
}
|
||||
|
||||
func TestCleanupDatabaseObservabilityDeletesTargetedRows(t *testing.T) {
|
||||
ctx := setupDatabaseCleanupTestDB(t)
|
||||
now := time.Now().UTC()
|
||||
|
||||
require.NoError(t, db.DB(ctx).Create(&model.OpenFlareMetricSnapshot{
|
||||
NodeID: "node-a",
|
||||
CapturedAt: now.Add(-10 * 24 * time.Hour),
|
||||
CPUUsagePercent: 10,
|
||||
}).Error)
|
||||
require.NoError(t, db.DB(ctx).Create(&model.OpenFlareMetricSnapshot{
|
||||
NodeID: "node-a",
|
||||
CapturedAt: now.Add(-12 * time.Hour),
|
||||
CPUUsagePercent: 20,
|
||||
}).Error)
|
||||
|
||||
retentionDays := 7
|
||||
result, err := CleanupDatabaseObservability(ctx, DatabaseCleanupInput{
|
||||
Target: DatabaseCleanupTargetMetricSnapshots,
|
||||
RetentionDays: &retentionDays,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.False(t, result.DeleteAll)
|
||||
assert.Equal(t, int64(1), result.DeletedCount)
|
||||
|
||||
rows, err := model.ListOpenFlareMetricSnapshotsSince(ctx, "", time.Time{}, 0)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, rows, 1)
|
||||
assert.Equal(t, float64(20), rows[0].CPUUsagePercent)
|
||||
}
|
||||
|
||||
func TestCleanupDatabaseObservabilityDeletesAllRowsWhenRetentionMissing(t *testing.T) {
|
||||
ctx := setupDatabaseCleanupTestDB(t)
|
||||
now := time.Now().UTC()
|
||||
|
||||
require.NoError(t, model.InsertOpenFlareAccessLogsBatch(ctx, []*model.OpenFlareAccessLog{
|
||||
{
|
||||
NodeID: "node-a",
|
||||
LoggedAt: now.Add(-3 * time.Hour),
|
||||
RemoteAddr: "203.0.113.1",
|
||||
Host: "example.com",
|
||||
Path: "/one",
|
||||
StatusCode: 200,
|
||||
},
|
||||
{
|
||||
NodeID: "node-a",
|
||||
LoggedAt: now.Add(-2 * time.Hour),
|
||||
RemoteAddr: "203.0.113.2",
|
||||
Host: "example.com",
|
||||
Path: "/two",
|
||||
StatusCode: 502,
|
||||
},
|
||||
}))
|
||||
|
||||
result, err := CleanupDatabaseObservability(ctx, DatabaseCleanupInput{
|
||||
Target: DatabaseCleanupTargetAccessLogs,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.True(t, result.DeleteAll)
|
||||
assert.Equal(t, int64(2), result.DeletedCount)
|
||||
|
||||
rows, err := model.ListOpenFlareAccessLogs(ctx, model.OpenFlareAccessLogQuery{Page: 0, PageSize: 10})
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, rows)
|
||||
}
|
||||
|
||||
func TestRunDatabaseAutoCleanupOnceDeletesAllObservabilityTargets(t *testing.T) {
|
||||
ctx := setupDatabaseCleanupTestDB(t)
|
||||
now := time.Now().UTC()
|
||||
|
||||
require.NoError(t, model.InsertOpenFlareAccessLogsBatch(ctx, []*model.OpenFlareAccessLog{{
|
||||
NodeID: "node-a",
|
||||
LoggedAt: now.Add(-48 * time.Hour),
|
||||
RemoteAddr: "203.0.113.10",
|
||||
Host: "example.com",
|
||||
Path: "/access",
|
||||
StatusCode: 200,
|
||||
}}))
|
||||
require.NoError(t, db.DB(ctx).Create(&model.OpenFlareMetricSnapshot{
|
||||
NodeID: "node-a",
|
||||
CapturedAt: now.Add(-48 * time.Hour),
|
||||
CPUUsagePercent: 10,
|
||||
}).Error)
|
||||
require.NoError(t, db.DB(ctx).Create(&model.OpenFlareRequestReport{
|
||||
NodeID: "node-a",
|
||||
WindowStartedAt: now.Add(-49 * time.Hour),
|
||||
WindowEndedAt: now.Add(-48 * time.Hour),
|
||||
RequestCount: 15,
|
||||
}).Error)
|
||||
|
||||
previousEnabled := model.DatabaseAutoCleanupEnabled
|
||||
previousRetentionDays := model.DatabaseAutoCleanupRetentionDays
|
||||
model.DatabaseAutoCleanupEnabled = true
|
||||
model.DatabaseAutoCleanupRetentionDays = 1
|
||||
t.Cleanup(func() {
|
||||
model.DatabaseAutoCleanupEnabled = previousEnabled
|
||||
model.DatabaseAutoCleanupRetentionDays = previousRetentionDays
|
||||
})
|
||||
|
||||
summary, err := RunDatabaseAutoCleanupOnce(now)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, summary)
|
||||
require.Len(t, summary.Results, 3)
|
||||
|
||||
accessLogs, err := model.ListOpenFlareAccessLogs(ctx, model.OpenFlareAccessLogQuery{Page: 0, PageSize: 10})
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, accessLogs)
|
||||
|
||||
metricSnapshots, err := model.ListOpenFlareMetricSnapshotsSince(ctx, "", time.Time{}, 0)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, metricSnapshots)
|
||||
|
||||
requestReports, err := model.ListOpenFlareRequestReportsSince(ctx, "", time.Time{}, 0)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, requestReports)
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Package tasks hosts shared OpenFlare background job business logic.
|
||||
//
|
||||
// Scheduled execution is handled by the Wavelet Asynq task framework; handlers
|
||||
// live in internal/apps/openflare/async_tasks.go and are registered via
|
||||
// bootstrap.RegisterTasks().
|
||||
package tasks
|
||||
@@ -0,0 +1,44 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package tasks
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/apps/openflare/tls"
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
"github.com/Rain-kl/Wavelet/pkg/logger"
|
||||
)
|
||||
|
||||
// RunSSLRenewJob renews all TLS certificates that are due for renewal.
|
||||
func RunSSLRenewJob(ctx context.Context) error {
|
||||
logger.InfoF(ctx, "[OpenFlareTasks] SSL renew job started")
|
||||
|
||||
certificates, err := model.ListTLSCertificates(ctx)
|
||||
if err != nil {
|
||||
logger.ErrorF(ctx, "[OpenFlareTasks] list certificates failed: %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
due := tls.CertificatesDueForRenewal(certificates, now)
|
||||
if len(due) == 0 {
|
||||
logger.InfoF(ctx, "[OpenFlareTasks] SSL renew job completed: no certificates due")
|
||||
return nil
|
||||
}
|
||||
|
||||
var triggered int
|
||||
for _, cert := range due {
|
||||
logger.InfoF(ctx, "[OpenFlareTasks] renewing certificate id=%d domain=%s", cert.ID, cert.PrimaryDomain)
|
||||
if _, err := tls.RenewCertificate(ctx, cert.ID); err != nil {
|
||||
logger.ErrorF(ctx, "[OpenFlareTasks] renew certificate id=%d domain=%s failed: %v", cert.ID, cert.PrimaryDomain, err)
|
||||
continue
|
||||
}
|
||||
triggered++
|
||||
}
|
||||
|
||||
logger.InfoF(ctx, "[OpenFlareTasks] SSL renew job completed: triggered=%d eligible=%d", triggered, len(due))
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package tasks
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/apps/openflare/tls"
|
||||
"github.com/Rain-kl/Wavelet/internal/config"
|
||||
"github.com/Rain-kl/Wavelet/internal/db"
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
"github.com/glebarez/sqlite"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func setupSSLRenewTestDB(t *testing.T) func() {
|
||||
t.Helper()
|
||||
|
||||
sqliteDB, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{
|
||||
DisableForeignKeyConstraintWhenMigrating: true,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, sqliteDB.AutoMigrate(&model.TLSCertificate{}))
|
||||
|
||||
db.SetDB(sqliteDB)
|
||||
oldSecret := config.Config.App.SessionSecret
|
||||
config.Config.App.SessionSecret = "test_session_secret_for_ssl_renew"
|
||||
return func() {
|
||||
db.SetDB(nil)
|
||||
config.Config.App.SessionSecret = oldSecret
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSSLRenewJobTriggersDueCertificates(t *testing.T) {
|
||||
cleanup := setupSSLRenewTestDB(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
|
||||
restore := tls.SetObtainCertificateFuncForTest(func(ctx context.Context, cert *model.TLSCertificate) error {
|
||||
return nil
|
||||
})
|
||||
defer restore()
|
||||
|
||||
now := time.Date(2026, 6, 18, 12, 0, 0, 0, time.UTC)
|
||||
due := &model.TLSCertificate{
|
||||
Name: "due-cert",
|
||||
Provider: "acme",
|
||||
AutoRenew: true,
|
||||
ApplyStatus: "ready",
|
||||
PrimaryDomain: "due.example.com",
|
||||
CertPEM: " ",
|
||||
KeyPEM: " ",
|
||||
NotAfter: now.Add(2 * 24 * time.Hour),
|
||||
}
|
||||
fresh := &model.TLSCertificate{
|
||||
Name: "fresh-cert",
|
||||
Provider: "acme",
|
||||
AutoRenew: true,
|
||||
ApplyStatus: "ready",
|
||||
PrimaryDomain: "fresh.example.com",
|
||||
CertPEM: " ",
|
||||
KeyPEM: " ",
|
||||
NotAfter: now.Add(30 * 24 * time.Hour),
|
||||
}
|
||||
require.NoError(t, model.CreateTLSCertificateRecord(ctx, due))
|
||||
require.NoError(t, model.CreateTLSCertificateRecord(ctx, fresh))
|
||||
|
||||
require.NoError(t, RunSSLRenewJob(ctx))
|
||||
|
||||
renewed, err := model.GetTLSCertificateByID(ctx, due.ID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "applying", renewed.ApplyStatus)
|
||||
|
||||
unchanged, err := model.GetTLSCertificateByID(ctx, fresh.ID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "ready", unchanged.ApplyStatus)
|
||||
}
|
||||
Reference in New Issue
Block a user