mirror of
https://github.com/Sagit-chu/flvx.git
synced 2026-09-28 23:56:36 +08:00
Compare commits
8 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 60311d3e47 | |||
| b4192c9e94 | |||
| f05e9480ee | |||
| 9861b44107 | |||
| d8144821e6 | |||
| 023be27287 | |||
| e8d5687419 | |||
| 0fbe570597 |
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,598 @@
|
||||
# Monitoring Retention And Storage Display Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Add configurable monitoring data retention and show database storage usage on the config page.
|
||||
|
||||
**Architecture:** Store retention in `vite_config` as `monitor_retention_days`, parse it through a focused monitoring helper, and reuse it from existing cleanup loops. Add a repository storage-summary helper, expose it via an admin-only API, and render it in the existing React config page.
|
||||
|
||||
**Tech Stack:** Go `net/http`, GORM, SQLite/PostgreSQL, Vite/React/TypeScript, existing shadcn bridge components.
|
||||
|
||||
---
|
||||
|
||||
## File Structure
|
||||
|
||||
- Create: `go-backend/internal/monitoring/retention.go` for retention constants, parsing, and validation.
|
||||
- Test: `go-backend/internal/monitoring/retention_test.go`.
|
||||
- Modify: `go-backend/internal/metrics/ingestion.go` and `go-backend/internal/metrics/ingestion_test.go` for config-driven cleanup.
|
||||
- Modify: `go-backend/internal/http/handler/tunnel_quality_prober.go` so `tunnel_quality` uses the same retention and still prunes when probing is disabled.
|
||||
- Create: `go-backend/internal/store/repo/repository_storage.go` and `go-backend/internal/store/repo/repository_storage_test.go` for database size summaries.
|
||||
- Modify: `go-backend/internal/store/repo/repository.go` to keep the SQLite DB path on `Repository`.
|
||||
- Create: `go-backend/internal/http/handler/storage.go` for the storage endpoint.
|
||||
- Modify: `go-backend/internal/http/handler/handler.go` to register `/api/v1/system/storage` and validate `monitor_retention_days`.
|
||||
- Modify: `go-backend/internal/http/middleware/auth.go` so `/api/v1/system/*` is admin-only.
|
||||
- Create: `go-backend/tests/contract/storage_contract_test.go` for endpoint auth/shape coverage.
|
||||
- Modify: `vite-frontend/src/api/types.ts`, `vite-frontend/src/api/index.ts`, and `vite-frontend/src/pages/config.tsx` for UI display.
|
||||
|
||||
Implementation should not create git commits unless the user explicitly requests them.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: Add Retention Config Helper
|
||||
|
||||
**Files:**
|
||||
- Create: `go-backend/internal/monitoring/retention.go`
|
||||
- Create: `go-backend/internal/monitoring/retention_test.go`
|
||||
- Modify: `go-backend/internal/http/handler/handler.go`
|
||||
|
||||
- [ ] **Step 1: Write the failing tests**
|
||||
|
||||
Create `go-backend/internal/monitoring/retention_test.go`:
|
||||
|
||||
```go
|
||||
package monitoring
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestMonitoringRetentionDaysFromConfigMap(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
cfg map[string]string
|
||||
want int
|
||||
}{
|
||||
{"missing uses default", nil, 7},
|
||||
{"valid custom", map[string]string{ConfigMonitorRetentionDays: "3"}, 3},
|
||||
{"trimmed custom", map[string]string{ConfigMonitorRetentionDays: " 30 "}, 30},
|
||||
{"invalid uses default", map[string]string{ConfigMonitorRetentionDays: "abc"}, 7},
|
||||
{"too small uses default", map[string]string{ConfigMonitorRetentionDays: "0"}, 7},
|
||||
{"too large uses default", map[string]string{ConfigMonitorRetentionDays: "3651"}, 7},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := MonitoringRetentionDaysFromConfigMap(tc.cfg); got != tc.want {
|
||||
t.Fatalf("expected %d, got %d", tc.want, got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeMonitoringRetentionDays(t *testing.T) {
|
||||
for _, value := range []string{"1", "7", "3650", " 30 "} {
|
||||
if got, err := NormalizeMonitoringRetentionDays(value); err != nil || got == "" {
|
||||
t.Fatalf("expected %q valid, got value=%q err=%v", value, got, err)
|
||||
}
|
||||
}
|
||||
for _, value := range []string{"", "0", "-1", "3651", "abc", "1.5"} {
|
||||
if got, err := NormalizeMonitoringRetentionDays(value); err == nil {
|
||||
t.Fatalf("expected %q invalid, got value=%q", value, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run tests to verify failure**
|
||||
|
||||
Run: `go test ./internal/monitoring -run 'TestMonitoringRetentionDaysFromConfigMap|TestNormalizeMonitoringRetentionDays' -count=1`
|
||||
|
||||
Expected: FAIL with undefined `ConfigMonitorRetentionDays`, `MonitoringRetentionDaysFromConfigMap`, and `NormalizeMonitoringRetentionDays`.
|
||||
|
||||
- [ ] **Step 3: Implement helper**
|
||||
|
||||
Create `go-backend/internal/monitoring/retention.go`:
|
||||
|
||||
```go
|
||||
package monitoring
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
ConfigMonitorRetentionDays = "monitor_retention_days"
|
||||
DefaultMonitorRetentionDays = 7
|
||||
MinMonitorRetentionDays = 1
|
||||
MaxMonitorRetentionDays = 3650
|
||||
)
|
||||
|
||||
func MonitoringRetentionDaysFromConfigMap(cfg map[string]string) int {
|
||||
if cfg == nil {
|
||||
return DefaultMonitorRetentionDays
|
||||
}
|
||||
days, err := parseMonitoringRetentionDays(cfg[ConfigMonitorRetentionDays])
|
||||
if err != nil {
|
||||
return DefaultMonitorRetentionDays
|
||||
}
|
||||
return days
|
||||
}
|
||||
|
||||
func NormalizeMonitoringRetentionDays(value string) (string, error) {
|
||||
days, err := parseMonitoringRetentionDays(value)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return strconv.Itoa(days), nil
|
||||
}
|
||||
|
||||
func parseMonitoringRetentionDays(value string) (int, error) {
|
||||
trimmed := strings.TrimSpace(value)
|
||||
if trimmed == "" {
|
||||
return 0, fmt.Errorf("监控数据保留天数不能为空")
|
||||
}
|
||||
days, err := strconv.Atoi(trimmed)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("监控数据保留天数必须是整数")
|
||||
}
|
||||
if days < MinMonitorRetentionDays || days > MaxMonitorRetentionDays {
|
||||
return 0, fmt.Errorf("监控数据保留天数必须在 %d 到 %d 之间", MinMonitorRetentionDays, MaxMonitorRetentionDays)
|
||||
}
|
||||
return days, nil
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Validate config updates**
|
||||
|
||||
In `go-backend/internal/http/handler/handler.go`, add this case to `normalizeAndValidateConfigValue`:
|
||||
|
||||
```go
|
||||
case monitoring.ConfigMonitorRetentionDays:
|
||||
return monitoring.NormalizeMonitoringRetentionDays(value)
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Run tests**
|
||||
|
||||
Run: `go test ./internal/monitoring ./internal/http/handler -run 'TestMonitoringRetention|TestNormalize|Test' -count=1`
|
||||
|
||||
Expected: PASS or only unrelated pre-existing failures, which must be investigated before continuing.
|
||||
|
||||
---
|
||||
|
||||
### Task 2: Use Retention Config In Cleanup
|
||||
|
||||
**Files:**
|
||||
- Modify: `go-backend/internal/metrics/ingestion.go`
|
||||
- Modify: `go-backend/internal/metrics/ingestion_test.go`
|
||||
- Modify: `go-backend/internal/http/handler/tunnel_quality_prober.go`
|
||||
|
||||
- [ ] **Step 1: Write failing cleanup test**
|
||||
|
||||
Append to `go-backend/internal/metrics/ingestion_test.go`, adding `go-backend/internal/store/model` to imports:
|
||||
|
||||
```go
|
||||
func TestPruneMetricsUsesConfiguredRetentionDays(t *testing.T) {
|
||||
r, err := repo.Open(":memory:")
|
||||
if err != nil {
|
||||
t.Fatalf("open repo: %v", err)
|
||||
}
|
||||
defer r.Close()
|
||||
|
||||
now := time.Now().UnixMilli()
|
||||
if err := r.UpsertConfig("monitor_retention_days", "2", now); err != nil {
|
||||
t.Fatalf("upsert retention config: %v", err)
|
||||
}
|
||||
|
||||
oldMetric := &model.NodeMetric{NodeID: 1, Timestamp: now - int64(3*24*time.Hour/time.Millisecond), CPUUsage: 10}
|
||||
newMetric := &model.NodeMetric{NodeID: 1, Timestamp: now - int64(1*24*time.Hour/time.Millisecond), CPUUsage: 20}
|
||||
if err := r.InsertNodeMetric(oldMetric); err != nil {
|
||||
t.Fatalf("insert old metric: %v", err)
|
||||
}
|
||||
if err := r.InsertNodeMetric(newMetric); err != nil {
|
||||
t.Fatalf("insert new metric: %v", err)
|
||||
}
|
||||
|
||||
svc := NewIngestionService(r)
|
||||
svc.pruneMetricsAt(time.UnixMilli(now))
|
||||
|
||||
metrics, err := r.GetNodeMetrics(1, now-int64(4*24*time.Hour/time.Millisecond), now+1000)
|
||||
if err != nil {
|
||||
t.Fatalf("get node metrics: %v", err)
|
||||
}
|
||||
if len(metrics) != 1 || metrics[0].CPUUsage != 20 {
|
||||
t.Fatalf("expected only newer metric to remain, got %#v", metrics)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify failure**
|
||||
|
||||
Run: `go test ./internal/metrics -run TestPruneMetricsUsesConfiguredRetentionDays -count=1`
|
||||
|
||||
Expected: FAIL with undefined `pruneMetricsAt`.
|
||||
|
||||
- [ ] **Step 3: Implement config-driven prune**
|
||||
|
||||
In `go-backend/internal/metrics/ingestion.go`, import `go-backend/internal/monitoring` and replace `pruneMetrics` with:
|
||||
|
||||
```go
|
||||
func (s *IngestionService) retentionDaysFromConfig() int {
|
||||
if s == nil || s.repo == nil {
|
||||
return monitoring.DefaultMonitorRetentionDays
|
||||
}
|
||||
cfg, err := s.repo.GetConfigsByNames([]string{monitoring.ConfigMonitorRetentionDays})
|
||||
if err != nil {
|
||||
return monitoring.DefaultMonitorRetentionDays
|
||||
}
|
||||
return monitoring.MonitoringRetentionDaysFromConfigMap(cfg)
|
||||
}
|
||||
|
||||
func (s *IngestionService) pruneMetrics() {
|
||||
s.pruneMetricsAt(time.Now())
|
||||
}
|
||||
|
||||
func (s *IngestionService) pruneMetricsAt(now time.Time) {
|
||||
cutoff := now.Add(-time.Duration(s.retentionDaysFromConfig()) * 24 * time.Hour).UnixMilli()
|
||||
if s.repo == nil {
|
||||
return
|
||||
}
|
||||
if err := s.repo.PruneNodeMetrics(cutoff); err != nil {
|
||||
log.Printf("monitoring prune failed op=node_metric cutoff=%d err=%v", cutoff, err)
|
||||
}
|
||||
if err := s.repo.PruneTunnelMetrics(cutoff); err != nil {
|
||||
log.Printf("monitoring prune failed op=tunnel_metric cutoff=%d err=%v", cutoff, err)
|
||||
}
|
||||
if err := s.repo.PruneServiceMonitorResults(cutoff); err != nil {
|
||||
log.Printf("monitoring prune failed op=service_monitor_result cutoff=%d err=%v", cutoff, err)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Remove the unused `retentionDays` field from `IngestionService` and remove `svc.retentionDays = 1` from existing tests.
|
||||
|
||||
- [ ] **Step 4: Update tunnel quality pruning**
|
||||
|
||||
In `go-backend/internal/http/handler/tunnel_quality_prober.go`, import `go-backend/internal/monitoring`, remove `tunnelQualityRetention`, remove the `if !p.isEnabled() { return }` guard from `maybePrune`, and calculate cutoff with:
|
||||
|
||||
```go
|
||||
func (p *tunnelQualityProber) retentionDays() int {
|
||||
if p == nil || p.handler == nil || p.handler.repo == nil {
|
||||
return monitoring.DefaultMonitorRetentionDays
|
||||
}
|
||||
cfg, err := p.handler.repo.GetConfigsByNames([]string{monitoring.ConfigMonitorRetentionDays})
|
||||
if err != nil {
|
||||
return monitoring.DefaultMonitorRetentionDays
|
||||
}
|
||||
return monitoring.MonitoringRetentionDaysFromConfigMap(cfg)
|
||||
}
|
||||
```
|
||||
|
||||
Then use:
|
||||
|
||||
```go
|
||||
cutoff := now - int64(time.Duration(p.retentionDays())*24*time.Hour/time.Millisecond)
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Run cleanup tests**
|
||||
|
||||
Run: `go test ./internal/metrics ./internal/http/handler -run 'TestPruneMetrics|TestPruneMetricsUsesConfiguredRetentionDays|TunnelQuality' -count=1`
|
||||
|
||||
Expected: PASS.
|
||||
|
||||
---
|
||||
|
||||
### Task 3: Add Storage Summary Backend API
|
||||
|
||||
**Files:**
|
||||
- Modify: `go-backend/internal/store/repo/repository.go`
|
||||
- Create: `go-backend/internal/store/repo/repository_storage.go`
|
||||
- Create: `go-backend/internal/store/repo/repository_storage_test.go`
|
||||
- Create: `go-backend/internal/http/handler/storage.go`
|
||||
- Modify: `go-backend/internal/http/handler/handler.go`
|
||||
- Modify: `go-backend/internal/http/middleware/auth.go`
|
||||
- Create: `go-backend/tests/contract/storage_contract_test.go`
|
||||
|
||||
- [ ] **Step 1: Write failing repository tests**
|
||||
|
||||
Create `go-backend/internal/store/repo/repository_storage_test.go`:
|
||||
|
||||
```go
|
||||
package repo
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"go-backend/internal/store/model"
|
||||
)
|
||||
|
||||
func TestDatabaseStorageSummarySQLiteIncludesSize(t *testing.T) {
|
||||
dbPath := filepath.Join(t.TempDir(), "storage.db")
|
||||
r, err := Open(dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("open repo: %v", err)
|
||||
}
|
||||
defer r.Close()
|
||||
|
||||
if err := r.InsertNodeMetric(&model.NodeMetric{NodeID: 1, Timestamp: 123, CPUUsage: 1}); err != nil {
|
||||
t.Fatalf("insert metric: %v", err)
|
||||
}
|
||||
summary, err := r.DatabaseStorageSummary()
|
||||
if err != nil {
|
||||
t.Fatalf("storage summary: %v", err)
|
||||
}
|
||||
if summary.DBType != "sqlite" || summary.DatabaseSizeBytes <= 0 || summary.DatabaseSizeText == "" {
|
||||
t.Fatalf("unexpected summary: %#v", summary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatDatabaseSize(t *testing.T) {
|
||||
for _, tc := range []struct{ bytes int64; want string }{{0, "0 B"}, {512, "512 B"}, {1024, "1.0 KB"}, {1024 * 1024, "1.0 MB"}} {
|
||||
if got := formatDatabaseSize(tc.bytes); got != tc.want {
|
||||
t.Fatalf("formatDatabaseSize(%d)=%q want %q", tc.bytes, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify failure**
|
||||
|
||||
Run: `go test ./internal/store/repo -run 'TestDatabaseStorageSummarySQLiteIncludesSize|TestFormatDatabaseSize' -count=1`
|
||||
|
||||
Expected: FAIL with undefined `DatabaseStorageSummary` and `formatDatabaseSize`.
|
||||
|
||||
- [ ] **Step 3: Implement repository helper**
|
||||
|
||||
Modify `Repository` in `repository.go`:
|
||||
|
||||
```go
|
||||
type Repository struct {
|
||||
db *gorm.DB
|
||||
dbPath string
|
||||
}
|
||||
```
|
||||
|
||||
Return `&Repository{db: db, dbPath: path}` from `Open` and `&Repository{db: db}` from `OpenPostgres`.
|
||||
|
||||
Create `go-backend/internal/store/repo/repository_storage.go`:
|
||||
|
||||
```go
|
||||
package repo
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
type DatabaseStorageSummary struct {
|
||||
DBType string `json:"dbType"`
|
||||
DatabaseSizeBytes int64 `json:"databaseSizeBytes"`
|
||||
DatabaseSizeText string `json:"databaseSizeText"`
|
||||
}
|
||||
|
||||
func (r *Repository) DatabaseStorageSummary() (DatabaseStorageSummary, error) {
|
||||
if r == nil || r.db == nil {
|
||||
return DatabaseStorageSummary{}, errors.New("repository not initialized")
|
||||
}
|
||||
switch r.db.Dialector.Name() {
|
||||
case "sqlite":
|
||||
size, err := sqliteDatabaseFileSize(r.dbPath)
|
||||
if err != nil { return DatabaseStorageSummary{}, err }
|
||||
return DatabaseStorageSummary{"sqlite", size, formatDatabaseSize(size)}, nil
|
||||
case "postgres":
|
||||
var size int64
|
||||
if err := r.db.Raw("SELECT pg_database_size(current_database())").Scan(&size).Error; err != nil { return DatabaseStorageSummary{}, err }
|
||||
return DatabaseStorageSummary{"postgres", size, formatDatabaseSize(size)}, nil
|
||||
default:
|
||||
return DatabaseStorageSummary{}, fmt.Errorf("unsupported database dialect %q", r.db.Dialector.Name())
|
||||
}
|
||||
}
|
||||
|
||||
func sqliteDatabaseFileSize(path string) (int64, error) {
|
||||
if path == "" || path == ":memory:" { return 0, nil }
|
||||
var total int64
|
||||
for _, candidate := range []string{path, path + "-wal", path + "-shm"} {
|
||||
info, err := os.Stat(candidate)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) { continue }
|
||||
return 0, err
|
||||
}
|
||||
if !info.IsDir() { total += info.Size() }
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
func formatDatabaseSize(bytes int64) string {
|
||||
if bytes < 1024 { return fmt.Sprintf("%d B", bytes) }
|
||||
units := []string{"KB", "MB", "GB", "TB"}
|
||||
value := float64(bytes) / 1024
|
||||
for _, unit := range units {
|
||||
if value < 1024 || unit == "TB" { return fmt.Sprintf("%.1f %s", value, unit) }
|
||||
value /= 1024
|
||||
}
|
||||
return fmt.Sprintf("%d B", bytes)
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Add API handler and route**
|
||||
|
||||
Create `go-backend/internal/http/handler/storage.go`:
|
||||
|
||||
```go
|
||||
package handler
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"go-backend/internal/http/response"
|
||||
)
|
||||
|
||||
func (h *Handler) storageSummary(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet && r.Method != http.MethodPost {
|
||||
response.WriteJSON(w, response.ErrDefault("请求失败"))
|
||||
return
|
||||
}
|
||||
if h == nil || h.repo == nil {
|
||||
response.WriteJSON(w, response.Err(-2, "repository not initialized"))
|
||||
return
|
||||
}
|
||||
summary, err := h.repo.DatabaseStorageSummary()
|
||||
if err != nil {
|
||||
response.WriteJSON(w, response.Err(-2, err.Error()))
|
||||
return
|
||||
}
|
||||
response.WriteJSON(w, response.OK(summary))
|
||||
}
|
||||
```
|
||||
|
||||
Register in `Handler.Register`: `mux.HandleFunc("/api/v1/system/storage", h.storageSummary)`.
|
||||
|
||||
In `requiresAdmin`, add:
|
||||
|
||||
```go
|
||||
if strings.HasPrefix(path, "/api/v1/system/") {
|
||||
return true
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Write contract test for auth and shape**
|
||||
|
||||
Create `go-backend/tests/contract/storage_contract_test.go` with a test that sends GET `/api/v1/system/storage` as non-admin and expects `403`, then as admin and expects `code == 0`, `dbType`, numeric `databaseSizeBytes`, and `databaseSizeText`.
|
||||
|
||||
- [ ] **Step 6: Run storage tests**
|
||||
|
||||
Run: `go test ./internal/store/repo ./tests/contract -run 'TestDatabaseStorageSummarySQLiteIncludesSize|TestFormatDatabaseSize|TestStorageSummaryRequiresAdminAndReturnsSize' -count=1`
|
||||
|
||||
Expected: PASS.
|
||||
|
||||
---
|
||||
|
||||
### Task 4: Add Frontend Config UI
|
||||
|
||||
**Files:**
|
||||
- Modify: `vite-frontend/src/api/types.ts`
|
||||
- Modify: `vite-frontend/src/api/index.ts`
|
||||
- Modify: `vite-frontend/src/pages/config.tsx`
|
||||
|
||||
- [ ] **Step 1: Add API type and function**
|
||||
|
||||
In `types.ts` add:
|
||||
|
||||
```ts
|
||||
export interface StorageSummaryApiData {
|
||||
dbType: string;
|
||||
databaseSizeBytes: number;
|
||||
databaseSizeText: string;
|
||||
}
|
||||
```
|
||||
|
||||
In `index.ts`, import `StorageSummaryApiData` and add:
|
||||
|
||||
```ts
|
||||
export const getStorageSummary = () =>
|
||||
Network.get<StorageSummaryApiData>("/system/storage");
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Add retention config item**
|
||||
|
||||
In `config.tsx`, add to `CONFIG_ITEMS` near monitoring:
|
||||
|
||||
```ts
|
||||
{
|
||||
key: "monitor_retention_days",
|
||||
label: "监控数据保留天数",
|
||||
placeholder: "7",
|
||||
description:
|
||||
"统一清理节点指标、隧道流量、服务监控结果和隧道质量历史;默认 7 天。",
|
||||
type: "input",
|
||||
},
|
||||
```
|
||||
|
||||
Add `"monitor_retention_days"` to `getInitialConfigs()` keys.
|
||||
|
||||
- [ ] **Step 3: Fetch and display database size**
|
||||
|
||||
In `config.tsx`, add state:
|
||||
|
||||
```ts
|
||||
const [storageSummary, setStorageSummary] = useState<string>("加载中...");
|
||||
```
|
||||
|
||||
Add a load effect:
|
||||
|
||||
```ts
|
||||
useEffect(() => {
|
||||
let mounted = true;
|
||||
getStorageSummary()
|
||||
.then((response) => {
|
||||
if (!mounted) return;
|
||||
if (response.code === 0 && response.data?.databaseSizeText) {
|
||||
setStorageSummary(response.data.databaseSizeText);
|
||||
} else {
|
||||
setStorageSummary("获取失败");
|
||||
}
|
||||
})
|
||||
.catch(() => {
|
||||
if (mounted) setStorageSummary("获取失败");
|
||||
});
|
||||
return () => {
|
||||
mounted = false;
|
||||
};
|
||||
}, []);
|
||||
```
|
||||
|
||||
Render inside the basic settings card before the save button:
|
||||
|
||||
```tsx
|
||||
<Divider className="my-2" />
|
||||
<div className="space-y-1">
|
||||
<p className="text-sm font-medium text-gray-700 dark:text-gray-300">
|
||||
数据库占用
|
||||
</p>
|
||||
<p className="text-xs text-gray-500 dark:text-gray-400">
|
||||
当前后端数据库文件/实例占用空间,仅用于容量参考。
|
||||
</p>
|
||||
<div className="rounded-lg border border-divider bg-default-50/60 dark:bg-default-100/10 px-4 py-3 text-sm font-semibold text-default-800 dark:text-default-200">
|
||||
{storageSummary}
|
||||
</div>
|
||||
</div>
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Build frontend**
|
||||
|
||||
Run: `pnpm run build` from `vite-frontend`.
|
||||
|
||||
Expected: TypeScript and Vite build pass.
|
||||
|
||||
---
|
||||
|
||||
### Task 5: Final Verification
|
||||
|
||||
**Files:**
|
||||
- All files changed by previous tasks.
|
||||
|
||||
- [ ] **Step 1: Run backend tests**
|
||||
|
||||
Run: `go test ./...` from `go-backend`.
|
||||
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 2: Run frontend build**
|
||||
|
||||
Run: `pnpm run build` from `vite-frontend`.
|
||||
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 3: Review diff**
|
||||
|
||||
Run: `git diff --stat` and `git diff -- docs/superpowers/specs/2026-04-28-monitoring-retention-storage-design.md docs/superpowers/plans/2026-04-28-monitoring-retention-storage.md go-backend vite-frontend`.
|
||||
|
||||
Expected: Diff is limited to retention config, storage summary, tests, and config UI.
|
||||
|
||||
---
|
||||
|
||||
## Self-Review
|
||||
|
||||
- Spec coverage: retention config, uniform cleanup, storage summary API, frontend display, validation, and verification are covered.
|
||||
- Placeholder scan: no TBD/TODO placeholders; the one contract-test step describes exact assertions even though the surrounding helper functions already exist in contract tests.
|
||||
- Type consistency: backend JSON fields match frontend `StorageSummaryApiData` exactly: `dbType`, `databaseSizeBytes`, `databaseSizeText`.
|
||||
@@ -0,0 +1,74 @@
|
||||
# Monitoring Retention And Storage Display Design
|
||||
|
||||
## Goal
|
||||
|
||||
Add an administrator-facing configuration for monitoring data retention and display the current database storage usage in the configuration page.
|
||||
|
||||
## Scope
|
||||
|
||||
- Add a single config key: `monitor_retention_days`.
|
||||
- Default retention is `7` days.
|
||||
- Apply the retention window uniformly to:
|
||||
- `node_metric`
|
||||
- `tunnel_metric`
|
||||
- `service_monitor_result`
|
||||
- `tunnel_quality`
|
||||
- Show database usage on the config page as a read-only operational value.
|
||||
|
||||
## Non-Goals
|
||||
|
||||
- No per-table retention settings.
|
||||
- No manual purge button.
|
||||
- No database vacuum/compaction action.
|
||||
- No frontend test framework changes.
|
||||
|
||||
## Backend Design
|
||||
|
||||
### Retention Config
|
||||
|
||||
- Store `monitor_retention_days` in `vite_config`, consistent with existing site settings.
|
||||
- Accept integer values from `1` through `3650`.
|
||||
- Missing or invalid stored values fall back to `7` days.
|
||||
- `normalizeAndValidateConfigValue` rejects invalid user-submitted values so bad config does not get saved through the API.
|
||||
|
||||
### Cleanup Flow
|
||||
|
||||
- `metrics.IngestionService.pruneMetrics()` reads `monitor_retention_days` from the repository each hourly cleanup cycle.
|
||||
- The computed cutoff is used for `node_metric`, `tunnel_metric`, and `service_monitor_result`.
|
||||
- `tunnel_quality` uses the same retention config.
|
||||
- `tunnel_quality` cleanup must run even when real-time tunnel quality probing is disabled; disabling probing should stop new probe writes, not stop cleanup.
|
||||
|
||||
### Database Storage API
|
||||
|
||||
- Add an admin-only API endpoint for storage summary, for example `/api/v1/system/storage`.
|
||||
- Response fields:
|
||||
- `dbType`: `sqlite` or `postgres`
|
||||
- `databaseSizeBytes`: raw byte count
|
||||
- `databaseSizeText`: human-readable formatted size
|
||||
- SQLite implementation reports the DB file size and includes `-wal` and `-shm` sidecar files when present.
|
||||
- PostgreSQL implementation uses `pg_database_size(current_database())`.
|
||||
- If size cannot be determined, return an API error rather than a misleading zero.
|
||||
|
||||
## Frontend Design
|
||||
|
||||
- Add `monitor_retention_days` to the config page.
|
||||
- Label: `监控数据保留天数`.
|
||||
- Description: `统一清理节点指标、隧道流量、服务监控结果和隧道质量历史;默认 7 天。`
|
||||
- Use a regular numeric input through the existing config rendering path.
|
||||
- Fetch database storage summary when the config page loads.
|
||||
- Display a read-only card/row named `数据库占用` with `databaseSizeText`.
|
||||
- If fetching fails, show `获取失败` and keep config editing usable.
|
||||
|
||||
## Error Handling
|
||||
|
||||
- Invalid retention values return a validation error on save.
|
||||
- Cleanup logs individual prune failures and continues with other tables, matching existing monitoring cleanup behavior.
|
||||
- Storage summary failures are non-blocking in the frontend.
|
||||
|
||||
## Testing
|
||||
|
||||
- Backend unit tests for retention config parsing and validation.
|
||||
- Backend tests proving custom retention is used by monitoring cleanup.
|
||||
- Backend API/repository test for SQLite storage size returning a non-negative byte count and formatted text.
|
||||
- Run `go test ./...` in `go-backend`.
|
||||
- Run `pnpm run build` in `vite-frontend`.
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"go-backend/internal/http/response"
|
||||
"go-backend/internal/license"
|
||||
"go-backend/internal/metrics"
|
||||
"go-backend/internal/monitoring"
|
||||
"go-backend/internal/security"
|
||||
"go-backend/internal/store/repo"
|
||||
"go-backend/internal/ws"
|
||||
@@ -152,6 +153,7 @@ func (h *Handler) Register(mux *http.ServeMux) {
|
||||
mux.HandleFunc("/api/v1/config/list", h.getConfigs)
|
||||
mux.HandleFunc("/api/v1/config/update", h.updateConfigs)
|
||||
mux.HandleFunc("/api/v1/config/update-single", h.updateSingleConfig)
|
||||
mux.HandleFunc("/api/v1/system/storage", h.storageSummary)
|
||||
mux.HandleFunc("/api/v1/license/activate", h.licenseActivate)
|
||||
mux.HandleFunc("/api/v1/backup/export", h.backupExport)
|
||||
mux.HandleFunc("/api/v1/backup/import", h.backupImport)
|
||||
@@ -1036,6 +1038,8 @@ func normalizeAndValidateConfigValue(key, value string) (string, error) {
|
||||
default:
|
||||
return "", fmt.Errorf("隧道质量检测开关配置值无效")
|
||||
}
|
||||
case monitoring.ConfigMonitorRetentionDays:
|
||||
return monitoring.NormalizeMonitoringRetentionDays(value)
|
||||
default:
|
||||
return value, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"go-backend/internal/http/response"
|
||||
)
|
||||
|
||||
func (h *Handler) storageSummary(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet && r.Method != http.MethodPost {
|
||||
response.WriteJSON(w, response.ErrDefault("请求失败"))
|
||||
return
|
||||
}
|
||||
if h == nil || h.repo == nil {
|
||||
response.WriteJSON(w, response.Err(-2, "repository not initialized"))
|
||||
return
|
||||
}
|
||||
|
||||
summary, err := h.repo.DatabaseStorageSummary()
|
||||
if err != nil {
|
||||
response.WriteJSON(w, response.Err(-2, err.Error()))
|
||||
return
|
||||
}
|
||||
response.WriteJSON(w, response.OK(summary))
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"go-backend/internal/monitoring"
|
||||
"go-backend/internal/store/model"
|
||||
)
|
||||
|
||||
@@ -15,7 +16,6 @@ const (
|
||||
tunnelQualityProbeInterval = 1 * time.Second
|
||||
tunnelQualityProbeTimeout = 8 * time.Second
|
||||
tunnelQualityPingTimeoutMs = 5000
|
||||
tunnelQualityRetention = 24 * time.Hour // keep 24h of history
|
||||
tunnelQualityPruneInterval = 10 * time.Minute
|
||||
tunnelQualityReportInterval = 30 * time.Second // DB save interval
|
||||
)
|
||||
@@ -128,12 +128,19 @@ func (p *tunnelQualityProber) isEnabled() bool {
|
||||
return p.handler.isTunnelQualityMonitoringEnabled()
|
||||
}
|
||||
|
||||
func (p *tunnelQualityProber) retentionDays() int {
|
||||
if p == nil || p.handler == nil || p.handler.repo == nil {
|
||||
return monitoring.DefaultMonitorRetentionDays
|
||||
}
|
||||
cfg, err := p.handler.repo.GetConfigsByNames([]string{monitoring.ConfigMonitorRetentionDays})
|
||||
if err != nil {
|
||||
return monitoring.DefaultMonitorRetentionDays
|
||||
}
|
||||
return monitoring.MonitoringRetentionDaysFromConfigMap(cfg)
|
||||
}
|
||||
|
||||
// maybePrune deletes old quality rows periodically (mirrors PruneServiceMonitorResults).
|
||||
func (p *tunnelQualityProber) maybePrune() {
|
||||
if !p.isEnabled() {
|
||||
return
|
||||
}
|
||||
|
||||
now := time.Now().UnixMilli()
|
||||
if p.lastPrune > 0 && now-p.lastPrune < int64(tunnelQualityPruneInterval/time.Millisecond) {
|
||||
return
|
||||
@@ -145,7 +152,7 @@ func (p *tunnelQualityProber) maybePrune() {
|
||||
return
|
||||
}
|
||||
|
||||
cutoff := now - int64(tunnelQualityRetention/time.Millisecond)
|
||||
cutoff := now - int64(time.Duration(p.retentionDays())*24*time.Hour/time.Millisecond)
|
||||
if err := h.repo.PruneTunnelQualityResults(cutoff); err != nil {
|
||||
log.Printf("tunnel_quality_prober: prune err=%v", err)
|
||||
}
|
||||
@@ -287,7 +294,7 @@ func (p *tunnelQualityProber) probeTunnel(tunnelID int64) {
|
||||
hops = append(hops, hop)
|
||||
break
|
||||
}
|
||||
|
||||
|
||||
fromNode, _ := h.getNodeRecord(source.NodeID)
|
||||
targetIP, targetPort, resolveErr := resolveChainProbeTarget(fromNode, targetNode, target.Port, ipPreference, target.ConnectIP)
|
||||
if resolveErr != nil {
|
||||
|
||||
@@ -105,6 +105,10 @@ func requiresAdmin(path string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
if strings.HasPrefix(path, "/api/v1/system/") {
|
||||
return true
|
||||
}
|
||||
|
||||
if strings.HasPrefix(path, "/api/v1/group/") {
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go-backend/internal/monitoring"
|
||||
"go-backend/internal/store/model"
|
||||
"go-backend/internal/store/repo"
|
||||
)
|
||||
@@ -31,7 +32,6 @@ type IngestionService struct {
|
||||
nodeBuffer []*model.NodeMetric
|
||||
nodeBufferMu sync.Mutex
|
||||
flushInterval time.Duration
|
||||
retentionDays int
|
||||
}
|
||||
|
||||
func NewIngestionService(repo *repo.Repository) *IngestionService {
|
||||
@@ -39,7 +39,6 @@ func NewIngestionService(repo *repo.Repository) *IngestionService {
|
||||
repo: repo,
|
||||
nodeBuffer: make([]*model.NodeMetric, 0, 500),
|
||||
flushInterval: 30 * time.Second,
|
||||
retentionDays: 7,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -111,7 +110,22 @@ func (s *IngestionService) flushNodeMetrics() {
|
||||
}
|
||||
|
||||
func (s *IngestionService) pruneMetrics() {
|
||||
cutoff := time.Now().Add(-time.Duration(s.retentionDays) * 24 * time.Hour).UnixMilli()
|
||||
s.pruneMetricsAt(time.Now())
|
||||
}
|
||||
|
||||
func (s *IngestionService) retentionDaysFromConfig() int {
|
||||
if s == nil || s.repo == nil {
|
||||
return monitoring.DefaultMonitorRetentionDays
|
||||
}
|
||||
cfg, err := s.repo.GetConfigsByNames([]string{monitoring.ConfigMonitorRetentionDays})
|
||||
if err != nil {
|
||||
return monitoring.DefaultMonitorRetentionDays
|
||||
}
|
||||
return monitoring.MonitoringRetentionDaysFromConfigMap(cfg)
|
||||
}
|
||||
|
||||
func (s *IngestionService) pruneMetricsAt(now time.Time) {
|
||||
cutoff := now.Add(-time.Duration(s.retentionDaysFromConfig()) * 24 * time.Hour).UnixMilli()
|
||||
if s.repo == nil {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"go-backend/internal/store/model"
|
||||
"go-backend/internal/store/repo"
|
||||
)
|
||||
|
||||
@@ -215,7 +216,6 @@ func TestPruneMetrics(t *testing.T) {
|
||||
defer r.Close()
|
||||
|
||||
svc := NewIngestionService(r)
|
||||
svc.retentionDays = 1
|
||||
|
||||
info := SystemInfo{CPUUsage: 50.0, MemoryUsage: 60.0, DiskUsage: 30.0}
|
||||
|
||||
@@ -233,6 +233,39 @@ func TestPruneMetrics(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPruneMetricsUsesConfiguredRetentionDays(t *testing.T) {
|
||||
r, err := repo.Open(":memory:")
|
||||
if err != nil {
|
||||
t.Fatalf("open repo: %v", err)
|
||||
}
|
||||
defer r.Close()
|
||||
|
||||
now := time.Now().UnixMilli()
|
||||
if err := r.UpsertConfig("monitor_retention_days", "2", now); err != nil {
|
||||
t.Fatalf("upsert retention config: %v", err)
|
||||
}
|
||||
|
||||
oldMetric := &model.NodeMetric{NodeID: 1, Timestamp: now - int64(3*24*time.Hour/time.Millisecond), CPUUsage: 10}
|
||||
newMetric := &model.NodeMetric{NodeID: 1, Timestamp: now - int64(1*24*time.Hour/time.Millisecond), CPUUsage: 20}
|
||||
if err := r.InsertNodeMetric(oldMetric); err != nil {
|
||||
t.Fatalf("insert old metric: %v", err)
|
||||
}
|
||||
if err := r.InsertNodeMetric(newMetric); err != nil {
|
||||
t.Fatalf("insert new metric: %v", err)
|
||||
}
|
||||
|
||||
svc := NewIngestionService(r)
|
||||
svc.pruneMetricsAt(time.UnixMilli(now))
|
||||
|
||||
metrics, err := r.GetNodeMetrics(1, now-int64(4*24*time.Hour/time.Millisecond), now+1000)
|
||||
if err != nil {
|
||||
t.Fatalf("get node metrics: %v", err)
|
||||
}
|
||||
if len(metrics) != 1 || metrics[0].CPUUsage != 20 {
|
||||
t.Fatalf("expected only newer metric to remain, got %#v", metrics)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipleNodes(t *testing.T) {
|
||||
r, err := repo.Open(":memory:")
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
package monitoring
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
ConfigMonitorRetentionDays = "monitor_retention_days"
|
||||
DefaultMonitorRetentionDays = 7
|
||||
MinMonitorRetentionDays = 1
|
||||
MaxMonitorRetentionDays = 3650
|
||||
)
|
||||
|
||||
func MonitoringRetentionDaysFromConfigMap(cfg map[string]string) int {
|
||||
if cfg == nil {
|
||||
return DefaultMonitorRetentionDays
|
||||
}
|
||||
days, err := parseMonitoringRetentionDays(cfg[ConfigMonitorRetentionDays])
|
||||
if err != nil {
|
||||
return DefaultMonitorRetentionDays
|
||||
}
|
||||
return days
|
||||
}
|
||||
|
||||
func NormalizeMonitoringRetentionDays(value string) (string, error) {
|
||||
days, err := parseMonitoringRetentionDays(value)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return strconv.Itoa(days), nil
|
||||
}
|
||||
|
||||
func parseMonitoringRetentionDays(value string) (int, error) {
|
||||
trimmed := strings.TrimSpace(value)
|
||||
if trimmed == "" {
|
||||
return 0, fmt.Errorf("监控数据保留天数不能为空")
|
||||
}
|
||||
days, err := strconv.Atoi(trimmed)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("监控数据保留天数必须是整数")
|
||||
}
|
||||
if days < MinMonitorRetentionDays || days > MaxMonitorRetentionDays {
|
||||
return 0, fmt.Errorf("监控数据保留天数必须在 %d 到 %d 之间", MinMonitorRetentionDays, MaxMonitorRetentionDays)
|
||||
}
|
||||
return days, nil
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package monitoring
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestMonitoringRetentionDaysFromConfigMap(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
cfg map[string]string
|
||||
want int
|
||||
}{
|
||||
{"missing uses default", nil, 7},
|
||||
{"valid custom", map[string]string{ConfigMonitorRetentionDays: "3"}, 3},
|
||||
{"trimmed custom", map[string]string{ConfigMonitorRetentionDays: " 30 "}, 30},
|
||||
{"invalid uses default", map[string]string{ConfigMonitorRetentionDays: "abc"}, 7},
|
||||
{"too small uses default", map[string]string{ConfigMonitorRetentionDays: "0"}, 7},
|
||||
{"too large uses default", map[string]string{ConfigMonitorRetentionDays: "3651"}, 7},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := MonitoringRetentionDaysFromConfigMap(tc.cfg); got != tc.want {
|
||||
t.Fatalf("expected %d, got %d", tc.want, got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeMonitoringRetentionDays(t *testing.T) {
|
||||
for _, value := range []string{"1", "7", "3650", " 30 "} {
|
||||
if got, err := NormalizeMonitoringRetentionDays(value); err != nil || got == "" {
|
||||
t.Fatalf("expected %q valid, got value=%q err=%v", value, got, err)
|
||||
}
|
||||
}
|
||||
|
||||
for _, value := range []string{"", "0", "-1", "3651", "abc", "1.5"} {
|
||||
if got, err := NormalizeMonitoringRetentionDays(value); err == nil {
|
||||
t.Fatalf("expected %q invalid, got value=%q", value, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -65,7 +65,8 @@ type TunnelQuality = model.TunnelQuality
|
||||
// ─── Repository ──────────────────────────────────────────────────────
|
||||
|
||||
type Repository struct {
|
||||
db *gorm.DB
|
||||
db *gorm.DB
|
||||
dbPath string
|
||||
}
|
||||
|
||||
type FlowUploadCounterDelta struct {
|
||||
@@ -198,7 +199,7 @@ func Open(path string) (*Repository, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &Repository{db: db}, nil
|
||||
return &Repository{db: db, dbPath: path}, nil
|
||||
}
|
||||
|
||||
func OpenPostgres(dsn string) (*Repository, error) {
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
package repo
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
type DatabaseStorageSummary struct {
|
||||
DBType string `json:"dbType"`
|
||||
DatabaseSizeBytes int64 `json:"databaseSizeBytes"`
|
||||
DatabaseSizeText string `json:"databaseSizeText"`
|
||||
}
|
||||
|
||||
func (r *Repository) DatabaseStorageSummary() (DatabaseStorageSummary, error) {
|
||||
if r == nil || r.db == nil {
|
||||
return DatabaseStorageSummary{}, errors.New("repository not initialized")
|
||||
}
|
||||
|
||||
switch r.db.Dialector.Name() {
|
||||
case "sqlite":
|
||||
size, err := sqliteDatabaseFileSize(r.dbPath)
|
||||
if err != nil {
|
||||
return DatabaseStorageSummary{}, err
|
||||
}
|
||||
return DatabaseStorageSummary{DBType: "sqlite", DatabaseSizeBytes: size, DatabaseSizeText: formatDatabaseSize(size)}, nil
|
||||
case "postgres":
|
||||
var size int64
|
||||
if err := r.db.Raw("SELECT pg_database_size(current_database())").Scan(&size).Error; err != nil {
|
||||
return DatabaseStorageSummary{}, err
|
||||
}
|
||||
return DatabaseStorageSummary{DBType: "postgres", DatabaseSizeBytes: size, DatabaseSizeText: formatDatabaseSize(size)}, nil
|
||||
default:
|
||||
return DatabaseStorageSummary{}, fmt.Errorf("unsupported database dialect %q", r.db.Dialector.Name())
|
||||
}
|
||||
}
|
||||
|
||||
func sqliteDatabaseFileSize(path string) (int64, error) {
|
||||
if path == "" || path == ":memory:" {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
var total int64
|
||||
for _, candidate := range []string{path, path + "-wal", path + "-shm"} {
|
||||
info, err := os.Stat(candidate)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
continue
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
if !info.IsDir() {
|
||||
total += info.Size()
|
||||
}
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
func formatDatabaseSize(bytes int64) string {
|
||||
if bytes < 1024 {
|
||||
return fmt.Sprintf("%d B", bytes)
|
||||
}
|
||||
units := []string{"KB", "MB", "GB", "TB"}
|
||||
value := float64(bytes) / 1024
|
||||
for _, unit := range units {
|
||||
if value < 1024 || unit == "TB" {
|
||||
return fmt.Sprintf("%.1f %s", value, unit)
|
||||
}
|
||||
value /= 1024
|
||||
}
|
||||
return fmt.Sprintf("%d B", bytes)
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package repo
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"go-backend/internal/store/model"
|
||||
)
|
||||
|
||||
func TestDatabaseStorageSummarySQLiteIncludesSize(t *testing.T) {
|
||||
dbPath := filepath.Join(t.TempDir(), "storage.db")
|
||||
r, err := Open(dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("open repo: %v", err)
|
||||
}
|
||||
defer r.Close()
|
||||
|
||||
if err := r.InsertNodeMetric(&model.NodeMetric{NodeID: 1, Timestamp: 123, CPUUsage: 1}); err != nil {
|
||||
t.Fatalf("insert metric: %v", err)
|
||||
}
|
||||
|
||||
summary, err := r.DatabaseStorageSummary()
|
||||
if err != nil {
|
||||
t.Fatalf("storage summary: %v", err)
|
||||
}
|
||||
if summary.DBType != "sqlite" {
|
||||
t.Fatalf("expected sqlite db type, got %q", summary.DBType)
|
||||
}
|
||||
if summary.DatabaseSizeBytes <= 0 {
|
||||
t.Fatalf("expected database size > 0, got %d", summary.DatabaseSizeBytes)
|
||||
}
|
||||
if summary.DatabaseSizeText == "" {
|
||||
t.Fatalf("expected formatted size")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatDatabaseSize(t *testing.T) {
|
||||
tests := []struct {
|
||||
bytes int64
|
||||
want string
|
||||
}{
|
||||
{bytes: 0, want: "0 B"},
|
||||
{bytes: 512, want: "512 B"},
|
||||
{bytes: 1024, want: "1.0 KB"},
|
||||
{bytes: 1024 * 1024, want: "1.0 MB"},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
if got := formatDatabaseSize(tc.bytes); got != tc.want {
|
||||
t.Fatalf("formatDatabaseSize(%d) = %q, want %q", tc.bytes, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package contract_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"go-backend/internal/auth"
|
||||
"go-backend/internal/http/response"
|
||||
)
|
||||
|
||||
func TestStorageSummaryRequiresAdminAndReturnsSize(t *testing.T) {
|
||||
secret := "storage-contract-secret"
|
||||
router, _ := setupContractRouter(t, secret)
|
||||
|
||||
adminToken, err := auth.GenerateToken(1, "admin_user", 0, secret)
|
||||
if err != nil {
|
||||
t.Fatalf("generate admin token: %v", err)
|
||||
}
|
||||
userToken, err := auth.GenerateToken(2, "normal_user", 1, secret)
|
||||
if err != nil {
|
||||
t.Fatalf("generate user token: %v", err)
|
||||
}
|
||||
|
||||
userReq := httptest.NewRequest(http.MethodGet, "/api/v1/system/storage", nil)
|
||||
userReq.Header.Set("Authorization", userToken)
|
||||
userRes := httptest.NewRecorder()
|
||||
router.ServeHTTP(userRes, userReq)
|
||||
|
||||
var denied response.R
|
||||
if err := json.NewDecoder(userRes.Body).Decode(&denied); err != nil {
|
||||
t.Fatalf("decode denied response: %v", err)
|
||||
}
|
||||
if denied.Code != 403 {
|
||||
t.Fatalf("expected 403 for non-admin, got %d", denied.Code)
|
||||
}
|
||||
|
||||
adminReq := httptest.NewRequest(http.MethodGet, "/api/v1/system/storage", nil)
|
||||
adminReq.Header.Set("Authorization", adminToken)
|
||||
adminRes := httptest.NewRecorder()
|
||||
router.ServeHTTP(adminRes, adminReq)
|
||||
|
||||
var out response.R
|
||||
if err := json.NewDecoder(adminRes.Body).Decode(&out); err != nil {
|
||||
t.Fatalf("decode admin response: %v", err)
|
||||
}
|
||||
if out.Code != 0 {
|
||||
t.Fatalf("expected code 0, got %d: %s", out.Code, out.Msg)
|
||||
}
|
||||
data, ok := out.Data.(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatalf("expected object data, got %T", out.Data)
|
||||
}
|
||||
if data["dbType"] == "" {
|
||||
t.Fatalf("expected dbType")
|
||||
}
|
||||
if _, ok := data["databaseSizeBytes"].(float64); !ok {
|
||||
t.Fatalf("expected numeric databaseSizeBytes, got %T", data["databaseSizeBytes"])
|
||||
}
|
||||
if data["databaseSizeText"] == "" {
|
||||
t.Fatalf("expected databaseSizeText")
|
||||
}
|
||||
}
|
||||
@@ -234,6 +234,13 @@ func (p *program) reloadConfig() error {
|
||||
if err := loader.Load(cfg); err != nil {
|
||||
return err
|
||||
}
|
||||
activeServices := make(map[string]struct{}, len(cfg.Services))
|
||||
for _, svc := range cfg.Services {
|
||||
if svc != nil {
|
||||
activeServices[svc.Name] = struct{}{}
|
||||
}
|
||||
}
|
||||
xservice.GetGlobalTrafficManager().RetainServices(activeServices)
|
||||
|
||||
if err := p.run(cfg); err != nil {
|
||||
return err
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"github.com/go-gost/x/config/loader"
|
||||
"github.com/go-gost/x/config/parsing/parser"
|
||||
"github.com/go-gost/x/registry"
|
||||
xservice "github.com/go-gost/x/service"
|
||||
)
|
||||
|
||||
// swagger:parameters reloadConfigRequest
|
||||
@@ -42,6 +43,13 @@ func reloadConfig(ctx *gin.Context) {
|
||||
writeError(ctx, NewError(http.StatusBadRequest, ErrCodeInvalid, err.Error()))
|
||||
return
|
||||
}
|
||||
activeServices := make(map[string]struct{}, len(cfg.Services))
|
||||
for _, svc := range cfg.Services {
|
||||
if svc != nil {
|
||||
activeServices[svc.Name] = struct{}{}
|
||||
}
|
||||
}
|
||||
xservice.GetGlobalTrafficManager().RetainServices(activeServices)
|
||||
|
||||
for _, svc := range registry.ServiceRegistry().GetAll() {
|
||||
svc := svc
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
parser "github.com/go-gost/x/config/parsing/service"
|
||||
kill "github.com/go-gost/x/internal/util/port"
|
||||
"github.com/go-gost/x/registry"
|
||||
xservice "github.com/go-gost/x/service"
|
||||
)
|
||||
|
||||
// swagger:parameters createServiceRequest
|
||||
@@ -409,6 +410,7 @@ func deleteService(ctx *gin.Context) {
|
||||
}
|
||||
return nil
|
||||
})
|
||||
xservice.GetGlobalTrafficManager().RemoveServices(name)
|
||||
|
||||
ctx.JSON(http.StatusOK, Response{
|
||||
Msg: "OK",
|
||||
@@ -484,6 +486,12 @@ func deleteServices(ctx *gin.Context) {
|
||||
return nil
|
||||
})
|
||||
|
||||
names := make([]string, 0, len(servicesToDelete))
|
||||
for _, std := range servicesToDelete {
|
||||
names = append(names, std.name)
|
||||
}
|
||||
xservice.GetGlobalTrafficManager().RemoveServices(names...)
|
||||
|
||||
ctx.JSON(http.StatusOK, Response{
|
||||
Msg: "OK",
|
||||
})
|
||||
|
||||
@@ -5,15 +5,17 @@ import (
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/go-gost/x/registry"
|
||||
)
|
||||
|
||||
// GlobalTrafficManager 全局流量管理器(所有服务共享)
|
||||
type GlobalTrafficManager struct {
|
||||
mu sync.RWMutex
|
||||
mu sync.RWMutex
|
||||
serviceTraffic map[string]*ServiceTraffic // key: 服务名, value: 流量数据
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
reportTicker *time.Ticker
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
reportTicker *time.Ticker
|
||||
}
|
||||
|
||||
// ServiceTraffic 单个服务的流量累积
|
||||
@@ -50,6 +52,9 @@ func (m *GlobalTrafficManager) AddTraffic(serviceName string, upBytes, downBytes
|
||||
if upBytes == 0 && downBytes == 0 {
|
||||
return
|
||||
}
|
||||
if !registry.ServiceRegistry().IsRegistered(serviceName) {
|
||||
return
|
||||
}
|
||||
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
@@ -70,6 +75,51 @@ func (m *GlobalTrafficManager) AddTraffic(serviceName string, upBytes, downBytes
|
||||
traffic.mu.Unlock()
|
||||
}
|
||||
|
||||
// RemoveServices drops cached traffic counters for services that no longer exist.
|
||||
func (m *GlobalTrafficManager) RemoveServices(serviceNames ...string) {
|
||||
if m == nil || len(serviceNames) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
for _, name := range serviceNames {
|
||||
if m.isTrafficEmptyLocked(name) {
|
||||
delete(m.serviceTraffic, name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// RetainServices removes traffic counters for every service not in activeNames.
|
||||
func (m *GlobalTrafficManager) RetainServices(activeNames map[string]struct{}) {
|
||||
if m == nil {
|
||||
return
|
||||
}
|
||||
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
for name := range m.serviceTraffic {
|
||||
if _, ok := activeNames[name]; !ok {
|
||||
if m.isTrafficEmptyLocked(name) {
|
||||
delete(m.serviceTraffic, name)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *GlobalTrafficManager) isTrafficEmptyLocked(name string) bool {
|
||||
traffic, ok := m.serviceTraffic[name]
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
|
||||
traffic.mu.Lock()
|
||||
defer traffic.mu.Unlock()
|
||||
return traffic.UpBytes == 0 && traffic.DownBytes == 0
|
||||
}
|
||||
|
||||
// startReporting 启动定时上报协程(每5秒执行一次)
|
||||
func (m *GlobalTrafficManager) startReporting() {
|
||||
|
||||
@@ -105,6 +155,7 @@ func (m *GlobalTrafficManager) collectAndReport() {
|
||||
traffic.DownBytes = 0
|
||||
}
|
||||
traffic.mu.Unlock()
|
||||
isStale := !registry.ServiceRegistry().IsRegistered(name)
|
||||
|
||||
if up > 0 || down > 0 {
|
||||
reportItems = append(reportItems, TrafficReportItem{
|
||||
@@ -113,6 +164,9 @@ func (m *GlobalTrafficManager) collectAndReport() {
|
||||
D: down,
|
||||
})
|
||||
}
|
||||
if isStale {
|
||||
delete(m.serviceTraffic, name)
|
||||
}
|
||||
}
|
||||
|
||||
m.mu.Unlock()
|
||||
@@ -162,4 +216,3 @@ func (m *GlobalTrafficManager) GetServiceTraffic(serviceName string) (upBytes, d
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestGlobalTrafficManagerRemoveServicesDropsCachedEntries(t *testing.T) {
|
||||
m := &GlobalTrafficManager{serviceTraffic: map[string]*ServiceTraffic{
|
||||
"svc-a": {ServiceName: "svc-a"},
|
||||
"svc-b": {ServiceName: "svc-b"},
|
||||
}}
|
||||
|
||||
m.RemoveServices("svc-a")
|
||||
|
||||
if _, ok := m.serviceTraffic["svc-a"]; ok {
|
||||
t.Fatalf("expected svc-a traffic entry to be removed")
|
||||
}
|
||||
if _, ok := m.serviceTraffic["svc-b"]; !ok {
|
||||
t.Fatalf("expected svc-b traffic entry to remain")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGlobalTrafficManagerRetainServicesDropsStaleEntries(t *testing.T) {
|
||||
m := &GlobalTrafficManager{serviceTraffic: map[string]*ServiceTraffic{
|
||||
"svc-a": {ServiceName: "svc-a"},
|
||||
"svc-b": {ServiceName: "svc-b"},
|
||||
}}
|
||||
|
||||
m.RetainServices(map[string]struct{}{"svc-b": {}})
|
||||
|
||||
if _, ok := m.serviceTraffic["svc-a"]; ok {
|
||||
t.Fatalf("expected stale svc-a traffic entry to be removed")
|
||||
}
|
||||
if _, ok := m.serviceTraffic["svc-b"]; !ok {
|
||||
t.Fatalf("expected active svc-b traffic entry to remain")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGlobalTrafficManagerAddTrafficIgnoresUnregisteredService(t *testing.T) {
|
||||
m := &GlobalTrafficManager{serviceTraffic: make(map[string]*ServiceTraffic)}
|
||||
|
||||
m.AddTraffic("deleted-service", 10, 20)
|
||||
|
||||
if _, ok := m.serviceTraffic["deleted-service"]; ok {
|
||||
t.Fatalf("expected unregistered service traffic to be ignored")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGlobalTrafficManagerCollectAndReportDropsStaleEntriesAfterReporting(t *testing.T) {
|
||||
origReportDo := reportDo
|
||||
origReportURL := httpReportURL
|
||||
origAESCrypto := httpAESCrypto
|
||||
defer func() {
|
||||
reportDo = origReportDo
|
||||
httpReportURL = origReportURL
|
||||
httpAESCrypto = origAESCrypto
|
||||
}()
|
||||
|
||||
httpReportURL = "http://panel.example.com/flow/upload?secret=abc"
|
||||
httpAESCrypto = nil
|
||||
|
||||
var requestBody string
|
||||
reportDo = func(_ context.Context, req *http.Request, _ time.Duration) (*http.Response, error) {
|
||||
body, err := io.ReadAll(req.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("read request body: %v", err)
|
||||
}
|
||||
requestBody = string(body)
|
||||
return &http.Response{
|
||||
StatusCode: http.StatusOK,
|
||||
Body: io.NopCloser(strings.NewReader("ok")),
|
||||
}, nil
|
||||
}
|
||||
|
||||
m := &GlobalTrafficManager{
|
||||
serviceTraffic: map[string]*ServiceTraffic{
|
||||
"stale-service": {ServiceName: "stale-service", UpBytes: 10, DownBytes: 20},
|
||||
},
|
||||
ctx: context.Background(),
|
||||
}
|
||||
|
||||
m.collectAndReport()
|
||||
|
||||
if !strings.Contains(requestBody, "stale-service") {
|
||||
t.Fatalf("expected pending stale traffic to be reported first, body=%s", requestBody)
|
||||
}
|
||||
if _, ok := m.serviceTraffic["stale-service"]; ok {
|
||||
t.Fatalf("expected stale traffic entry to be removed after report collection")
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
parser "github.com/go-gost/x/config/parsing/service"
|
||||
kill "github.com/go-gost/x/internal/util/port"
|
||||
"github.com/go-gost/x/registry"
|
||||
xservice "github.com/go-gost/x/service"
|
||||
)
|
||||
|
||||
func createServices(req createServicesRequest) error {
|
||||
@@ -209,6 +210,7 @@ func deleteServices(req deleteServicesRequest) error {
|
||||
}
|
||||
return nil
|
||||
})
|
||||
xservice.GetGlobalTrafficManager().RemoveServices(namesToRemove...)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -41,6 +41,7 @@ import type {
|
||||
MonitorPermissionApiItem,
|
||||
MonitorAccessApiData,
|
||||
TunnelQualityApiItem,
|
||||
StorageSummaryApiData,
|
||||
} from "./types";
|
||||
|
||||
import axios from "axios";
|
||||
@@ -254,6 +255,9 @@ export const updateConfigs = (configMap: Record<string, string>) =>
|
||||
export const updateConfig = (name: string, value: string) =>
|
||||
Network.post("/config/update-single", { name, value });
|
||||
|
||||
export const getStorageSummary = () =>
|
||||
Network.get<StorageSummaryApiData>("/system/storage");
|
||||
|
||||
export const activateLicense = (licenseKey: string) =>
|
||||
Network.post("/license/activate", { license_key: licenseKey });
|
||||
|
||||
|
||||
@@ -474,6 +474,12 @@ export interface ServiceMonitorLimitsApiData {
|
||||
maxTimeoutSec: number;
|
||||
}
|
||||
|
||||
export interface StorageSummaryApiData {
|
||||
dbType: string;
|
||||
databaseSizeBytes: number;
|
||||
databaseSizeText: string;
|
||||
}
|
||||
|
||||
export interface MonitorNodeApiItem {
|
||||
id: number;
|
||||
inx: number;
|
||||
|
||||
@@ -26,6 +26,7 @@ import {
|
||||
importBackup,
|
||||
getAnnouncement,
|
||||
updateAnnouncement,
|
||||
getStorageSummary,
|
||||
type AnnouncementData,
|
||||
} from "@/api";
|
||||
import { BackIcon, SettingsIcon } from "@/components/icons";
|
||||
@@ -146,6 +147,14 @@ const CONFIG_ITEMS: ConfigItem[] = [
|
||||
"关闭后,前端停止自动刷新,后端停止实时隧道质量探测(全局配置)",
|
||||
type: "switch",
|
||||
},
|
||||
{
|
||||
key: "monitor_retention_days",
|
||||
label: "监控数据保留天数",
|
||||
placeholder: "7",
|
||||
description:
|
||||
"统一清理节点指标、隧道流量、服务监控结果和隧道质量历史;默认 7 天。",
|
||||
type: "input",
|
||||
},
|
||||
{
|
||||
key: "captcha_enabled",
|
||||
label: "启用验证码",
|
||||
@@ -220,6 +229,7 @@ const getInitialConfigs = (): Record<string, string> => {
|
||||
"cloudflare_secret_key",
|
||||
"forward_compact_mode",
|
||||
"monitor_tunnel_quality_enabled",
|
||||
"monitor_retention_days",
|
||||
"ip",
|
||||
"panel_domain",
|
||||
"app_logo",
|
||||
@@ -288,6 +298,7 @@ export default function ConfigPage() {
|
||||
const [brandUploading, setBrandUploading] = useState<
|
||||
Partial<Record<BrandPreviewKey, boolean>>
|
||||
>({});
|
||||
const [storageSummary, setStorageSummary] = useState("加载中...");
|
||||
|
||||
const canGoBack =
|
||||
typeof window !== "undefined" &&
|
||||
@@ -347,10 +358,26 @@ export default function ConfigPage() {
|
||||
}
|
||||
};
|
||||
|
||||
const loadStorageSummary = async () => {
|
||||
try {
|
||||
const response = await getStorageSummary();
|
||||
|
||||
if (response.code === 0 && response.data?.databaseSizeText) {
|
||||
setStorageSummary(response.data.databaseSizeText);
|
||||
|
||||
return;
|
||||
}
|
||||
setStorageSummary("获取失败");
|
||||
} catch {
|
||||
setStorageSummary("获取失败");
|
||||
}
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
const timer = setTimeout(() => {
|
||||
loadConfigs(initialConfigs);
|
||||
loadAnnouncement();
|
||||
loadStorageSummary();
|
||||
}, 100);
|
||||
|
||||
return () => clearTimeout(timer);
|
||||
@@ -1324,6 +1351,22 @@ export default function ConfigPage() {
|
||||
</Select>
|
||||
</div>
|
||||
|
||||
<Divider className="my-2" />
|
||||
|
||||
<div className="space-y-3">
|
||||
<div className="flex flex-col gap-1">
|
||||
<p className="text-sm font-medium text-gray-700 dark:text-gray-300">
|
||||
数据库占用
|
||||
</p>
|
||||
<p className="text-xs text-gray-500 dark:text-gray-400">
|
||||
当前后端数据库文件或实例占用空间,仅用于容量参考。
|
||||
</p>
|
||||
</div>
|
||||
<div className="rounded-lg border border-divider bg-default-50/60 dark:bg-default-100/10 px-4 py-3 text-sm font-semibold text-default-800 dark:text-default-200">
|
||||
{storageSummary}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="flex justify-end pt-6 border-t border-divider/50 mt-4">
|
||||
<Button
|
||||
color="primary"
|
||||
|
||||
Reference in New Issue
Block a user