mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-03 07:06:36 +08:00
refactor(backend): rename OpenFlare directory to lowercase openflare
This commit is contained in:
@@ -0,0 +1,45 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Package apiutil provides HTTP helpers for OpenFlare v1 custom API handlers.
|
||||
package apiutil
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
|
||||
"Wavelet/pkg/response"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
const errInvalidParams = "参数错误"
|
||||
const errInvalidID = "无效的 ID"
|
||||
|
||||
// BindJSON binds JSON body; returns false after aborting with 400.
|
||||
func BindJSON(c *gin.Context, dst any) bool {
|
||||
if err := c.ShouldBindJSON(dst); err != nil {
|
||||
response.AbortBadRequest(c, errInvalidParams)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// IDParam parses :id from the URL path.
|
||||
func IDParam(c *gin.Context) (uint, bool) {
|
||||
return NamedIDParam(c, "id")
|
||||
}
|
||||
|
||||
// NamedIDParam parses a named path parameter as a positive uint ID.
|
||||
func NamedIDParam(c *gin.Context, name string) (uint, bool) {
|
||||
raw := c.Param(name)
|
||||
if raw == "" {
|
||||
response.AbortBadRequest(c, errInvalidID)
|
||||
return 0, false
|
||||
}
|
||||
id64, err := strconv.ParseUint(raw, 10, 64)
|
||||
if err != nil || id64 == 0 {
|
||||
response.AbortBadRequest(c, errInvalidID)
|
||||
return 0, false
|
||||
}
|
||||
return uint(id64), true
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package apiutil
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"Wavelet/pkg/response"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// AbortNotFoundIfMissing maps gorm.ErrRecordNotFound to 404; other errors to 400.
|
||||
func AbortNotFoundIfMissing(c *gin.Context, err error, notFoundMsg string) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
response.AbortNotFound(c, notFoundMsg)
|
||||
return true
|
||||
}
|
||||
response.AbortBadRequest(c, err.Error())
|
||||
return true
|
||||
}
|
||||
|
||||
// AbortBadRequestOnError writes a 400 for any non-nil error.
|
||||
func AbortBadRequestOnError(c *gin.Context, err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
response.AbortBadRequest(c, err.Error())
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package apiutil
|
||||
|
||||
import "Wavelet/core/contracts"
|
||||
|
||||
// AdminMiddlewares returns Wavelet-standard middlewares for OpenFlare console routes.
|
||||
// OpenFlare no longer distinguishes Admin vs Root tiers; all management endpoints share
|
||||
// the same gate: RequireAuth + RequireAdmin from the platform AuthService.
|
||||
//
|
||||
// 返回 []any 而非 []gin.HandlerFunc:内核 RouterExtension.Use 收 ...any,
|
||||
// 而 Go 不允许把 []T 直接展开成 ...any。
|
||||
func AdminMiddlewares(auth contracts.AuthService) []any {
|
||||
return []any{auth.RequireAuthMiddleware(), auth.RequireAdminMiddleware()}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package apiutil
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/testhelper"
|
||||
"Wavelet/pkg/response"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestAdminMiddlewaresRunsAuthThenAdmin(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
var order []string
|
||||
auth := sequentialAuth{
|
||||
auth: func(c *gin.Context) {
|
||||
order = append(order, "auth")
|
||||
c.Next()
|
||||
},
|
||||
admin: func(c *gin.Context) {
|
||||
order = append(order, "admin")
|
||||
c.Next()
|
||||
},
|
||||
}
|
||||
|
||||
engine := testhelper.NewTestGinEngine()
|
||||
group := engine.Group("/protected", ginHandlerMiddlewares(AdminMiddlewares(auth)...)...)
|
||||
group.GET("", func(c *gin.Context) {
|
||||
c.JSON(http.StatusOK, response.OKNil())
|
||||
})
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/protected", nil)
|
||||
engine.ServeHTTP(rec, req)
|
||||
|
||||
assert.Equal(t, http.StatusOK, rec.Code)
|
||||
assert.Equal(t, []string{"auth", "admin"}, order)
|
||||
}
|
||||
|
||||
func ginHandlerMiddlewares(middlewares ...any) []gin.HandlerFunc {
|
||||
handlers := make([]gin.HandlerFunc, 0, len(middlewares))
|
||||
for _, m := range middlewares {
|
||||
switch h := m.(type) {
|
||||
case gin.HandlerFunc:
|
||||
handlers = append(handlers, h)
|
||||
case func(*gin.Context):
|
||||
handlers = append(handlers, gin.HandlerFunc(h))
|
||||
default:
|
||||
panic("unexpected middleware type")
|
||||
}
|
||||
}
|
||||
return handlers
|
||||
}
|
||||
|
||||
type sequentialAuth struct {
|
||||
testhelper.StubAuth
|
||||
auth, admin gin.HandlerFunc
|
||||
}
|
||||
|
||||
func (s sequentialAuth) RequireAuthMiddleware() any { return s.auth }
|
||||
func (s sequentialAuth) RequireAdminMiddleware() any { return s.admin }
|
||||
@@ -0,0 +1,26 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package apiutil
|
||||
|
||||
import (
|
||||
"Wavelet/core"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
// RegisterCollection registers a collection endpoint on both "" and "/" so requests
|
||||
// work with or without a trailing slash.
|
||||
//
|
||||
// 尾部斜杠变体必须用 HandleRaw 注册:RouterExtension.Handle 会经 cleanPath 归一化
|
||||
// 掉尾部斜杠,而部署关闭了 gin 的 RedirectTrailingSlash,缺一条即 404。
|
||||
func RegisterCollection(route core.RouterExtension, method string, handlers ...gin.HandlerFunc) {
|
||||
hs := make([]any, len(handlers))
|
||||
for i, h := range handlers {
|
||||
hs[i] = h
|
||||
}
|
||||
route.Handle(method, "", hs...)
|
||||
if route.BasePath() != "" {
|
||||
route.HandleRaw(method, "/", hs...)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Package credential seals and opens OpenFlare integration credentials.
|
||||
package credential
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"strings"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/runtimeconfig"
|
||||
"Wavelet/pkg/util"
|
||||
)
|
||||
|
||||
// Prefix identifies values encrypted with the current credential format.
|
||||
const Prefix = "enc:v1:"
|
||||
|
||||
var sessionSecret string
|
||||
|
||||
// SetSessionSecret binds the host session secret used to seal credentials.
|
||||
func SetSessionSecret(secret string) {
|
||||
sessionSecret = strings.TrimSpace(secret)
|
||||
}
|
||||
|
||||
func encryptionKey() string {
|
||||
secret := strings.TrimSpace(sessionSecret)
|
||||
if secret == "" {
|
||||
secret = strings.TrimSpace(runtimeconfig.SessionSecret())
|
||||
}
|
||||
if secret == "" {
|
||||
return ""
|
||||
}
|
||||
sum := sha256.Sum256([]byte(secret))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// Seal trims and encrypts plaintext when a session secret is configured.
|
||||
// Plaintext storage is preserved for installations without a session secret.
|
||||
func Seal(plaintext string) (string, error) {
|
||||
plaintext = strings.TrimSpace(plaintext)
|
||||
if plaintext == "" {
|
||||
return "", nil
|
||||
}
|
||||
key := encryptionKey()
|
||||
if key == "" {
|
||||
return plaintext, nil
|
||||
}
|
||||
encrypted, err := util.Encrypt(key, plaintext)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return Prefix + encrypted, nil
|
||||
}
|
||||
|
||||
// Open decrypts a sealed value and accepts legacy plaintext values.
|
||||
func Open(stored string) (string, error) {
|
||||
stored = strings.TrimSpace(stored)
|
||||
if stored == "" {
|
||||
return "", nil
|
||||
}
|
||||
if !strings.HasPrefix(stored, Prefix) {
|
||||
return stored, nil
|
||||
}
|
||||
key := encryptionKey()
|
||||
if key == "" {
|
||||
return "", errors.New("cannot decrypt sensitive field without session secret")
|
||||
}
|
||||
return util.Decrypt(key, strings.TrimPrefix(stored, Prefix))
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package credential
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/runtimeconfig"
|
||||
)
|
||||
|
||||
func TestSealAndOpenSensitiveValue(t *testing.T) {
|
||||
previous := runtimeconfig.Get()
|
||||
SetSessionSecret("cloudflare-pointing-test-secret")
|
||||
t.Cleanup(func() {
|
||||
SetSessionSecret(previous.SessionSecret)
|
||||
runtimeconfig.Set(previous)
|
||||
})
|
||||
|
||||
sealed, err := Seal(`{"api_token":"secret-token"}`)
|
||||
if err != nil {
|
||||
t.Fatalf("Seal() error = %v", err)
|
||||
}
|
||||
if !strings.HasPrefix(sealed, Prefix) {
|
||||
t.Fatalf("Seal() = %q, want prefix %q", sealed, Prefix)
|
||||
}
|
||||
if strings.Contains(sealed, "secret-token") {
|
||||
t.Fatalf("Seal() = %q, want token redacted", sealed)
|
||||
}
|
||||
|
||||
opened, err := Open(sealed)
|
||||
if err != nil {
|
||||
t.Fatalf("Open() error = %v", err)
|
||||
}
|
||||
if want := `{"api_token":"secret-token"}`; opened != want {
|
||||
t.Errorf("Open(Seal(value)) = %q, want %q", opened, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSealWithoutSessionSecretKeepsPlaintextCompatibility(t *testing.T) {
|
||||
previous := runtimeconfig.Get()
|
||||
SetSessionSecret("")
|
||||
runtimeconfig.Set(runtimeconfig.Snapshot{})
|
||||
t.Cleanup(func() {
|
||||
SetSessionSecret(previous.SessionSecret)
|
||||
runtimeconfig.Set(previous)
|
||||
})
|
||||
|
||||
sealed, err := Seal(" legacy-value ")
|
||||
if err != nil {
|
||||
t.Fatalf("Seal() error = %v", err)
|
||||
}
|
||||
if sealed != "legacy-value" {
|
||||
t.Errorf("Seal() = %q, want %q", sealed, "legacy-value")
|
||||
}
|
||||
|
||||
opened, err := Open(sealed)
|
||||
if err != nil {
|
||||
t.Fatalf("Open(plaintext) error = %v", err)
|
||||
}
|
||||
if opened != "legacy-value" {
|
||||
t.Errorf("Open(plaintext) = %q, want %q", opened, "legacy-value")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenEncryptedValueRequiresSessionSecret(t *testing.T) {
|
||||
previous := runtimeconfig.Get()
|
||||
SetSessionSecret("cloudflare-pointing-test-secret")
|
||||
sealed, err := Seal("secret")
|
||||
if err != nil {
|
||||
t.Fatalf("Seal() error = %v", err)
|
||||
}
|
||||
|
||||
SetSessionSecret("")
|
||||
runtimeconfig.Set(runtimeconfig.Snapshot{})
|
||||
t.Cleanup(func() {
|
||||
SetSessionSecret(previous.SessionSecret)
|
||||
runtimeconfig.Set(previous)
|
||||
})
|
||||
if _, err := Open(sealed); err == nil {
|
||||
t.Fatal("Open(encrypted) error = nil, want missing session secret error")
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@@ -0,0 +1,18 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Package data embeds the MaxMind GeoLite2 Country database for the control plane.
|
||||
//
|
||||
// Server keeps a Country-only embed so MaxMind provider can seed without network.
|
||||
// Agent does NOT use this package — Agent MMDB files are image COPY / download only.
|
||||
package data
|
||||
|
||||
import "embed"
|
||||
|
||||
// FS holds the embedded GeoLite2-Country.mmdb database.
|
||||
//
|
||||
//go:embed GeoLite2-Country.mmdb
|
||||
var FS embed.FS
|
||||
|
||||
// DefaultMMDBName is the filename of the embedded MaxMind Country database.
|
||||
const DefaultMMDBName = "GeoLite2-Country.mmdb"
|
||||
@@ -0,0 +1,28 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package data
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/oschwald/maxminddb-golang"
|
||||
)
|
||||
|
||||
func TestEmbeddedCountryDatabaseIsValid(t *testing.T) {
|
||||
raw, err := fs.ReadFile(FS, DefaultMMDBName)
|
||||
if err != nil {
|
||||
t.Fatalf("read embedded Country database: %v", err)
|
||||
}
|
||||
reader, err := maxminddb.FromBytes(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("open embedded Country database: %v", err)
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
if !strings.Contains(reader.Metadata.DatabaseType, "Country") {
|
||||
t.Fatalf("unexpected database type %q", reader.Metadata.DatabaseType)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Package geoip provides OpenFlare-compatible GeoIP lookup helpers.
|
||||
package geoip
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net"
|
||||
"strings"
|
||||
|
||||
pkggeoip "Wavelet/openflare/share/geoip"
|
||||
)
|
||||
|
||||
// LookupView is the legacy OpenFlare GeoIP lookup response shape.
|
||||
type LookupView struct {
|
||||
Provider string `json:"provider"`
|
||||
IP string `json:"ip"`
|
||||
ISOCode string `json:"iso_code"`
|
||||
Name string `json:"name"`
|
||||
Latitude *float64 `json:"latitude,omitempty"`
|
||||
Longitude *float64 `json:"longitude,omitempty"`
|
||||
}
|
||||
|
||||
const (
|
||||
errProviderInvalid = "归属方式仅支持 disabled、mmdb、ip-api、geojs、ipinfo"
|
||||
errIPEmpty = "IP 不能为空"
|
||||
errIPInvalid = "IP 格式无效"
|
||||
errLookupEmpty = "未获取到 IP 归属结果"
|
||||
)
|
||||
|
||||
// IsValidProvider reports whether provider is a supported GeoIP backend.
|
||||
func IsValidProvider(provider string) bool {
|
||||
return pkggeoip.IsValidProvider(provider)
|
||||
}
|
||||
|
||||
// GeoInfoFromIP resolves geographic information using the configured default provider.
|
||||
func GeoInfoFromIP(ip net.IP) (*pkggeoip.GeoInfo, error) {
|
||||
return pkggeoip.GetGeoInfo(ip)
|
||||
}
|
||||
|
||||
// Lookup resolves geographic information for rawIP using the given provider.
|
||||
func Lookup(provider, rawIP string) (*LookupView, error) {
|
||||
trimmedProvider := strings.TrimSpace(provider)
|
||||
if !pkggeoip.IsValidProvider(trimmedProvider) {
|
||||
return nil, errors.New(errProviderInvalid)
|
||||
}
|
||||
|
||||
trimmedIP := strings.TrimSpace(rawIP)
|
||||
if trimmedIP == "" {
|
||||
return nil, errors.New(errIPEmpty)
|
||||
}
|
||||
parsedIP := net.ParseIP(trimmedIP)
|
||||
if parsedIP == nil {
|
||||
return nil, errors.New(errIPInvalid)
|
||||
}
|
||||
|
||||
if trimmedProvider == pkggeoip.ProviderDisabled {
|
||||
return &LookupView{
|
||||
Provider: trimmedProvider,
|
||||
IP: parsedIP.String(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
info, err := pkggeoip.LookupGeoInfoWithProvider(trimmedProvider, parsedIP)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if info == nil {
|
||||
return nil, errors.New(errLookupEmpty)
|
||||
}
|
||||
|
||||
return &LookupView{
|
||||
Provider: trimmedProvider,
|
||||
IP: parsedIP.String(),
|
||||
ISOCode: info.ISOCode,
|
||||
Name: info.Name,
|
||||
Latitude: info.Latitude,
|
||||
Longitude: info.Longitude,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package geoip
|
||||
|
||||
import (
|
||||
"net"
|
||||
"testing"
|
||||
|
||||
pkggeoip "Wavelet/openflare/share/geoip"
|
||||
)
|
||||
|
||||
type fakeLookupProvider struct{}
|
||||
|
||||
func (f *fakeLookupProvider) Name() string { return "fake-lookup" }
|
||||
|
||||
func (f *fakeLookupProvider) GetGeoInfo(_ net.IP) (*pkggeoip.GeoInfo, error) {
|
||||
lat := 37.7749
|
||||
lon := -122.4194
|
||||
return &pkggeoip.GeoInfo{
|
||||
ISOCode: "US",
|
||||
Name: "United States",
|
||||
Latitude: &lat,
|
||||
Longitude: &lon,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (f *fakeLookupProvider) UpdateDatabase() error { return nil }
|
||||
|
||||
func (f *fakeLookupProvider) Close() error { return nil }
|
||||
|
||||
func TestLookupWithProvider(t *testing.T) {
|
||||
previousFactory := pkggeoip.ProviderFactoryForTest()
|
||||
pkggeoip.SetProviderFactoryForTest(func(provider string) (pkggeoip.Service, error) {
|
||||
return &fakeLookupProvider{}, nil
|
||||
})
|
||||
t.Cleanup(func() {
|
||||
pkggeoip.SetProviderFactoryForTest(previousFactory)
|
||||
})
|
||||
|
||||
view, err := Lookup("ipinfo", "8.8.8.8")
|
||||
if err != nil {
|
||||
t.Fatalf("Lookup() error = %v", err)
|
||||
}
|
||||
if view.Provider != "ipinfo" || view.IP != "8.8.8.8" {
|
||||
t.Fatalf("unexpected lookup view: %+v", view)
|
||||
}
|
||||
if view.ISOCode != "US" || view.Name != "United States" {
|
||||
t.Fatalf("unexpected geo fields: %+v", view)
|
||||
}
|
||||
if view.Latitude == nil || view.Longitude == nil {
|
||||
t.Fatalf("expected coordinates, got %+v", view)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLookupRejectsInvalidInput(t *testing.T) {
|
||||
if _, err := Lookup("invalid", "8.8.8.8"); err == nil {
|
||||
t.Fatal("expected invalid provider to fail")
|
||||
}
|
||||
if _, err := Lookup("ipinfo", "not-an-ip"); err == nil {
|
||||
t.Fatal("expected invalid IP to fail")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLookupDisabledProvider(t *testing.T) {
|
||||
view, err := Lookup("disabled", "8.8.8.8")
|
||||
if err != nil {
|
||||
t.Fatalf("Lookup() error = %v", err)
|
||||
}
|
||||
if view.Provider != "disabled" || view.IP != "8.8.8.8" {
|
||||
t.Fatalf("unexpected disabled view: %+v", view)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package geoip
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"strings"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/model"
|
||||
pkggeoip "Wavelet/openflare/share/geoip"
|
||||
"Wavelet/pkg/logger"
|
||||
)
|
||||
|
||||
// ApplyNodeGeoFromIP resolves geographic metadata for node when geo is not manually locked.
|
||||
func ApplyNodeGeoFromIP(ctx context.Context, node *model.OpenFlareNode, rawIP string) {
|
||||
if node == nil || node.GeoManualOverride {
|
||||
return
|
||||
}
|
||||
node.GeoName = ""
|
||||
node.GeoLatitude = nil
|
||||
node.GeoLongitude = nil
|
||||
|
||||
ip := net.ParseIP(strings.TrimSpace(rawIP))
|
||||
if ip == nil {
|
||||
return
|
||||
}
|
||||
|
||||
info, err := GeoInfoFromIP(ip)
|
||||
if err != nil {
|
||||
logger.WarnF(ctx, "[GeoIP] resolve node geo failed: ip=%s error=%v", ip.String(), err)
|
||||
return
|
||||
}
|
||||
if info == nil {
|
||||
return
|
||||
}
|
||||
if strings.TrimSpace(info.Name) != "" {
|
||||
node.GeoName = strings.TrimSpace(info.Name)
|
||||
}
|
||||
if info.Latitude != nil && info.Longitude != nil {
|
||||
node.GeoLatitude = cloneCoordinate(info.Latitude)
|
||||
node.GeoLongitude = cloneCoordinate(info.Longitude)
|
||||
return
|
||||
}
|
||||
applyCountryCentroid(node, info.ISOCode, info.Name)
|
||||
}
|
||||
|
||||
func applyCountryCentroid(node *model.OpenFlareNode, isoCode, name string) {
|
||||
if lat, lon, ok := pkggeoip.CountryCentroidByISO(isoCode); ok {
|
||||
node.GeoLatitude = cloneFloat(lat)
|
||||
node.GeoLongitude = cloneFloat(lon)
|
||||
return
|
||||
}
|
||||
if lat, lon, ok := pkggeoip.CountryCentroidByName(name); ok {
|
||||
node.GeoLatitude = cloneFloat(lat)
|
||||
node.GeoLongitude = cloneFloat(lon)
|
||||
}
|
||||
}
|
||||
|
||||
func cloneFloat(value float64) *float64 {
|
||||
cloned := value
|
||||
return &cloned
|
||||
}
|
||||
|
||||
func cloneCoordinate(value *float64) *float64 {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
cloned := *value
|
||||
return &cloned
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package geoip
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
geodata "Wavelet/openflare/plugins/server/kernel/geoip/data"
|
||||
"Wavelet/openflare/plugins/server/kernel/model"
|
||||
"Wavelet/openflare/plugins/server/kernel/repository"
|
||||
pkggeoip "Wavelet/openflare/share/geoip"
|
||||
"Wavelet/pkg/logger"
|
||||
)
|
||||
|
||||
const (
|
||||
serverMMDBRelativePath = "data/GeoLite2-Country.mmdb"
|
||||
serverMMDBDirPerm = 0o750
|
||||
serverMMDBFilePerm = 0o644
|
||||
)
|
||||
|
||||
var (
|
||||
runtimeOnce sync.Once
|
||||
errRuntimeInit error
|
||||
currentProviderMu sync.RWMutex
|
||||
currentProvider string
|
||||
)
|
||||
|
||||
// EnsureRuntimeProvider loads GeoIP provider config from SystemConfig.
|
||||
func EnsureRuntimeProvider(ctx context.Context) error {
|
||||
runtimeOnce.Do(func() {
|
||||
errRuntimeInit = applyProviderFromSystemConfig(ctx)
|
||||
})
|
||||
return errRuntimeInit
|
||||
}
|
||||
|
||||
// RefreshRuntimeProvider reapplies GeoIPProvider after config updates.
|
||||
func RefreshRuntimeProvider(ctx context.Context) error {
|
||||
return applyProviderFromSystemConfig(ctx)
|
||||
}
|
||||
|
||||
func applyProviderFromSystemConfig(ctx context.Context) error {
|
||||
config, err := repository.GetSystemConfigByKey(ctx, model.ConfigKeyGeoIPProvider)
|
||||
if err != nil {
|
||||
// 降级到默认值
|
||||
return ApplyProvider(ctx, "ipinfo")
|
||||
}
|
||||
provider := strings.TrimSpace(config.Value)
|
||||
if provider == "" {
|
||||
provider = "ipinfo"
|
||||
}
|
||||
return ApplyProvider(ctx, provider)
|
||||
}
|
||||
|
||||
// ApplyProvider switches the process-wide GeoIP backend.
|
||||
func ApplyProvider(ctx context.Context, provider string) error {
|
||||
normalized := strings.TrimSpace(strings.ToLower(provider))
|
||||
if normalized == "" {
|
||||
normalized = pkggeoip.ProviderDisabled
|
||||
}
|
||||
|
||||
currentProviderMu.Lock()
|
||||
if currentProvider == normalized {
|
||||
currentProviderMu.Unlock()
|
||||
return nil
|
||||
}
|
||||
currentProvider = normalized
|
||||
currentProviderMu.Unlock()
|
||||
|
||||
if normalized == pkggeoip.ProviderMaxMind {
|
||||
path, err := ensureServerMMDB()
|
||||
if err != nil {
|
||||
logger.WarnF(ctx, "[GeoIP] seed MaxMind database failed: %v", err)
|
||||
}
|
||||
if path != "" {
|
||||
pkggeoip.GeoIPFilePath = path
|
||||
}
|
||||
}
|
||||
|
||||
pkggeoip.InitGeoIP(normalized)
|
||||
return nil
|
||||
}
|
||||
|
||||
func ensureServerMMDB() (string, error) {
|
||||
path, err := filepath.Abs(serverMMDBRelativePath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
_, statErr := os.Stat(path)
|
||||
if statErr == nil {
|
||||
return path, nil
|
||||
}
|
||||
if !os.IsNotExist(statErr) {
|
||||
return "", statErr
|
||||
}
|
||||
|
||||
// Control plane: seed Country from embedded asset (no City; Agent uses disk/image).
|
||||
data, err := fs.ReadFile(geodata.FS, geodata.DefaultMMDBName)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(path), serverMMDBDirPerm); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := os.WriteFile(path, data, serverMMDBFilePerm); err != nil { //nolint:gosec // world-readable mmdb
|
||||
return "", err
|
||||
}
|
||||
return path, nil
|
||||
}
|
||||
|
||||
// ResetRuntimeForTest clears lazy-init state for unit tests.
|
||||
func ResetRuntimeForTest() {
|
||||
runtimeOnce = sync.Once{}
|
||||
errRuntimeInit = nil
|
||||
currentProviderMu.Lock()
|
||||
currentProvider = ""
|
||||
currentProviderMu.Unlock()
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package geoip
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/model"
|
||||
pkggeoip "Wavelet/openflare/share/geoip"
|
||||
db "Wavelet/plugins/infra/database"
|
||||
|
||||
"github.com/glebarez/sqlite"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func TestEnsureRuntimeProviderInitializesConfiguredProvider(t *testing.T) {
|
||||
sqliteDB, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{DisableForeignKeyConstraintWhenMigrating: true})
|
||||
if err != nil {
|
||||
t.Fatalf("open sqlite: %v", err)
|
||||
}
|
||||
if err := sqliteDB.AutoMigrate(&model.SystemConfig{}); err != nil {
|
||||
t.Fatalf("migrate: %v", err)
|
||||
}
|
||||
db.SetDB(sqliteDB)
|
||||
t.Cleanup(func() {
|
||||
db.SetDB(nil)
|
||||
ResetRuntimeForTest()
|
||||
})
|
||||
|
||||
ctx := context.Background()
|
||||
ResetRuntimeForTest()
|
||||
// 通过 SystemConfig 设置 GeoIPProvider 配置
|
||||
if err := db.DB(ctx).Create(&model.SystemConfig{
|
||||
Key: model.ConfigKeyGeoIPProvider,
|
||||
Value: pkggeoip.ProviderIPInfo,
|
||||
Type: "business",
|
||||
Visibility: 0,
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("create system config: %v", err)
|
||||
}
|
||||
|
||||
if err := EnsureRuntimeProvider(ctx); err != nil {
|
||||
t.Fatalf("EnsureRuntimeProvider error = %v", err)
|
||||
}
|
||||
if pkggeoip.CurrentProvider == nil || pkggeoip.CurrentProvider.Name() == "EmptyProvider" {
|
||||
t.Fatalf("expected ipinfo provider, got %#v", pkggeoip.CurrentProvider)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,722 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Package githubrelease resolves and downloads public GitHub Release assets.
|
||||
// It deliberately does not know about Pages projects, deployments or runtime
|
||||
// state so other callers can reuse the same constrained HTTP contract.
|
||||
package githubrelease
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"math"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
const (
|
||||
// APIVersion is the GitHub REST API contract used by this package.
|
||||
APIVersion = "2026-03-10"
|
||||
|
||||
// SelectorLatest uses GitHub's repository latest-release endpoint.
|
||||
SelectorLatest Selector = "latest"
|
||||
// SelectorTag resolves one exact GitHub release tag.
|
||||
SelectorTag Selector = "tag"
|
||||
|
||||
defaultAPIBaseURL = "https://api.github.com"
|
||||
defaultUserAgent = "OpenFlare-GitHubRelease/1.0"
|
||||
metadataAccept = "application/vnd.github+json"
|
||||
assetAccept = "application/octet-stream"
|
||||
maxMetadataBytes = 4 << 20
|
||||
maxAssetErrorNames = 10
|
||||
maxSafeTextBytes = 255
|
||||
maxSafeAssetNameLen = 96
|
||||
maxDigestBytes = 96
|
||||
maxETagBytes = 512
|
||||
safePartsCapacity = 6
|
||||
)
|
||||
|
||||
var (
|
||||
errInvalidRequest = errors.New("GitHub Release 请求参数无效")
|
||||
errMetadata = errors.New("GitHub Release 元数据响应无效")
|
||||
errAssetMissing = errors.New("GitHub Release 中未找到指定的已上传 asset")
|
||||
errDownload = errors.New("GitHub Release asset 下载失败")
|
||||
errTooLarge = errors.New("GitHub Release asset 超过大小限制")
|
||||
errEmptyAsset = errors.New("GitHub Release asset 内容为空")
|
||||
errDigest = errors.New("GitHub Release asset digest 无效或校验失败")
|
||||
errCleanup = errors.New("GitHub Release 临时文件清理失败")
|
||||
|
||||
ownerPattern = regexp.MustCompile(`^[A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?$`)
|
||||
repoPattern = regexp.MustCompile(`^[A-Za-z0-9._-]+$`)
|
||||
hexPattern = regexp.MustCompile(`^[0-9a-fA-F]{64}$`)
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrInvalidRequest identifies caller configuration errors.
|
||||
ErrInvalidRequest = errInvalidRequest
|
||||
// ErrMetadata identifies malformed, unavailable or failed Release metadata requests.
|
||||
ErrMetadata = errMetadata
|
||||
// ErrAssetNotFound identifies an otherwise valid Release without the exact uploaded asset.
|
||||
ErrAssetNotFound = errAssetMissing
|
||||
// ErrDownload identifies network or HTTP failures while downloading an asset.
|
||||
ErrDownload = errDownload
|
||||
// ErrAssetTooLarge identifies assets that exceed the caller's hard byte limit.
|
||||
ErrAssetTooLarge = errTooLarge
|
||||
// ErrEmptyAsset identifies an empty downloaded asset.
|
||||
ErrEmptyAsset = errEmptyAsset
|
||||
// ErrDigestMismatch identifies malformed or mismatched declared SHA-256 digests.
|
||||
ErrDigestMismatch = errDigest
|
||||
)
|
||||
|
||||
// Selector identifies GitHub's own latest endpoint or one exact tag.
|
||||
type Selector string
|
||||
|
||||
// ResolveRequest describes one public repository release asset lookup.
|
||||
type ResolveRequest struct {
|
||||
Repository string
|
||||
Selector Selector
|
||||
Tag string
|
||||
AssetName string
|
||||
ETag string
|
||||
}
|
||||
|
||||
// Release contains only metadata safe and necessary for source resolution.
|
||||
type Release struct {
|
||||
ID string `json:"release_id"`
|
||||
Tag string `json:"tag"`
|
||||
Name string `json:"name,omitempty"`
|
||||
Draft bool `json:"draft"`
|
||||
Prerelease bool `json:"prerelease"`
|
||||
PublishedAt time.Time `json:"published_at,omitzero"`
|
||||
}
|
||||
|
||||
// Asset contains the immutable target metadata returned by a resolve call.
|
||||
type Asset struct {
|
||||
ID string `json:"asset_id"`
|
||||
Name string `json:"asset_name"`
|
||||
State string `json:"state"`
|
||||
Size int64 `json:"size"`
|
||||
UpdatedAt time.Time `json:"updated_at,omitzero"`
|
||||
Digest string `json:"digest,omitempty"`
|
||||
}
|
||||
|
||||
// ResolveResult is either a selected uploaded asset or a not-modified marker.
|
||||
type ResolveResult struct {
|
||||
NotModified bool `json:"not_modified"`
|
||||
ETag string `json:"etag,omitempty"`
|
||||
Release Release `json:"release,omitempty"`
|
||||
Asset Asset `json:"asset,omitempty"`
|
||||
RetryAt *time.Time `json:"retry_at,omitempty"`
|
||||
}
|
||||
|
||||
// DownloadRequest identifies an already resolved asset. Asset IDs never come
|
||||
// from an untrusted URL and the download endpoint is built locally.
|
||||
type DownloadRequest struct {
|
||||
Repository string
|
||||
Asset Asset
|
||||
MaxBytes int64
|
||||
}
|
||||
|
||||
// DownloadResult owns a temporary file. Call Cleanup after ingestion.
|
||||
type DownloadResult struct {
|
||||
Path string
|
||||
Size int64
|
||||
SHA256 string
|
||||
DeclaredDigest string
|
||||
}
|
||||
|
||||
// Cleanup removes the temporary file and is safe to call more than once.
|
||||
func (result *DownloadResult) Cleanup() error {
|
||||
if result == nil || result.Path == "" {
|
||||
return nil
|
||||
}
|
||||
name := result.Path
|
||||
err := os.Remove(name)
|
||||
if err == nil || errors.Is(err, os.ErrNotExist) {
|
||||
result.Path = ""
|
||||
return nil
|
||||
}
|
||||
return errCleanup
|
||||
}
|
||||
|
||||
// Error is a safe provider error. It never retains a response body, request
|
||||
// URL, redirect location or request headers.
|
||||
type Error struct {
|
||||
Kind error
|
||||
StatusCode int
|
||||
RequestID string
|
||||
Repository string
|
||||
Tag string
|
||||
AssetName string
|
||||
AvailableAssets []string
|
||||
RetryAt *time.Time
|
||||
}
|
||||
|
||||
func (providerError *Error) Error() string {
|
||||
if providerError == nil {
|
||||
return "GitHub Release 请求失败"
|
||||
}
|
||||
message := "GitHub Release 请求失败"
|
||||
if providerError.Kind != nil {
|
||||
message = providerError.Kind.Error()
|
||||
}
|
||||
parts := make([]string, 0, safePartsCapacity)
|
||||
if providerError.StatusCode != 0 {
|
||||
parts = append(parts, "status="+strconv.Itoa(providerError.StatusCode))
|
||||
}
|
||||
if providerError.RequestID != "" {
|
||||
parts = append(parts, "request_id="+providerError.RequestID)
|
||||
}
|
||||
if providerError.Repository != "" {
|
||||
parts = append(parts, "repo="+providerError.Repository)
|
||||
}
|
||||
if providerError.Tag != "" {
|
||||
parts = append(parts, "tag="+providerError.Tag)
|
||||
}
|
||||
if providerError.AssetName != "" {
|
||||
parts = append(parts, "asset="+providerError.AssetName)
|
||||
}
|
||||
if len(providerError.AvailableAssets) > 0 {
|
||||
parts = append(parts, "available="+strings.Join(providerError.AvailableAssets, ","))
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return message
|
||||
}
|
||||
return message + " (" + strings.Join(parts, " ") + ")"
|
||||
}
|
||||
|
||||
func (providerError *Error) Unwrap() error {
|
||||
if providerError == nil {
|
||||
return nil
|
||||
}
|
||||
return providerError.Kind
|
||||
}
|
||||
|
||||
// RetryAt extracts the server-directed retry deadline from an error.
|
||||
func RetryAt(err error) (time.Time, bool) {
|
||||
var providerError *Error
|
||||
if !errors.As(err, &providerError) || providerError.RetryAt == nil {
|
||||
return time.Time{}, false
|
||||
}
|
||||
return *providerError.RetryAt, true
|
||||
}
|
||||
|
||||
// RetryTime is retained as a compatibility alias for early callers.
|
||||
//
|
||||
// Deprecated: use RetryAt.
|
||||
func RetryTime(err error) (time.Time, bool) {
|
||||
return RetryAt(err)
|
||||
}
|
||||
|
||||
// IsNotFound reports both a missing Release endpoint and a Release that lacks
|
||||
// the exact uploaded asset requested by the caller.
|
||||
func IsNotFound(err error) bool {
|
||||
if errors.Is(err, ErrAssetNotFound) {
|
||||
return true
|
||||
}
|
||||
var providerError *Error
|
||||
return errors.As(err, &providerError) && providerError.StatusCode == http.StatusNotFound
|
||||
}
|
||||
|
||||
// IsDigestError reports malformed or mismatched declared asset digests.
|
||||
func IsDigestError(err error) bool {
|
||||
return errors.Is(err, ErrDigestMismatch)
|
||||
}
|
||||
|
||||
// IsRetryable classifies provider failures without relying on localized error
|
||||
// strings. Configuration, not-found, size, empty-content and digest failures
|
||||
// are permanent. Network failures, 408/425/429 and 5xx responses are retryable.
|
||||
func IsRetryable(err error) bool {
|
||||
if err == nil || errors.Is(err, ErrInvalidRequest) || IsNotFound(err) ||
|
||||
errors.Is(err, ErrAssetTooLarge) || errors.Is(err, ErrEmptyAsset) || IsDigestError(err) {
|
||||
return false
|
||||
}
|
||||
var providerError *Error
|
||||
if !errors.As(err, &providerError) {
|
||||
return false
|
||||
}
|
||||
if providerError.StatusCode == 0 {
|
||||
return errors.Is(err, ErrMetadata) || errors.Is(err, ErrDownload) || errors.Is(err, errCleanup)
|
||||
}
|
||||
if providerError.StatusCode < http.StatusBadRequest {
|
||||
return errors.Is(err, ErrMetadata) || errors.Is(err, ErrDownload) || errors.Is(err, errCleanup)
|
||||
}
|
||||
if providerError.RetryAt != nil {
|
||||
return true
|
||||
}
|
||||
return providerError.StatusCode == http.StatusRequestTimeout ||
|
||||
providerError.StatusCode == http.StatusTooEarly ||
|
||||
providerError.StatusCode == http.StatusTooManyRequests ||
|
||||
providerError.StatusCode >= http.StatusInternalServerError
|
||||
}
|
||||
|
||||
// Client accesses public GitHub Releases using a fixed, constrained transport.
|
||||
type Client struct {
|
||||
httpClient *http.Client
|
||||
baseURL string
|
||||
createTemp func(string, string) (*os.File, error)
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
// NewClient constructs a production client for api.github.com. Public
|
||||
// repositories do not require or send a token.
|
||||
func NewClient() *Client {
|
||||
return newClient(defaultClientOptions())
|
||||
}
|
||||
|
||||
// Resolve calls GitHub's latest or exact-tag endpoint and selects one exact,
|
||||
// case-sensitive uploaded asset. It never falls back to source archives.
|
||||
func (client *Client) Resolve(ctx context.Context, request ResolveRequest) (ResolveResult, error) {
|
||||
repository, tag, endpoint, err := normalizeResolveRequest(client.baseURL, request)
|
||||
if err != nil {
|
||||
return ResolveResult{}, safeError(errInvalidRequest, 0, "", repository, tag, validErrorAssetName(request.AssetName), nil, nil)
|
||||
}
|
||||
httpRequest, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
|
||||
if err != nil {
|
||||
return ResolveResult{}, safeError(errInvalidRequest, 0, "", repository, tag, validErrorAssetName(request.AssetName), nil, nil)
|
||||
}
|
||||
applyMetadataHeaders(httpRequest, request.ETag)
|
||||
response, err := client.httpClient.Do(httpRequest) //nolint:gosec // endpoint and every dial target are constrained
|
||||
if err != nil {
|
||||
return ResolveResult{}, safeError(errMetadata, 0, "", repository, tag, request.AssetName, nil, nil)
|
||||
}
|
||||
defer func() { _ = response.Body.Close() }()
|
||||
|
||||
retryAt := responseRetryAt(response, client.now())
|
||||
etag := safeETag(response.Header.Get("ETag"))
|
||||
if response.StatusCode == http.StatusNotModified {
|
||||
if etag == "" {
|
||||
etag = safeETag(request.ETag)
|
||||
}
|
||||
return ResolveResult{NotModified: true, ETag: etag, RetryAt: retryAt}, nil
|
||||
}
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return ResolveResult{}, safeHTTPError(errMetadata, response, repository, tag, request.AssetName, retryAt)
|
||||
}
|
||||
|
||||
body, readErr := io.ReadAll(io.LimitReader(response.Body, maxMetadataBytes+1))
|
||||
if readErr != nil || len(body) > maxMetadataBytes || !utf8.Valid(body) {
|
||||
return ResolveResult{}, safeHTTPError(errMetadata, response, repository, tag, request.AssetName, retryAt)
|
||||
}
|
||||
var payload releasePayload
|
||||
decoder := json.NewDecoder(bytes.NewReader(body))
|
||||
decoder.UseNumber()
|
||||
if err := decoder.Decode(&payload); err != nil {
|
||||
return ResolveResult{}, safeHTTPError(errMetadata, response, repository, tag, request.AssetName, retryAt)
|
||||
}
|
||||
if err := ensureJSONEOF(decoder); err != nil {
|
||||
return ResolveResult{}, safeHTTPError(errMetadata, response, repository, tag, request.AssetName, retryAt)
|
||||
}
|
||||
release, assets, err := convertRelease(payload)
|
||||
if err != nil {
|
||||
return ResolveResult{}, safeHTTPError(errMetadata, response, repository, tag, request.AssetName, retryAt)
|
||||
}
|
||||
for _, asset := range assets {
|
||||
if asset.State == "uploaded" && asset.Name == request.AssetName {
|
||||
return ResolveResult{
|
||||
ETag: etag,
|
||||
Release: release,
|
||||
Asset: asset,
|
||||
RetryAt: retryAt,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
available := safeAssetNames(assets)
|
||||
return ResolveResult{}, safeError(
|
||||
errAssetMissing,
|
||||
response.StatusCode,
|
||||
response.Header.Get("X-Github-Request-Id"),
|
||||
repository,
|
||||
release.Tag,
|
||||
request.AssetName,
|
||||
available,
|
||||
retryAt,
|
||||
)
|
||||
}
|
||||
|
||||
// Download streams an asset into a package-owned temporary file while
|
||||
// enforcing a hard byte limit and verifying GitHub's declared sha256 digest.
|
||||
func (client *Client) Download(ctx context.Context, request DownloadRequest) (*DownloadResult, error) {
|
||||
repository, err := normalizeRepository(request.Repository)
|
||||
if err != nil || request.MaxBytes <= 0 || !validPositiveID(request.Asset.ID) ||
|
||||
!validAssetName(request.Asset.Name) || request.Asset.Size < 0 {
|
||||
return nil, safeError(errInvalidRequest, 0, "", repository, "", validErrorAssetName(request.Asset.Name), nil, nil)
|
||||
}
|
||||
if request.Asset.Size > request.MaxBytes {
|
||||
return nil, safeError(errTooLarge, 0, "", repository, "", validErrorAssetName(request.Asset.Name), nil, nil)
|
||||
}
|
||||
endpoint := strings.TrimRight(client.baseURL, "/") + "/repos/" + repository + "/releases/assets/" + request.Asset.ID
|
||||
httpRequest, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
|
||||
if err != nil {
|
||||
return nil, safeError(errInvalidRequest, 0, "", repository, "", request.Asset.Name, nil, nil)
|
||||
}
|
||||
applyAssetHeaders(httpRequest)
|
||||
response, err := client.httpClient.Do(httpRequest) //nolint:gosec // endpoint and every dial target are constrained
|
||||
if err != nil {
|
||||
return nil, safeError(errDownload, 0, "", repository, "", request.Asset.Name, nil, nil)
|
||||
}
|
||||
defer func() { _ = response.Body.Close() }()
|
||||
retryAt := responseRetryAt(response, client.now())
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return nil, safeHTTPError(errDownload, response, repository, "", request.Asset.Name, retryAt)
|
||||
}
|
||||
if response.ContentLength > request.MaxBytes {
|
||||
return nil, safeHTTPError(errTooLarge, response, repository, "", request.Asset.Name, retryAt)
|
||||
}
|
||||
|
||||
result, err := client.streamAsset(response.Body, request.MaxBytes, request.Asset.Digest)
|
||||
if err != nil {
|
||||
return nil, safeError(err, response.StatusCode, response.Header.Get("X-Github-Request-Id"), repository, "", request.Asset.Name, nil, retryAt)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (client *Client) streamAsset(body io.Reader, maxBytes int64, declaredDigest string) (result *DownloadResult, resultErr error) {
|
||||
tempFile, err := client.createTemp("", "openflare-github-release-*")
|
||||
if err != nil {
|
||||
return nil, errDownload
|
||||
}
|
||||
tempPath := tempFile.Name()
|
||||
defer func() {
|
||||
closeErr := tempFile.Close()
|
||||
if resultErr == nil && closeErr != nil {
|
||||
resultErr = errDownload
|
||||
}
|
||||
if resultErr != nil {
|
||||
if removeErr := os.Remove(tempPath); removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) {
|
||||
resultErr = errCleanup
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
hasher := sha256.New()
|
||||
readLimit := maxBytes
|
||||
if readLimit < math.MaxInt64 {
|
||||
readLimit++
|
||||
}
|
||||
size, err := io.Copy(io.MultiWriter(tempFile, hasher), io.LimitReader(body, readLimit))
|
||||
if err != nil {
|
||||
return nil, errDownload
|
||||
}
|
||||
if size > maxBytes {
|
||||
return nil, errTooLarge
|
||||
}
|
||||
if size == 0 {
|
||||
return nil, errEmptyAsset
|
||||
}
|
||||
checksum := hex.EncodeToString(hasher.Sum(nil))
|
||||
if err := verifyDeclaredDigest(declaredDigest, checksum); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &DownloadResult{
|
||||
Path: tempPath,
|
||||
Size: size,
|
||||
SHA256: checksum,
|
||||
DeclaredDigest: strings.ToLower(strings.TrimSpace(declaredDigest)),
|
||||
}, nil
|
||||
}
|
||||
|
||||
type releasePayload struct {
|
||||
ID json.Number `json:"id"`
|
||||
Tag string `json:"tag_name"`
|
||||
Name string `json:"name"`
|
||||
Draft bool `json:"draft"`
|
||||
Prerelease bool `json:"prerelease"`
|
||||
PublishedAt string `json:"published_at"`
|
||||
Assets []assetPayload `json:"assets"`
|
||||
}
|
||||
|
||||
type assetPayload struct {
|
||||
ID json.Number `json:"id"`
|
||||
Name string `json:"name"`
|
||||
State string `json:"state"`
|
||||
Size int64 `json:"size"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
Digest string `json:"digest"`
|
||||
}
|
||||
|
||||
func convertRelease(payload releasePayload) (Release, []Asset, error) {
|
||||
releaseID, err := positiveJSONID(payload.ID)
|
||||
if err != nil {
|
||||
return Release{}, nil, err
|
||||
}
|
||||
if !validReleaseDisplayTag(payload.Tag) {
|
||||
return Release{}, nil, errMetadata
|
||||
}
|
||||
publishedAt, err := parseOptionalTime(payload.PublishedAt)
|
||||
if err != nil {
|
||||
return Release{}, nil, err
|
||||
}
|
||||
release := Release{
|
||||
ID: releaseID,
|
||||
Tag: payload.Tag,
|
||||
Name: safeText(payload.Name, maxSafeTextBytes),
|
||||
Draft: payload.Draft,
|
||||
Prerelease: payload.Prerelease,
|
||||
PublishedAt: publishedAt,
|
||||
}
|
||||
assets := make([]Asset, 0, len(payload.Assets))
|
||||
for _, rawAsset := range payload.Assets {
|
||||
assetID, assetErr := positiveJSONID(rawAsset.ID)
|
||||
if assetErr != nil || rawAsset.Size < 0 {
|
||||
return Release{}, nil, errMetadata
|
||||
}
|
||||
updatedAt, assetErr := parseOptionalTime(rawAsset.UpdatedAt)
|
||||
if assetErr != nil {
|
||||
return Release{}, nil, errMetadata
|
||||
}
|
||||
assets = append(assets, Asset{
|
||||
ID: assetID,
|
||||
Name: rawAsset.Name,
|
||||
State: rawAsset.State,
|
||||
Size: rawAsset.Size,
|
||||
UpdatedAt: updatedAt,
|
||||
Digest: safeText(rawAsset.Digest, maxDigestBytes),
|
||||
})
|
||||
}
|
||||
return release, assets, nil
|
||||
}
|
||||
|
||||
func normalizeResolveRequest(baseURL string, request ResolveRequest) (string, string, string, error) {
|
||||
repository, err := normalizeRepository(request.Repository)
|
||||
if err != nil || !validAssetName(request.AssetName) {
|
||||
return repository, validErrorTag(request.Tag), "", errInvalidRequest
|
||||
}
|
||||
baseURL = strings.TrimRight(baseURL, "/")
|
||||
switch request.Selector {
|
||||
case SelectorLatest:
|
||||
if strings.TrimSpace(request.Tag) != "" {
|
||||
return repository, "", "", errInvalidRequest
|
||||
}
|
||||
return repository, "latest", baseURL + "/repos/" + repository + "/releases/latest", nil
|
||||
case SelectorTag:
|
||||
if !validTag(request.Tag) {
|
||||
return repository, validErrorTag(request.Tag), "", errInvalidRequest
|
||||
}
|
||||
return repository, request.Tag, baseURL + "/repos/" + repository + "/releases/tags/" + url.PathEscape(request.Tag), nil
|
||||
default:
|
||||
return repository, validErrorTag(request.Tag), "", errInvalidRequest
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeRepository(repository string) (string, error) {
|
||||
repository = strings.TrimSpace(repository)
|
||||
parts := strings.Split(repository, "/")
|
||||
if len(parts) != 2 || !ownerPattern.MatchString(parts[0]) || !repoPattern.MatchString(parts[1]) ||
|
||||
len(parts[1]) > 100 || parts[1] == "." || parts[1] == ".." {
|
||||
return "", errInvalidRequest
|
||||
}
|
||||
return parts[0] + "/" + parts[1], nil
|
||||
}
|
||||
|
||||
func validAssetName(assetName string) bool {
|
||||
return validLogText(assetName, maxSafeTextBytes, false) && path.Base(assetName) == assetName &&
|
||||
assetName != "." && assetName != ".." && !strings.ContainsAny(assetName, `/\`)
|
||||
}
|
||||
|
||||
func validTag(tag string) bool {
|
||||
if !validLogText(tag, maxSafeTextBytes, false) || strings.ContainsAny(tag, " ~^:?*[\\") ||
|
||||
strings.Contains(tag, "..") || strings.Contains(tag, "@{") || strings.Contains(tag, "//") ||
|
||||
strings.HasPrefix(tag, "/") || strings.HasSuffix(tag, "/") || strings.HasSuffix(tag, ".") {
|
||||
return false
|
||||
}
|
||||
for component := range strings.SplitSeq(tag, "/") {
|
||||
if component == "" || strings.HasPrefix(component, ".") || strings.HasSuffix(component, ".lock") {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func validReleaseDisplayTag(tag string) bool {
|
||||
return validLogText(tag, maxSafeTextBytes, false)
|
||||
}
|
||||
|
||||
func validLogText(value string, maxBytes int, allowEmpty bool) bool {
|
||||
if (!allowEmpty && value == "") || len(value) > maxBytes || !utf8.ValidString(value) {
|
||||
return false
|
||||
}
|
||||
for _, character := range value {
|
||||
if isLogControl(character) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func isLogControl(character rune) bool {
|
||||
if unicode.IsControl(character) || character == '\u2028' || character == '\u2029' {
|
||||
return true
|
||||
}
|
||||
switch character {
|
||||
case '\u061c', '\u200e', '\u200f',
|
||||
'\u202a', '\u202b', '\u202c', '\u202d', '\u202e',
|
||||
'\u2066', '\u2067', '\u2068', '\u2069':
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func validErrorTag(tag string) string {
|
||||
if !validTag(tag) || containsSecretDelimiter(tag) {
|
||||
return ""
|
||||
}
|
||||
return tag
|
||||
}
|
||||
|
||||
func validErrorAssetName(assetName string) string {
|
||||
if !validAssetName(assetName) || containsSecretDelimiter(assetName) {
|
||||
return ""
|
||||
}
|
||||
return assetName
|
||||
}
|
||||
|
||||
func containsSecretDelimiter(value string) bool {
|
||||
return strings.ContainsAny(value, "?&=#") || strings.Contains(value, "://")
|
||||
}
|
||||
|
||||
func validPositiveID(id string) bool {
|
||||
parsed, err := strconv.ParseInt(id, 10, 64)
|
||||
return err == nil && parsed > 0 && strconv.FormatInt(parsed, 10) == id
|
||||
}
|
||||
|
||||
func positiveJSONID(id json.Number) (string, error) {
|
||||
parsed, err := strconv.ParseInt(id.String(), 10, 64)
|
||||
if err != nil || parsed <= 0 {
|
||||
return "", errMetadata
|
||||
}
|
||||
return strconv.FormatInt(parsed, 10), nil
|
||||
}
|
||||
|
||||
func parseOptionalTime(value string) (time.Time, error) {
|
||||
if value == "" {
|
||||
return time.Time{}, nil
|
||||
}
|
||||
parsed, err := time.Parse(time.RFC3339, value)
|
||||
if err != nil {
|
||||
return time.Time{}, errMetadata
|
||||
}
|
||||
return parsed, nil
|
||||
}
|
||||
|
||||
func ensureJSONEOF(decoder *json.Decoder) error {
|
||||
var trailing any
|
||||
if err := decoder.Decode(&trailing); errors.Is(err, io.EOF) {
|
||||
return nil
|
||||
}
|
||||
return errMetadata
|
||||
}
|
||||
|
||||
func verifyDeclaredDigest(declaredDigest string, checksum string) error {
|
||||
declaredDigest = strings.TrimSpace(declaredDigest)
|
||||
if declaredDigest == "" {
|
||||
return nil
|
||||
}
|
||||
algorithm, digest, ok := strings.Cut(declaredDigest, ":")
|
||||
if !ok || !strings.EqualFold(algorithm, "sha256") || !hexPattern.MatchString(digest) ||
|
||||
!strings.EqualFold(digest, checksum) {
|
||||
return errDigest
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func safeAssetNames(assets []Asset) []string {
|
||||
count := min(len(assets), maxAssetErrorNames)
|
||||
names := make([]string, 0, count)
|
||||
for _, asset := range assets[:count] {
|
||||
name := safeText(asset.Name, maxSafeAssetNameLen)
|
||||
if containsSecretDelimiter(name) {
|
||||
name = "<redacted>"
|
||||
}
|
||||
names = append(names, name)
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
func safeText(value string, maxBytes int) string {
|
||||
var builder strings.Builder
|
||||
for _, character := range value {
|
||||
if isLogControl(character) {
|
||||
builder.WriteByte('?')
|
||||
continue
|
||||
}
|
||||
builder.WriteRune(character)
|
||||
if builder.Len() >= maxBytes {
|
||||
break
|
||||
}
|
||||
}
|
||||
result := builder.String()
|
||||
for len(result) > maxBytes {
|
||||
_, size := utf8.DecodeLastRuneInString(result)
|
||||
result = result[:len(result)-size]
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func safeETag(value string) string {
|
||||
value = strings.TrimSpace(value)
|
||||
if len(value) > maxETagBytes || safeText(value, maxETagBytes) != value {
|
||||
return ""
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func safeHTTPError(kind error, response *http.Response, repository string, tag string, assetName string, retryAt *time.Time) error {
|
||||
return safeError(
|
||||
kind,
|
||||
response.StatusCode,
|
||||
response.Header.Get("X-Github-Request-Id"),
|
||||
repository,
|
||||
tag,
|
||||
assetName,
|
||||
nil,
|
||||
retryAt,
|
||||
)
|
||||
}
|
||||
|
||||
func safeError(
|
||||
kind error,
|
||||
statusCode int,
|
||||
requestID string,
|
||||
repository string,
|
||||
tag string,
|
||||
assetName string,
|
||||
availableAssets []string,
|
||||
retryAt *time.Time,
|
||||
) error {
|
||||
return &Error{
|
||||
Kind: kind,
|
||||
StatusCode: statusCode,
|
||||
RequestID: safeErrorToken(requestID, maxSafeTextBytes),
|
||||
Repository: safeErrorToken(repository, maxSafeTextBytes),
|
||||
Tag: safeErrorToken(tag, maxSafeTextBytes),
|
||||
AssetName: safeErrorToken(assetName, maxSafeAssetNameLen),
|
||||
AvailableAssets: availableAssets,
|
||||
RetryAt: retryAt,
|
||||
}
|
||||
}
|
||||
|
||||
func safeErrorToken(value string, maxBytes int) string {
|
||||
if !validLogText(value, maxBytes, true) {
|
||||
return ""
|
||||
}
|
||||
value = safeText(value, maxBytes)
|
||||
if containsSecretDelimiter(value) {
|
||||
return ""
|
||||
}
|
||||
return value
|
||||
}
|
||||
@@ -0,0 +1,748 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package githubrelease
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/netip"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type resolverFunc func(context.Context, string, string) ([]netip.Addr, error)
|
||||
|
||||
func (resolve resolverFunc) LookupNetIP(ctx context.Context, network string, host string) ([]netip.Addr, error) {
|
||||
return resolve(ctx, network, host)
|
||||
}
|
||||
|
||||
func TestResolveLatestUsesGitHubContractAndSelectsUploadedAsset(t *testing.T) {
|
||||
t.Parallel()
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
if request.URL.Path != "/repos/acme/site/releases/latest" {
|
||||
t.Errorf("path = %q", request.URL.Path)
|
||||
}
|
||||
assertHeader(t, request, "Accept", metadataAccept)
|
||||
assertHeader(t, request, "User-Agent", defaultUserAgent)
|
||||
assertHeader(t, request, "X-GitHub-Api-Version", APIVersion)
|
||||
assertHeader(t, request, "If-None-Match", `W/"old"`)
|
||||
writer.Header().Set("ETag", `W/"new"`)
|
||||
writer.Header().Set("X-RateLimit-Remaining", "0")
|
||||
writer.Header().Set("X-RateLimit-Reset", "1800000000")
|
||||
_, _ = writer.Write([]byte(`{
|
||||
"id": 9007199254740991,
|
||||
"tag_name": "v1.2.3",
|
||||
"name": "Stable",
|
||||
"published_at": "2026-07-18T12:00:00Z",
|
||||
"assets": [
|
||||
{"id": 11, "name": "dist.zip", "state": "new", "size": 1},
|
||||
{"id": 9007199254740990, "name": "dist.zip", "state": "uploaded", "size": 42,
|
||||
"updated_at": "2026-07-18T12:10:00Z", "digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}
|
||||
]
|
||||
}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client := newTestClient(t, server.URL, nil)
|
||||
result, err := client.Resolve(context.Background(), ResolveRequest{
|
||||
Repository: "acme/site",
|
||||
Selector: SelectorLatest,
|
||||
AssetName: "dist.zip",
|
||||
ETag: `W/"old"`,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
if result.Release.ID != "9007199254740991" || result.Asset.ID != "9007199254740990" {
|
||||
t.Fatalf("IDs lost precision: release=%q asset=%q", result.Release.ID, result.Asset.ID)
|
||||
}
|
||||
if result.ETag != `W/"new"` || result.Asset.Name != "dist.zip" || result.Asset.State != "uploaded" {
|
||||
t.Fatalf("Resolve() = %+v", result)
|
||||
}
|
||||
if result.RetryAt == nil || result.RetryAt.Unix() != 1800000000 {
|
||||
t.Fatalf("RetryAt = %v", result.RetryAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveTagEscapesPathAndHandlesNotModified(t *testing.T) {
|
||||
t.Parallel()
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
if request.RequestURI != "/repos/acme/site/releases/tags/release%2Fcandidate" {
|
||||
t.Errorf("RequestURI = %q", request.RequestURI)
|
||||
}
|
||||
writer.WriteHeader(http.StatusNotModified)
|
||||
}))
|
||||
defer server.Close()
|
||||
client := newTestClient(t, server.URL, nil)
|
||||
result, err := client.Resolve(context.Background(), ResolveRequest{
|
||||
Repository: "acme/site",
|
||||
Selector: SelectorTag,
|
||||
Tag: "release/candidate",
|
||||
AssetName: "dist.zip",
|
||||
ETag: `"cached"`,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
if !result.NotModified || result.ETag != `"cached"` {
|
||||
t.Fatalf("Resolve() = %+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveAssetMissingTruncatesSafeNamesAndNeverIncludesBody(t *testing.T) {
|
||||
t.Parallel()
|
||||
assets := make([]string, 0, 12)
|
||||
for index := 0; index < 12; index++ {
|
||||
assets = append(assets, fmt.Sprintf(`{"id":%d,"name":"asset-%02d.zip","state":"uploaded","size":1}`, index+1, index))
|
||||
}
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = writer.Write([]byte(`{"id":1,"tag_name":"v1","message":"body-token","assets":[` + strings.Join(assets, ",") + `]}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
client := newTestClient(t, server.URL, nil)
|
||||
_, err := client.Resolve(context.Background(), ResolveRequest{
|
||||
Repository: "acme/site", Selector: SelectorLatest, AssetName: "dist.zip",
|
||||
})
|
||||
if !errors.Is(err, errAssetMissing) {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
message := err.Error()
|
||||
if !strings.Contains(message, "asset-00.zip") || !strings.Contains(message, "asset-09.zip") {
|
||||
t.Fatalf("error misses safe truncated names: %s", message)
|
||||
}
|
||||
if strings.Contains(message, "asset-10.zip") || strings.Contains(message, "asset-11.zip") || strings.Contains(message, "body-token") {
|
||||
t.Fatalf("error leaked/truncation failed: %s", message)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveHTTPErrorParsesRateLimitWithoutBodyLeak(t *testing.T) {
|
||||
t.Parallel()
|
||||
now := time.Date(2026, time.July, 19, 10, 0, 0, 0, time.UTC)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
|
||||
writer.Header().Set("Retry-After", "90")
|
||||
writer.Header().Set("X-GitHub-Request-Id", "request-123")
|
||||
writer.WriteHeader(http.StatusTooManyRequests)
|
||||
_, _ = writer.Write([]byte(`{"message":"signed_url=https://secret.example/a?token=hidden"}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
client := newTestClient(t, server.URL, func(options *clientOptions) { options.now = func() time.Time { return now } })
|
||||
_, err := client.Resolve(context.Background(), ResolveRequest{
|
||||
Repository: "acme/site", Selector: SelectorLatest, AssetName: "dist.zip",
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "status=429") || !strings.Contains(err.Error(), "request_id=request-123") {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
if strings.Contains(err.Error(), "secret.example") || strings.Contains(err.Error(), "hidden") {
|
||||
t.Fatalf("error leaked body: %s", err)
|
||||
}
|
||||
retryAt, ok := RetryTime(err)
|
||||
if !ok || !retryAt.Equal(now.Add(90*time.Second)) {
|
||||
t.Fatalf("RetryTime() = %v, %v", retryAt, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadStreamsVerifiesDigestAndCleansUp(t *testing.T) {
|
||||
t.Parallel()
|
||||
payload := []byte("package bytes")
|
||||
digest := sha256.Sum256(payload)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
if request.URL.Path != "/repos/acme/site/releases/assets/42" {
|
||||
t.Errorf("path = %q", request.URL.Path)
|
||||
}
|
||||
assertHeader(t, request, "Accept", assetAccept)
|
||||
assertHeader(t, request, "Accept-Encoding", "identity")
|
||||
_, _ = writer.Write(payload)
|
||||
}))
|
||||
defer server.Close()
|
||||
client := newTestClient(t, server.URL, nil)
|
||||
result, err := client.Download(context.Background(), DownloadRequest{
|
||||
Repository: "acme/site",
|
||||
Asset: Asset{
|
||||
ID: "42", Name: "dist.zip", Digest: "sha256:" + hex.EncodeToString(digest[:]),
|
||||
},
|
||||
MaxBytes: 1024,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Download() error = %v", err)
|
||||
}
|
||||
if result.Size != int64(len(payload)) || result.SHA256 != hex.EncodeToString(digest[:]) {
|
||||
t.Fatalf("Download() = %+v", result)
|
||||
}
|
||||
if _, err := os.Stat(result.Path); err != nil {
|
||||
t.Fatalf("temp file stat: %v", err)
|
||||
}
|
||||
if err := result.Cleanup(); err != nil {
|
||||
t.Fatalf("Cleanup() error = %v", err)
|
||||
}
|
||||
if err := result.Cleanup(); err != nil {
|
||||
t.Fatalf("second Cleanup() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadFollows302AndStripsCrossHostSensitiveHeaders(t *testing.T) {
|
||||
t.Parallel()
|
||||
var targetHost string
|
||||
target := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
for _, header := range []string{"Authorization", "Cookie", "Proxy-Authorization", "Referer", "If-None-Match", "If-Modified-Since", "X-GitHub-Api-Version"} {
|
||||
if value := request.Header.Get(header); value != "" {
|
||||
t.Errorf("redirect leaked %s=%q", header, value)
|
||||
}
|
||||
}
|
||||
_, _ = writer.Write([]byte("redirected package"))
|
||||
}))
|
||||
defer target.Close()
|
||||
targetURL, _ := url.Parse(target.URL)
|
||||
targetHost = "asset.example.test:" + targetURL.Port()
|
||||
|
||||
api := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
|
||||
writer.Header().Set("Location", "http://"+targetHost+"/signed/package.zip?token=must-not-leak")
|
||||
writer.WriteHeader(http.StatusFound)
|
||||
}))
|
||||
defer api.Close()
|
||||
apiURL, _ := url.Parse(api.URL)
|
||||
baseURL := "http://api.example.test:" + apiURL.Port()
|
||||
client := newMappedTestClient(t, baseURL, nil)
|
||||
result, err := client.Download(context.Background(), DownloadRequest{
|
||||
Repository: "acme/site", Asset: Asset{ID: "42", Name: "dist.zip"}, MaxBytes: 1024,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Download() redirect error = %v", err)
|
||||
}
|
||||
if cleanupErr := result.Cleanup(); cleanupErr != nil {
|
||||
t.Fatalf("Cleanup() error = %v", cleanupErr)
|
||||
}
|
||||
|
||||
request, _ := http.NewRequestWithContext(context.Background(), http.MethodGet, baseURL+"/repos/acme/site/releases/assets/42", nil)
|
||||
applyAssetHeaders(request)
|
||||
request.Header.Set("Authorization", "Bearer secret")
|
||||
request.Header.Set("Cookie", "session=secret")
|
||||
request.Header.Set("Proxy-Authorization", "proxy-secret")
|
||||
request.Header.Set("Referer", "https://secret.example/path?token=x")
|
||||
request.Header.Set("If-None-Match", `"secret-etag"`)
|
||||
request.Header.Set("If-Modified-Since", time.Now().Format(http.TimeFormat))
|
||||
response, err := client.httpClient.Do(request)
|
||||
if err != nil {
|
||||
t.Fatalf("Do() error = %v", err)
|
||||
}
|
||||
_ = response.Body.Close()
|
||||
}
|
||||
|
||||
func TestRedirectSSRFAndDNSRebindingAreRejectedWithoutURLLeak(t *testing.T) {
|
||||
t.Parallel()
|
||||
t.Run("literal private redirect", func(t *testing.T) {
|
||||
api := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
|
||||
writer.Header().Set("Location", "http://127.0.0.1/private?token=secret-query")
|
||||
writer.WriteHeader(http.StatusFound)
|
||||
}))
|
||||
defer api.Close()
|
||||
client := newTestClient(t, api.URL, nil)
|
||||
_, err := client.Download(context.Background(), DownloadRequest{
|
||||
Repository: "acme/site", Asset: Asset{ID: "1", Name: "dist.zip"}, MaxBytes: 100,
|
||||
})
|
||||
if err == nil || strings.Contains(err.Error(), "secret-query") || strings.Contains(err.Error(), "127.0.0.1") {
|
||||
t.Fatalf("Download() error = %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("DNS rebind between redirect and dial", func(t *testing.T) {
|
||||
api := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
|
||||
writer.Header().Set("Location", "http://rebind.example.test/package.zip")
|
||||
writer.WriteHeader(http.StatusFound)
|
||||
}))
|
||||
defer api.Close()
|
||||
var lock sync.Mutex
|
||||
calls := map[string]int{}
|
||||
resolve := resolverFunc(func(_ context.Context, _ string, host string) ([]netip.Addr, error) {
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
calls[host]++
|
||||
if host == "rebind.example.test" && calls[host] > 1 {
|
||||
return []netip.Addr{netip.MustParseAddr("127.0.0.1")}, nil
|
||||
}
|
||||
return []netip.Addr{netip.MustParseAddr("8.8.8.8")}, nil
|
||||
})
|
||||
client := newTestClient(t, api.URL, func(options *clientOptions) { options.resolver = resolve })
|
||||
_, err := client.Download(context.Background(), DownloadRequest{
|
||||
Repository: "acme/site", Asset: Asset{ID: "1", Name: "dist.zip"}, MaxBytes: 100,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("Download() error = nil")
|
||||
}
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
if calls["rebind.example.test"] != 2 {
|
||||
t.Fatalf("rebind lookup calls = %d", calls["rebind.example.test"])
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestDownloadFailureRemovesTemporaryFile(t *testing.T) {
|
||||
t.Parallel()
|
||||
payload := []byte("package bytes")
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = writer.Write(payload)
|
||||
}))
|
||||
defer server.Close()
|
||||
tempDir := t.TempDir()
|
||||
client := newTestClient(t, server.URL, func(options *clientOptions) {
|
||||
options.createTemp = func(_ string, pattern string) (*os.File, error) {
|
||||
return os.CreateTemp(tempDir, pattern)
|
||||
}
|
||||
})
|
||||
_, err := client.Download(context.Background(), DownloadRequest{
|
||||
Repository: "acme/site",
|
||||
Asset: Asset{
|
||||
ID: "42", Name: "dist.zip", Digest: "sha256:" + strings.Repeat("0", 64),
|
||||
},
|
||||
MaxBytes: 1024,
|
||||
})
|
||||
if !errors.Is(err, errDigest) {
|
||||
t.Fatalf("Download() error = %v", err)
|
||||
}
|
||||
files, readErr := filepath.Glob(filepath.Join(tempDir, "*"))
|
||||
if readErr != nil || len(files) != 0 {
|
||||
t.Fatalf("temporary files after failure = %v, err=%v", files, readErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveRejectsInvalidRepositoryAndAssetWithoutRequest(t *testing.T) {
|
||||
t.Parallel()
|
||||
client := NewClient()
|
||||
for _, request := range []ResolveRequest{
|
||||
{Repository: "https://github.com/acme/site", Selector: SelectorLatest, AssetName: "dist.zip"},
|
||||
{Repository: "acme/site/extra", Selector: SelectorLatest, AssetName: "dist.zip"},
|
||||
{Repository: "acme/site", Selector: SelectorLatest, AssetName: "../dist.zip"},
|
||||
{Repository: "acme/site", Selector: SelectorLatest, AssetName: `dir\dist.zip`},
|
||||
{Repository: "acme/site", Selector: SelectorLatest, AssetName: string([]byte{'d', 'i', 's', 't', 0xff})},
|
||||
{Repository: "acme/site", Selector: SelectorLatest, AssetName: "dist\nsecret.zip"},
|
||||
{Repository: "acme/site", Selector: SelectorLatest, AssetName: "dist\u2028secret.zip"},
|
||||
{Repository: "acme/site", Selector: SelectorLatest, AssetName: "dist\u202esecret.zip"},
|
||||
{Repository: "acme/site", Selector: SelectorTag, AssetName: "dist.zip"},
|
||||
} {
|
||||
_, err := client.Resolve(context.Background(), request)
|
||||
if !errors.Is(err, errInvalidRequest) {
|
||||
t.Errorf("Resolve(%+v) error = %v", request, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveAndDownloadAssetNameWithDelimiters(t *testing.T) {
|
||||
t.Parallel()
|
||||
assetName := "dist?channel=stable#1&x.zip"
|
||||
payload := []byte("package with delimiter name")
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
switch request.URL.Path {
|
||||
case "/repos/acme/site/releases/latest":
|
||||
if request.Header.Get("If-None-Match") == "missing" {
|
||||
_, _ = writer.Write([]byte(`{"id":1,"tag_name":"v1","assets":[]}`))
|
||||
return
|
||||
}
|
||||
_, _ = fmt.Fprintf(writer, `{"id":1,"tag_name":"release/v1","assets":[{"id":42,"name":%q,"state":"uploaded","size":%d}]}`,
|
||||
assetName, len(payload))
|
||||
case "/repos/acme/site/releases/assets/42":
|
||||
_, _ = writer.Write(payload)
|
||||
default:
|
||||
http.NotFound(writer, request)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
client := newTestClient(t, server.URL, nil)
|
||||
|
||||
resolved, err := client.Resolve(context.Background(), ResolveRequest{
|
||||
Repository: "acme/site", Selector: SelectorLatest, AssetName: assetName,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
if resolved.Asset.Name != assetName || resolved.Release.Tag != "release/v1" {
|
||||
t.Fatalf("Resolve() = %+v", resolved)
|
||||
}
|
||||
download, err := client.Download(context.Background(), DownloadRequest{
|
||||
Repository: "acme/site", Asset: resolved.Asset, MaxBytes: 1024,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Download() error = %v", err)
|
||||
}
|
||||
if cleanupErr := download.Cleanup(); cleanupErr != nil {
|
||||
t.Fatalf("Cleanup() error = %v", cleanupErr)
|
||||
}
|
||||
|
||||
_, err = client.Resolve(context.Background(), ResolveRequest{
|
||||
Repository: "acme/site", Selector: SelectorLatest, AssetName: assetName, ETag: "missing",
|
||||
})
|
||||
if !errors.Is(err, ErrAssetNotFound) {
|
||||
t.Fatalf("missing Resolve() error = %v", err)
|
||||
}
|
||||
if strings.Contains(err.Error(), assetName) || strings.Contains(err.Error(), "channel=stable") {
|
||||
t.Fatalf("missing error leaked delimiter-bearing name: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFixedTagGitRefRulesAndEscaping(t *testing.T) {
|
||||
t.Parallel()
|
||||
valid := []string{"@", "release/v1#stable&channel=prod", "foo.LOCK", "中文/发布=稳定"}
|
||||
for _, tag := range valid {
|
||||
if !validTag(tag) {
|
||||
t.Errorf("validTag(%q) = false", tag)
|
||||
}
|
||||
}
|
||||
invalid := []string{
|
||||
"", "release v1", "release~v1", "release^v1", "release:v1", "release?v1", "release*v1",
|
||||
"release[v1", `release\v1`, "release..v1", "release@{v1", "release//v1", "/release", "release/",
|
||||
"release.", ".release", "release/.candidate", "release.lock", "release/v1.lock", "release\nsecret",
|
||||
"release\u2028secret", "release\u202esecret", string([]byte{'v', '1', 0xff}), strings.Repeat("a", 256),
|
||||
}
|
||||
for _, tag := range invalid {
|
||||
if validTag(tag) {
|
||||
t.Errorf("validTag(%q) = true", tag)
|
||||
}
|
||||
}
|
||||
|
||||
tag := "release/v1#stable&channel=prod"
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
wantURI := "/repos/acme/site/releases/tags/" + url.PathEscape(tag)
|
||||
if request.RequestURI != wantURI || request.URL.RawQuery != "" || request.URL.Fragment != "" {
|
||||
t.Errorf("tag request = %q query=%q fragment=%q, want %q", request.RequestURI, request.URL.RawQuery, request.URL.Fragment, wantURI)
|
||||
}
|
||||
_, _ = fmt.Fprintf(writer, `{"id":1,"tag_name":%q,"assets":[{"id":2,"name":"dist.zip","state":"uploaded","size":1}]}`, tag)
|
||||
}))
|
||||
defer server.Close()
|
||||
client := newTestClient(t, server.URL, nil)
|
||||
result, err := client.Resolve(context.Background(), ResolveRequest{
|
||||
Repository: "acme/site", Selector: SelectorTag, Tag: tag, AssetName: "dist.zip",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
if result.Release.Tag != tag {
|
||||
t.Fatalf("Release.Tag = %q", result.Release.Tag)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveDoesNotMatchSanitizedRemoteAssetName(t *testing.T) {
|
||||
t.Parallel()
|
||||
tests := []struct {
|
||||
name string
|
||||
remote string
|
||||
requested string
|
||||
}{
|
||||
{name: "unicode line separator", remote: "dist\u2028.zip", requested: "dist?.zip"},
|
||||
{name: "overlong", remote: strings.Repeat("a", 256), requested: strings.Repeat("a", 255)},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = fmt.Fprintf(writer, `{"id":1,"tag_name":"release/v1","assets":[{"id":2,"name":%q,"state":"uploaded","size":1}]}`, test.remote)
|
||||
}))
|
||||
defer server.Close()
|
||||
client := newTestClient(t, server.URL, nil)
|
||||
_, err := client.Resolve(context.Background(), ResolveRequest{
|
||||
Repository: "acme/site", Selector: SelectorLatest, AssetName: test.requested,
|
||||
})
|
||||
if !errors.Is(err, ErrAssetNotFound) {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReleaseDisplayTagValidation(t *testing.T) {
|
||||
t.Parallel()
|
||||
for _, tag := range []string{"release/v1", "release v1", "release/v1#stable&channel=prod"} {
|
||||
if !validReleaseDisplayTag(tag) {
|
||||
t.Errorf("validReleaseDisplayTag(%q) = false", tag)
|
||||
}
|
||||
}
|
||||
for _, tag := range []string{"", strings.Repeat("a", 256), "release\nsecret", "release\u2028secret", "release\u202esecret"} {
|
||||
if validReleaseDisplayTag(tag) {
|
||||
t.Errorf("validReleaseDisplayTag(%q) = true", tag)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveRejectsInvalidUTF8Metadata(t *testing.T) {
|
||||
t.Parallel()
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = writer.Write(append([]byte(`{"id":1,"tag_name":"v1","assets":[{"id":2,"name":"dist`),
|
||||
append([]byte{0xff}, []byte(`.zip","state":"uploaded","size":1}]}`)...)...))
|
||||
}))
|
||||
defer server.Close()
|
||||
client := newTestClient(t, server.URL, nil)
|
||||
_, err := client.Resolve(context.Background(), ResolveRequest{
|
||||
Repository: "acme/site", Selector: SelectorLatest, AssetName: "dist�.zip",
|
||||
})
|
||||
if !errors.Is(err, ErrMetadata) {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadRejectsImpossibleMetadataBeforeNetwork(t *testing.T) {
|
||||
t.Parallel()
|
||||
var requests atomic.Int64
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
|
||||
requests.Add(1)
|
||||
writer.WriteHeader(http.StatusInternalServerError)
|
||||
}))
|
||||
defer server.Close()
|
||||
client := newTestClient(t, server.URL, nil)
|
||||
tests := []struct {
|
||||
name string
|
||||
size int64
|
||||
kind error
|
||||
limit int64
|
||||
}{
|
||||
{name: "negative", size: -1, kind: ErrInvalidRequest, limit: 100},
|
||||
{name: "declared too large", size: 101, kind: ErrAssetTooLarge, limit: 100},
|
||||
}
|
||||
for _, test := range tests {
|
||||
_, err := client.Download(context.Background(), DownloadRequest{
|
||||
Repository: "acme/site",
|
||||
Asset: Asset{ID: "1", Name: "dist?token=hidden#asset.zip", Size: test.size},
|
||||
MaxBytes: test.limit,
|
||||
})
|
||||
if !errors.Is(err, test.kind) {
|
||||
t.Errorf("%s Download() error = %v", test.name, err)
|
||||
}
|
||||
if strings.Contains(err.Error(), "token=hidden") {
|
||||
t.Errorf("%s error leaked asset name: %v", test.name, err)
|
||||
}
|
||||
}
|
||||
if got := requests.Load(); got != 0 {
|
||||
t.Fatalf("HTTP requests = %d, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLogControlCharactersNeverEnterSafeErrors(t *testing.T) {
|
||||
t.Parallel()
|
||||
controls := []string{"\u2028", "\u2029", "\u061c", "\u200e", "\u200f", "\u202e", "\u2066", "\u2069"}
|
||||
for _, control := range controls {
|
||||
secret := "before" + control + "after"
|
||||
err := safeError(errInvalidRequest, 0, secret, secret, secret, secret, nil, nil)
|
||||
message := err.Error()
|
||||
if strings.Contains(message, secret) || strings.Contains(message, control) || strings.Contains(message, "before") {
|
||||
t.Errorf("safe error retained control %U: %q", []rune(control)[0], message)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveRejectsMetadataOverHardLimit(t *testing.T) {
|
||||
t.Parallel()
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = writer.Write([]byte(`{"id":1,"tag_name":"v1","assets":[]}` + strings.Repeat(" ", maxMetadataBytes)))
|
||||
}))
|
||||
defer server.Close()
|
||||
client := newTestClient(t, server.URL, nil)
|
||||
_, err := client.Resolve(context.Background(), ResolveRequest{
|
||||
Repository: "acme/site", Selector: SelectorLatest, AssetName: "dist.zip",
|
||||
})
|
||||
if !errors.Is(err, ErrMetadata) {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProductionTransportRejectsSelfSignedTLS(t *testing.T) {
|
||||
t.Parallel()
|
||||
server := httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = writer.Write([]byte("package"))
|
||||
}))
|
||||
defer server.Close()
|
||||
parsed, _ := url.Parse(server.URL)
|
||||
dialer := &net.Dialer{Timeout: time.Second}
|
||||
client := newClient(clientOptions{
|
||||
baseURL: "https://api.example.test:" + parsed.Port(),
|
||||
resolver: resolverFunc(func(_ context.Context, _ string, _ string) ([]netip.Addr, error) {
|
||||
return []netip.Addr{netip.MustParseAddr("8.8.8.8")}, nil
|
||||
}),
|
||||
dialContext: func(ctx context.Context, network string, _ string) (net.Conn, error) {
|
||||
return dialer.DialContext(ctx, network, server.Listener.Addr().String())
|
||||
},
|
||||
tlsConfig: &tls.Config{MinVersion: tls.VersionTLS12},
|
||||
createTemp: os.CreateTemp,
|
||||
now: time.Now,
|
||||
clientTimeout: 5 * time.Second,
|
||||
})
|
||||
_, err := client.Download(context.Background(), DownloadRequest{
|
||||
Repository: "acme/site", Asset: Asset{ID: "1", Name: "dist.zip"}, MaxBytes: 100,
|
||||
})
|
||||
if !errors.Is(err, ErrDownload) {
|
||||
t.Fatalf("Download() error = %v", err)
|
||||
}
|
||||
if strings.Contains(err.Error(), "api.example.test") || strings.Contains(err.Error(), server.URL) {
|
||||
t.Fatalf("TLS error leaked URL: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStableErrorClassification(t *testing.T) {
|
||||
t.Parallel()
|
||||
now := time.Now()
|
||||
assetMissing := safeError(errAssetMissing, http.StatusOK, "", "acme/site", "v1", "dist.zip", nil, nil)
|
||||
if !IsNotFound(assetMissing) || IsRetryable(assetMissing) {
|
||||
t.Fatalf("asset missing classification failed: %v", assetMissing)
|
||||
}
|
||||
metadata404 := safeError(errMetadata, http.StatusNotFound, "", "acme/site", "v1", "dist.zip", nil, nil)
|
||||
if !IsNotFound(metadata404) || IsRetryable(metadata404) {
|
||||
t.Fatalf("metadata 404 classification failed: %v", metadata404)
|
||||
}
|
||||
digest := safeError(errDigest, http.StatusOK, "", "acme/site", "", "dist.zip", nil, nil)
|
||||
if !IsDigestError(digest) || IsRetryable(digest) {
|
||||
t.Fatalf("digest classification failed: %v", digest)
|
||||
}
|
||||
for _, retryable := range []error{
|
||||
safeError(errMetadata, 0, "", "acme/site", "", "dist.zip", nil, nil),
|
||||
safeError(errMetadata, http.StatusOK, "", "acme/site", "", "dist.zip", nil, nil),
|
||||
safeError(errDownload, http.StatusOK, "", "acme/site", "", "dist.zip", nil, nil),
|
||||
safeError(errMetadata, http.StatusInternalServerError, "", "acme/site", "", "dist.zip", nil, nil),
|
||||
safeError(errMetadata, http.StatusForbidden, "", "acme/site", "", "dist.zip", nil, &now),
|
||||
} {
|
||||
if !IsRetryable(retryable) {
|
||||
t.Errorf("IsRetryable(%v) = false", retryable)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadRedirectLimitIsSafe(t *testing.T) {
|
||||
t.Parallel()
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
step, _ := strconv.Atoi(request.URL.Query().Get("step"))
|
||||
writer.Header().Set("Location", fmt.Sprintf("/repos/acme/site/releases/assets/1?step=%d&token=redirect-secret", step+1))
|
||||
writer.WriteHeader(http.StatusFound)
|
||||
}))
|
||||
defer server.Close()
|
||||
client := newTestClient(t, server.URL, nil)
|
||||
_, err := client.Download(context.Background(), DownloadRequest{
|
||||
Repository: "acme/site", Asset: Asset{ID: "1", Name: "dist.zip"}, MaxBytes: 100,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("Download() error = nil")
|
||||
}
|
||||
if strings.Contains(err.Error(), "redirect-secret") || strings.Contains(err.Error(), "step=") {
|
||||
t.Fatalf("redirect error leaked Location: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInvalidRequestDoesNotEchoURLQueryTagOrAsset(t *testing.T) {
|
||||
t.Parallel()
|
||||
client := NewClient()
|
||||
requests := []ResolveRequest{
|
||||
{Repository: "https://github.com/acme/site?token=repo-secret", Selector: SelectorLatest, AssetName: "dist.zip"},
|
||||
{Repository: "acme/site", Selector: SelectorTag, Tag: "?token=tag-secret", AssetName: "dist.zip"},
|
||||
{Repository: "acme/site", Selector: SelectorLatest, AssetName: "../dist.zip?token=asset-secret"},
|
||||
}
|
||||
for _, request := range requests {
|
||||
_, err := client.Resolve(context.Background(), request)
|
||||
if err == nil {
|
||||
t.Fatalf("Resolve(%+v) error = nil", request)
|
||||
}
|
||||
for _, secret := range []string{"repo-secret", "tag-secret", "asset-secret", "https://github.com"} {
|
||||
if strings.Contains(err.Error(), secret) {
|
||||
t.Fatalf("Resolve(%+v) leaked %q: %v", request, secret, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func newTestClient(t *testing.T, rawBaseURL string, customize func(*clientOptions)) *Client {
|
||||
t.Helper()
|
||||
parsed, err := url.Parse(rawBaseURL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
baseURL := "http://api.example.test:" + parsed.Port()
|
||||
return newMappedTestClient(t, baseURL, customize)
|
||||
}
|
||||
|
||||
func newMappedTestClient(t *testing.T, baseURL string, customize func(*clientOptions)) *Client {
|
||||
t.Helper()
|
||||
resolve := resolverFunc(func(_ context.Context, _ string, _ string) ([]netip.Addr, error) {
|
||||
return []netip.Addr{netip.MustParseAddr("8.8.8.8")}, nil
|
||||
})
|
||||
dialer := &net.Dialer{Timeout: time.Second}
|
||||
options := clientOptions{
|
||||
baseURL: baseURL,
|
||||
resolver: resolve,
|
||||
allowHTTP: true,
|
||||
tlsConfig: &tls.Config{MinVersion: tls.VersionTLS12},
|
||||
dialContext: func(ctx context.Context, network string, address string) (net.Conn, error) {
|
||||
_, port, splitErr := net.SplitHostPort(address)
|
||||
if splitErr != nil {
|
||||
return nil, splitErr
|
||||
}
|
||||
return dialer.DialContext(ctx, network, net.JoinHostPort("127.0.0.1", port))
|
||||
},
|
||||
createTemp: os.CreateTemp,
|
||||
now: time.Now,
|
||||
clientTimeout: 5 * time.Second,
|
||||
}
|
||||
if customize != nil {
|
||||
customize(&options)
|
||||
}
|
||||
return newClient(options)
|
||||
}
|
||||
|
||||
func assertHeader(t *testing.T, request *http.Request, name string, expected string) {
|
||||
t.Helper()
|
||||
if actual := request.Header.Get(name); actual != expected {
|
||||
t.Errorf("%s = %q, want %q", name, actual, expected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResponseRetryAtHTTPDate(t *testing.T) {
|
||||
t.Parallel()
|
||||
want := time.Date(2026, time.July, 19, 12, 30, 0, 0, time.UTC)
|
||||
response := &http.Response{Header: make(http.Header)}
|
||||
response.Header.Set("Retry-After", want.Format(http.TimeFormat))
|
||||
if got := responseRetryAt(response, time.Time{}); got == nil || !got.Equal(want) {
|
||||
t.Fatalf("responseRetryAt() = %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResponseRetryAtRejectsDurationOverflow(t *testing.T) {
|
||||
t.Parallel()
|
||||
response := &http.Response{Header: make(http.Header)}
|
||||
response.Header.Set("Retry-After", strconv.FormatInt(maxRetryAfterSeconds+1, 10))
|
||||
if got := responseRetryAt(response, time.Now()); got != nil {
|
||||
t.Fatalf("responseRetryAt(overflow) = %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSafeETagDropsOversizedOrControlValue(t *testing.T) {
|
||||
t.Parallel()
|
||||
if got := safeETag(strings.Repeat("x", 513)); got != "" {
|
||||
t.Fatalf("safeETag(overlong) = %q", got)
|
||||
}
|
||||
if got := safeETag("ok\nsecret"); got != "" {
|
||||
t.Fatalf("safeETag(control) = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadSizeLimit(t *testing.T) {
|
||||
t.Parallel()
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
|
||||
writer.Header().Set("Content-Length", strconv.Itoa(20))
|
||||
_, _ = writer.Write([]byte(strings.Repeat("x", 20)))
|
||||
}))
|
||||
defer server.Close()
|
||||
client := newTestClient(t, server.URL, nil)
|
||||
_, err := client.Download(context.Background(), DownloadRequest{
|
||||
Repository: "acme/site", Asset: Asset{ID: "1", Name: "dist.zip"}, MaxBytes: 10,
|
||||
})
|
||||
if !errors.Is(err, errTooLarge) {
|
||||
t.Fatalf("Download() error = %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,312 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package githubrelease
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"math"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"Wavelet/pkg/httppool"
|
||||
)
|
||||
|
||||
const (
|
||||
clientTimeout = 10 * time.Minute
|
||||
dialTimeout = 30 * time.Second
|
||||
dialKeepAlive = 30 * time.Second
|
||||
responseHeaderTimeout = 30 * time.Second
|
||||
maxRedirects = 5
|
||||
maxRetryAfterSeconds = math.MaxInt64 / int64(time.Second)
|
||||
)
|
||||
|
||||
var (
|
||||
errBlockedTarget = errors.New("GitHub Release 请求目标不是公网地址")
|
||||
errResolveTarget = errors.New("GitHub Release 请求目标解析失败")
|
||||
errRedirectLimit = errors.New("GitHub Release asset 重定向次数过多")
|
||||
|
||||
publicIPv6Prefix = netip.MustParsePrefix("2000::/3")
|
||||
nonPublicPrefixes = []netip.Prefix{
|
||||
netip.MustParsePrefix("0.0.0.0/8"),
|
||||
netip.MustParsePrefix("10.0.0.0/8"),
|
||||
netip.MustParsePrefix("100.64.0.0/10"),
|
||||
netip.MustParsePrefix("127.0.0.0/8"),
|
||||
netip.MustParsePrefix("169.254.0.0/16"),
|
||||
netip.MustParsePrefix("172.16.0.0/12"),
|
||||
netip.MustParsePrefix("192.0.0.0/24"),
|
||||
netip.MustParsePrefix("192.0.2.0/24"),
|
||||
netip.MustParsePrefix("192.168.0.0/16"),
|
||||
netip.MustParsePrefix("198.18.0.0/15"),
|
||||
netip.MustParsePrefix("198.51.100.0/24"),
|
||||
netip.MustParsePrefix("203.0.113.0/24"),
|
||||
netip.MustParsePrefix("224.0.0.0/4"),
|
||||
netip.MustParsePrefix("240.0.0.0/4"),
|
||||
netip.MustParsePrefix("::/128"),
|
||||
netip.MustParsePrefix("::1/128"),
|
||||
netip.MustParsePrefix("::ffff:0:0/96"),
|
||||
netip.MustParsePrefix("64:ff9b::/96"),
|
||||
netip.MustParsePrefix("100::/64"),
|
||||
netip.MustParsePrefix("2001:db8::/32"),
|
||||
netip.MustParsePrefix("fc00::/7"),
|
||||
netip.MustParsePrefix("fe80::/10"),
|
||||
netip.MustParsePrefix("ff00::/8"),
|
||||
}
|
||||
)
|
||||
|
||||
type resolver interface {
|
||||
LookupNetIP(context.Context, string, string) ([]netip.Addr, error)
|
||||
}
|
||||
|
||||
type clientOptions struct {
|
||||
baseURL string
|
||||
resolver resolver
|
||||
dialContext func(context.Context, string, string) (net.Conn, error)
|
||||
tlsConfig *tls.Config
|
||||
allowHTTP bool
|
||||
createTemp func(string, string) (*os.File, error)
|
||||
now func() time.Time
|
||||
clientTimeout time.Duration
|
||||
}
|
||||
|
||||
func defaultClientOptions() clientOptions {
|
||||
dialer := &net.Dialer{Timeout: dialTimeout, KeepAlive: dialKeepAlive}
|
||||
return clientOptions{
|
||||
baseURL: defaultAPIBaseURL,
|
||||
resolver: net.DefaultResolver,
|
||||
dialContext: dialer.DialContext,
|
||||
tlsConfig: &tls.Config{MinVersion: tls.VersionTLS12},
|
||||
createTemp: os.CreateTemp,
|
||||
now: time.Now,
|
||||
clientTimeout: clientTimeout,
|
||||
}
|
||||
}
|
||||
|
||||
func newClient(options clientOptions) *Client {
|
||||
if options.baseURL == "" {
|
||||
options.baseURL = defaultAPIBaseURL
|
||||
}
|
||||
if options.resolver == nil {
|
||||
options.resolver = net.DefaultResolver
|
||||
}
|
||||
if options.dialContext == nil {
|
||||
dialer := &net.Dialer{Timeout: dialTimeout, KeepAlive: dialKeepAlive}
|
||||
options.dialContext = dialer.DialContext
|
||||
}
|
||||
if options.tlsConfig == nil {
|
||||
options.tlsConfig = &tls.Config{MinVersion: tls.VersionTLS12}
|
||||
}
|
||||
if options.createTemp == nil {
|
||||
options.createTemp = os.CreateTemp
|
||||
}
|
||||
if options.now == nil {
|
||||
options.now = time.Now
|
||||
}
|
||||
if options.clientTimeout <= 0 {
|
||||
options.clientTimeout = clientTimeout
|
||||
}
|
||||
|
||||
secureDial := publicDialer(options.resolver, options.dialContext)
|
||||
transport := httppool.NewTransport(httppool.TransportOptions{
|
||||
Proxy: nil,
|
||||
DialContext: secureDial,
|
||||
TLSClientConfig: options.tlsConfig,
|
||||
ResponseHeaderTimeout: responseHeaderTimeout,
|
||||
TraceFilter: func(request *http.Request) bool {
|
||||
return request.URL == nil || request.URL.RawQuery == ""
|
||||
},
|
||||
})
|
||||
httpClient := &http.Client{Timeout: options.clientTimeout, Transport: transport}
|
||||
httpClient.CheckRedirect = func(next *http.Request, previous []*http.Request) error {
|
||||
if len(previous) > maxRedirects {
|
||||
return errRedirectLimit
|
||||
}
|
||||
if err := validateTarget(next.Context(), next.URL, options.resolver, options.allowHTTP); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(previous) > 0 && !sameHost(previous[len(previous)-1].URL, next.URL) {
|
||||
stripCrossHostHeaders(next)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return &Client{
|
||||
httpClient: httpClient,
|
||||
baseURL: strings.TrimRight(options.baseURL, "/"),
|
||||
createTemp: options.createTemp,
|
||||
now: options.now,
|
||||
}
|
||||
}
|
||||
|
||||
func applyMetadataHeaders(request *http.Request, etag string) {
|
||||
request.Header.Set("Accept", metadataAccept)
|
||||
request.Header.Set("User-Agent", defaultUserAgent)
|
||||
request.Header.Set("X-Github-Api-Version", APIVersion)
|
||||
if etag = safeETag(etag); etag != "" {
|
||||
request.Header.Set("If-None-Match", etag)
|
||||
}
|
||||
}
|
||||
|
||||
func applyAssetHeaders(request *http.Request) {
|
||||
request.Header.Set("Accept", assetAccept)
|
||||
request.Header.Set("Accept-Encoding", "identity")
|
||||
request.Header.Set("User-Agent", defaultUserAgent)
|
||||
request.Header.Set("X-Github-Api-Version", APIVersion)
|
||||
}
|
||||
|
||||
func stripCrossHostHeaders(request *http.Request) {
|
||||
for _, header := range []string{
|
||||
"Authorization",
|
||||
"Cookie",
|
||||
"Proxy-Authorization",
|
||||
"Referer",
|
||||
"If-None-Match",
|
||||
"If-Modified-Since",
|
||||
"X-GitHub-Api-Version",
|
||||
} {
|
||||
request.Header.Del(header)
|
||||
}
|
||||
}
|
||||
|
||||
func sameHost(left *url.URL, right *url.URL) bool {
|
||||
if left == nil || right == nil {
|
||||
return false
|
||||
}
|
||||
return strings.EqualFold(left.Hostname(), right.Hostname()) && effectivePort(left) == effectivePort(right)
|
||||
}
|
||||
|
||||
func effectivePort(target *url.URL) string {
|
||||
if port := target.Port(); port != "" {
|
||||
return port
|
||||
}
|
||||
if strings.EqualFold(target.Scheme, "https") {
|
||||
return "443"
|
||||
}
|
||||
return "80"
|
||||
}
|
||||
|
||||
func validateTarget(ctx context.Context, target *url.URL, targetResolver resolver, allowHTTP bool) error {
|
||||
if target == nil || target.User != nil || target.Fragment != "" || target.Opaque != "" || target.Hostname() == "" {
|
||||
return errBlockedTarget
|
||||
}
|
||||
isHTTPS := strings.EqualFold(target.Scheme, "https")
|
||||
isAllowedHTTP := allowHTTP && strings.EqualFold(target.Scheme, "http")
|
||||
if !isHTTPS && !isAllowedHTTP {
|
||||
return errBlockedTarget
|
||||
}
|
||||
_, err := resolvePublicIPs(ctx, targetResolver, target.Hostname())
|
||||
return err
|
||||
}
|
||||
|
||||
func publicDialer(
|
||||
targetResolver resolver,
|
||||
directDial func(context.Context, string, string) (net.Conn, error),
|
||||
) func(context.Context, string, string) (net.Conn, error) {
|
||||
return func(ctx context.Context, network string, address string) (net.Conn, error) {
|
||||
host, port, err := net.SplitHostPort(address)
|
||||
if err != nil {
|
||||
return nil, errResolveTarget
|
||||
}
|
||||
addresses, err := resolvePublicIPs(ctx, targetResolver, host)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, resolved := range addresses {
|
||||
if !ipMatchesNetwork(resolved, network) {
|
||||
continue
|
||||
}
|
||||
connection, dialErr := directDial(ctx, network, net.JoinHostPort(resolved.String(), port))
|
||||
if dialErr == nil {
|
||||
return connection, nil
|
||||
}
|
||||
}
|
||||
return nil, errDownload
|
||||
}
|
||||
}
|
||||
|
||||
func resolvePublicIPs(ctx context.Context, targetResolver resolver, host string) ([]netip.Addr, error) {
|
||||
if strings.Contains(host, "%") {
|
||||
return nil, errBlockedTarget
|
||||
}
|
||||
if literal, err := netip.ParseAddr(host); err == nil {
|
||||
if !isPublicIP(literal) {
|
||||
return nil, errBlockedTarget
|
||||
}
|
||||
return []netip.Addr{literal}, nil
|
||||
}
|
||||
if targetResolver == nil {
|
||||
return nil, errResolveTarget
|
||||
}
|
||||
addresses, err := targetResolver.LookupNetIP(ctx, "ip", host)
|
||||
if err != nil || len(addresses) == 0 {
|
||||
return nil, errResolveTarget
|
||||
}
|
||||
for _, address := range addresses {
|
||||
if !isPublicIP(address) {
|
||||
return nil, errBlockedTarget
|
||||
}
|
||||
}
|
||||
return addresses, nil
|
||||
}
|
||||
|
||||
func isPublicIP(address netip.Addr) bool {
|
||||
if !address.IsValid() || address.Zone() != "" {
|
||||
return false
|
||||
}
|
||||
address = address.Unmap()
|
||||
if !address.IsGlobalUnicast() {
|
||||
return false
|
||||
}
|
||||
if address.Is6() && !publicIPv6Prefix.Contains(address) {
|
||||
return false
|
||||
}
|
||||
for _, prefix := range nonPublicPrefixes {
|
||||
if prefix.Contains(address) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func ipMatchesNetwork(address netip.Addr, network string) bool {
|
||||
switch network {
|
||||
case "tcp4":
|
||||
return address.Unmap().Is4()
|
||||
case "tcp6":
|
||||
return address.Unmap().Is6()
|
||||
default:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
func responseRetryAt(response *http.Response, now time.Time) *time.Time {
|
||||
if response == nil {
|
||||
return nil
|
||||
}
|
||||
if retryAfter := strings.TrimSpace(response.Header.Get("Retry-After")); retryAfter != "" {
|
||||
if seconds, err := strconv.ParseInt(retryAfter, 10, 64); err == nil && seconds >= 0 && seconds <= maxRetryAfterSeconds {
|
||||
retryAt := now.Add(time.Duration(seconds) * time.Second)
|
||||
return &retryAt
|
||||
}
|
||||
if retryAt, err := http.ParseTime(retryAfter); err == nil {
|
||||
retryAt = retryAt.UTC()
|
||||
return &retryAt
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(response.Header.Get("X-Ratelimit-Remaining")) != "0" {
|
||||
return nil
|
||||
}
|
||||
reset, err := strconv.ParseInt(strings.TrimSpace(response.Header.Get("X-Ratelimit-Reset")), 10, 64)
|
||||
if err != nil || reset <= 0 {
|
||||
return nil
|
||||
}
|
||||
retryAt := time.Unix(reset, 0).UTC()
|
||||
return &retryAt
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Package analytics defines ClickHouse analytics domain models and query DTOs
|
||||
// (pure data, no IO).
|
||||
package analytics
|
||||
|
||||
import "time"
|
||||
|
||||
// AccessLogFilter scopes user access log queries.
|
||||
// 单一权威字段集(CH 原字段,Task 1 迁入):禁止追加仅某实现使用的字段(避免双字段集分叉)。
|
||||
type AccessLogFilter struct {
|
||||
// UserIDs filters by user IDs. nil means no user filter; an empty slice means no matches.
|
||||
UserIDs []uint64
|
||||
Path string
|
||||
// StartTime filters created_at >= StartTime when non-nil.
|
||||
StartTime *time.Time
|
||||
// EndTime filters created_at <= EndTime when non-nil(闭区间,与 CH/GORM 实现一致)。
|
||||
EndTime *time.Time
|
||||
}
|
||||
|
||||
// NodeAccessLogFilter scopes ClickHouse node access log queries.
|
||||
type NodeAccessLogFilter struct {
|
||||
NodeID string
|
||||
RemoteAddr string
|
||||
Host string
|
||||
// Hosts exact-matches any host (case-insensitive). Prefer over Host for multi-domain scopes.
|
||||
Hosts []string
|
||||
Path string
|
||||
// StatusCode filters by exact HTTP status code when > 0.
|
||||
StatusCode int
|
||||
Since time.Time
|
||||
Until time.Time
|
||||
Page int
|
||||
PageSize int
|
||||
SortBy string
|
||||
SortOrder string
|
||||
}
|
||||
|
||||
// NodeObservabilityFilter scopes ClickHouse node observability queries.
|
||||
type NodeObservabilityFilter struct {
|
||||
NodeID string
|
||||
Since time.Time
|
||||
Limit int
|
||||
}
|
||||
|
||||
// DailyTrend is a single day's access count.
|
||||
type DailyTrend struct {
|
||||
Date string
|
||||
Count uint64
|
||||
}
|
||||
|
||||
// BrowserShare is a browser group's share of access logs.
|
||||
type BrowserShare struct {
|
||||
Browser string
|
||||
Count uint64
|
||||
}
|
||||
|
||||
// TopUser is an active user ranked by access count.
|
||||
type TopUser struct {
|
||||
UserID uint64
|
||||
Count uint64
|
||||
}
|
||||
|
||||
// NodeAccessLogRegionCount aggregates access log regions.
|
||||
type NodeAccessLogRegionCount struct {
|
||||
Region string
|
||||
Count int64
|
||||
}
|
||||
|
||||
// NodeAccessLogTrafficSummary is a window-level access log traffic summary.
|
||||
type NodeAccessLogTrafficSummary struct {
|
||||
RequestCount int64
|
||||
ErrorCount int64
|
||||
UniqueIPCount int64
|
||||
BytesSent int64
|
||||
RequestLength int64
|
||||
NodeCount int64
|
||||
}
|
||||
|
||||
// NodeAccessLogValueCount is a grouped value count (status_code, host, ...).
|
||||
type NodeAccessLogValueCount struct {
|
||||
Value string
|
||||
Count int64
|
||||
}
|
||||
|
||||
// NodeAccessLogNodeAggregate is per-node traffic over a window.
|
||||
type NodeAccessLogNodeAggregate struct {
|
||||
NodeID string
|
||||
RequestCount int64
|
||||
ErrorCount int64
|
||||
UniqueIPCount int64
|
||||
}
|
||||
|
||||
// BatchWriterStats is a point-in-time snapshot of a batch writer queue and failure counters.
|
||||
type BatchWriterStats struct {
|
||||
Name string `json:"name"`
|
||||
Depth int `json:"depth"`
|
||||
Cap int `json:"cap"`
|
||||
Drops int64 `json:"drops"`
|
||||
FlushErrors int64 `json:"flush_errors"`
|
||||
Running bool `json:"running"`
|
||||
}
|
||||
|
||||
// ClickHouseOperationalStats summarizes ClickHouse merge/mutation pressure
|
||||
// and in-process batch writer queue health.
|
||||
type ClickHouseOperationalStats struct {
|
||||
Database string `json:"database"`
|
||||
ActiveParts int64 `json:"active_parts"`
|
||||
TotalRows int64 `json:"total_rows"`
|
||||
PendingMutations int64 `json:"pending_mutations"`
|
||||
AsyncInsertQueue int64 `json:"async_insert_queue"`
|
||||
AsyncInsertBytes int64 `json:"async_insert_bytes"`
|
||||
// BatchWriters reports in-process queue depth/drops/flush errors for CH writers.
|
||||
BatchWriters []BatchWriterStats `json:"batch_writers,omitempty"`
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
nodeAccessLogTableName = "of_node_access_logs"
|
||||
nodeAccessLogInsertColumns = "id, node_id, logged_at, remote_addr, region, host, path, user_agent, cache_status, status_code, bytes_sent, request_length, request_time_ms, created_at"
|
||||
)
|
||||
|
||||
// NodeAccessLog stores OpenFlare edge node access records in ClickHouse.
|
||||
type NodeAccessLog struct {
|
||||
ID uint64 `gorm:"column:id"`
|
||||
NodeID string `gorm:"column:node_id"`
|
||||
LoggedAt time.Time `gorm:"column:logged_at"`
|
||||
RemoteAddr string `gorm:"column:remote_addr"`
|
||||
Region string `gorm:"column:region"`
|
||||
Host string `gorm:"column:host"`
|
||||
Path string `gorm:"column:path"`
|
||||
UserAgent string `gorm:"column:user_agent"`
|
||||
CacheStatus string `gorm:"column:cache_status"`
|
||||
StatusCode int32 `gorm:"column:status_code"`
|
||||
BytesSent uint64 `gorm:"column:bytes_sent"`
|
||||
RequestLength uint64 `gorm:"column:request_length"`
|
||||
RequestTimeMs uint32 `gorm:"column:request_time_ms"`
|
||||
CreatedAt time.Time `gorm:"column:created_at"`
|
||||
}
|
||||
|
||||
// TableName returns the ClickHouse table name.
|
||||
func (NodeAccessLog) TableName() string {
|
||||
return nodeAccessLogTableName
|
||||
}
|
||||
|
||||
// InsertColumns returns comma-separated column names for batch insert.
|
||||
func (NodeAccessLog) InsertColumns() string {
|
||||
return nodeAccessLogInsertColumns
|
||||
}
|
||||
|
||||
// BatchInsertSQL returns the INSERT prefix used by native batch writers.
|
||||
func (NodeAccessLog) BatchInsertSQL() string {
|
||||
return fmt.Sprintf("INSERT INTO %s (%s)", nodeAccessLogTableName, nodeAccessLogInsertColumns)
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
// NodeAccessLogBucketAggregate is a folded bucket aggregate row.
|
||||
type NodeAccessLogBucketAggregate struct {
|
||||
BucketEpoch int64 `gorm:"column:bucket_epoch"`
|
||||
RequestCount int64 `gorm:"column:request_count"`
|
||||
SuccessCount int64 `gorm:"column:success_count"`
|
||||
ClientErrorCount int64 `gorm:"column:client_error_count"`
|
||||
ServerErrorCount int64 `gorm:"column:server_error_count"`
|
||||
Status2xxCount int64 `gorm:"column:status_2xx_count"`
|
||||
Status4xxCount int64 `gorm:"column:status_4xx_count"`
|
||||
Status5xxCount int64 `gorm:"column:status_5xx_count"`
|
||||
UniqueIPCount int64 `gorm:"column:unique_ip_count"`
|
||||
UniqueHostCount int64 `gorm:"column:unique_host_count"`
|
||||
BytesSent int64 `gorm:"column:bytes_sent"`
|
||||
RequestLength int64 `gorm:"column:request_length"`
|
||||
}
|
||||
|
||||
// NodeAccessLogWAFIPAggregate is a per-IP aggregate row for WAF automatic rules.
|
||||
type NodeAccessLogWAFIPAggregate struct {
|
||||
RemoteAddr string
|
||||
RequestCount int64
|
||||
Status404Count int64
|
||||
ClientErrorCount int64
|
||||
ServerErrorCount int64
|
||||
IPHostCount int64
|
||||
LastSeenEpoch int64
|
||||
StatusCounts map[int]int64
|
||||
}
|
||||
|
||||
// NodeAccessLogBucketDimension is a bucket dimension value.
|
||||
type NodeAccessLogBucketDimension struct {
|
||||
BucketEpoch int64 `gorm:"column:bucket_epoch"`
|
||||
Value string `gorm:"column:value"`
|
||||
}
|
||||
|
||||
// NodeAccessLogIPAggregate is an IP aggregate row.
|
||||
type NodeAccessLogIPAggregate struct {
|
||||
RemoteAddr string `gorm:"column:remote_addr"`
|
||||
RequestCount int64 `gorm:"column:request_count"`
|
||||
SuccessCount int64 `gorm:"column:success_count"`
|
||||
ClientErrorCount int64 `gorm:"column:client_error_count"`
|
||||
ServerErrorCount int64 `gorm:"column:server_error_count"`
|
||||
LastSeenEpoch int64 `gorm:"column:last_seen_epoch"`
|
||||
}
|
||||
|
||||
// NodeAccessLogIPSummary is an IP summary row.
|
||||
type NodeAccessLogIPSummary struct {
|
||||
RemoteAddr string `gorm:"column:remote_addr"`
|
||||
Region string `gorm:"column:region"`
|
||||
TotalRequests int64 `gorm:"column:total_requests"`
|
||||
Success2xxCount int64 `gorm:"column:success_2xx_count"`
|
||||
SuccessRatio float64 `gorm:"column:success_ratio"`
|
||||
BytesReceived int64 `gorm:"column:request_length"`
|
||||
BytesSent int64 `gorm:"column:bytes_sent"`
|
||||
// RecentRequests is deprecated (always 0); kept for wire compatibility.
|
||||
RecentRequests int64 `gorm:"column:recent_requests"`
|
||||
LastSeenEpoch int64 `gorm:"column:last_seen_epoch"`
|
||||
}
|
||||
|
||||
// NodeAccessLogIPTrend is an IP trend bucket row.
|
||||
type NodeAccessLogIPTrend struct {
|
||||
BucketEpoch int64 `gorm:"column:bucket_epoch"`
|
||||
RequestCount int64 `gorm:"column:request_count"`
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
nodeMetricSnapshotTableName = "of_node_metric_snapshots"
|
||||
nodeMetricSnapshotInsertColumns = "id, node_id, captured_at, cpu_usage_percent, memory_used_bytes, memory_total_bytes, storage_used_bytes, storage_total_bytes, disk_read_bytes, disk_write_bytes, network_rx_bytes, network_tx_bytes, created_at"
|
||||
|
||||
nodeEdgeHealthTableName = "of_node_edge_health"
|
||||
nodeEdgeHealthInsertColumns = "id, node_id, captured_at, status, connections, created_at"
|
||||
|
||||
nodeObsFrpsTableName = "of_node_obs_frps"
|
||||
nodeObsFrpsInsertColumns = "id, node_id, captured_at, frps_connections, frps_proxy_count, frps_client_count, frps_proxies, created_at"
|
||||
|
||||
nodeObsFrpcTableName = "of_node_obs_frpc"
|
||||
nodeObsFrpcInsertColumns = "id, node_id, captured_at, tunnel_status, connected_relays_count, created_at"
|
||||
)
|
||||
|
||||
// NodeMetricSnapshot stores periodic node resource utilization metrics in ClickHouse.
|
||||
type NodeMetricSnapshot struct {
|
||||
ID uint64 `gorm:"column:id"`
|
||||
NodeID string `gorm:"column:node_id"`
|
||||
CapturedAt time.Time `gorm:"column:captured_at"`
|
||||
CPUUsagePercent float64 `gorm:"column:cpu_usage_percent"`
|
||||
MemoryUsedBytes int64 `gorm:"column:memory_used_bytes"`
|
||||
MemoryTotalBytes int64 `gorm:"column:memory_total_bytes"`
|
||||
StorageUsedBytes int64 `gorm:"column:storage_used_bytes"`
|
||||
StorageTotalBytes int64 `gorm:"column:storage_total_bytes"`
|
||||
DiskReadBytes int64 `gorm:"column:disk_read_bytes"`
|
||||
DiskWriteBytes int64 `gorm:"column:disk_write_bytes"`
|
||||
NetworkRxBytes int64 `gorm:"column:network_rx_bytes"`
|
||||
NetworkTxBytes int64 `gorm:"column:network_tx_bytes"`
|
||||
CreatedAt time.Time `gorm:"column:created_at"`
|
||||
}
|
||||
|
||||
// TableName returns the ClickHouse table name.
|
||||
func (NodeMetricSnapshot) TableName() string {
|
||||
return nodeMetricSnapshotTableName
|
||||
}
|
||||
|
||||
// InsertColumns returns comma-separated column names for batch insert.
|
||||
func (NodeMetricSnapshot) InsertColumns() string {
|
||||
return nodeMetricSnapshotInsertColumns
|
||||
}
|
||||
|
||||
// BatchInsertSQL returns the INSERT prefix used by native batch writers.
|
||||
func (NodeMetricSnapshot) BatchInsertSQL() string {
|
||||
return fmt.Sprintf("INSERT INTO %s (%s)", nodeMetricSnapshotTableName, nodeMetricSnapshotInsertColumns)
|
||||
}
|
||||
|
||||
// NodeEdgeHealth stores L2 OpenResty health snapshots (connections + status).
|
||||
type NodeEdgeHealth struct {
|
||||
ID uint64 `gorm:"column:id"`
|
||||
NodeID string `gorm:"column:node_id"`
|
||||
CapturedAt time.Time `gorm:"column:captured_at"`
|
||||
Status string `gorm:"column:status"`
|
||||
Connections int64 `gorm:"column:connections"`
|
||||
CreatedAt time.Time `gorm:"column:created_at"`
|
||||
}
|
||||
|
||||
// TableName returns the ClickHouse table name.
|
||||
func (NodeEdgeHealth) TableName() string {
|
||||
return nodeEdgeHealthTableName
|
||||
}
|
||||
|
||||
// InsertColumns returns comma-separated column names for batch insert.
|
||||
func (NodeEdgeHealth) InsertColumns() string {
|
||||
return nodeEdgeHealthInsertColumns
|
||||
}
|
||||
|
||||
// BatchInsertSQL returns the INSERT prefix used by native batch writers.
|
||||
func (NodeEdgeHealth) BatchInsertSQL() string {
|
||||
return fmt.Sprintf("INSERT INTO %s (%s)", nodeEdgeHealthTableName, nodeEdgeHealthInsertColumns)
|
||||
}
|
||||
|
||||
// AccessLogHourly is a Server-side hourly rollup of access logs.
|
||||
type AccessLogHourly struct {
|
||||
NodeID string `gorm:"column:node_id"`
|
||||
Hour time.Time `gorm:"column:hour"`
|
||||
Host string `gorm:"column:host"`
|
||||
RequestCount int64 `gorm:"column:request_count"`
|
||||
ErrorCount int64 `gorm:"column:error_count"`
|
||||
BytesSent int64 `gorm:"column:bytes_sent"`
|
||||
RequestLength int64 `gorm:"column:request_length"`
|
||||
}
|
||||
|
||||
// NodeTrafficHourly is an hourly traffic rollup row.
|
||||
//
|
||||
// UniqueVisitorCount is always 0 when sourced from of_access_log_hourly
|
||||
// (true UV requires raw uniqExact on access logs).
|
||||
type NodeTrafficHourly struct {
|
||||
NodeID string
|
||||
Hour time.Time
|
||||
RequestCount int64
|
||||
ErrorCount int64
|
||||
UniqueVisitorCount int64
|
||||
}
|
||||
|
||||
// NodeMetricHourly is an hourly metric snapshot aggregation row.
|
||||
//
|
||||
// Disk and host network counters are cumulative. Prefer pre-aggregated min/max
|
||||
// deltas from of_node_metric_capacity_hourly; raw fallback uses consecutive
|
||||
// lagInFrame samples per node (negative deltas after counter reset are dropped).
|
||||
type NodeMetricHourly struct {
|
||||
Hour time.Time
|
||||
AverageCPUUsagePercent float64
|
||||
AverageMemoryUsagePercent float64
|
||||
NetworkRxBytes int64
|
||||
NetworkTxBytes int64
|
||||
DiskReadBytes int64
|
||||
DiskWriteBytes int64
|
||||
ReportedNodes int
|
||||
}
|
||||
|
||||
// NodeObsFrps stores FRPS observability snapshots in ClickHouse.
|
||||
type NodeObsFrps struct {
|
||||
ID uint64 `gorm:"column:id"`
|
||||
NodeID string `gorm:"column:node_id"`
|
||||
CapturedAt time.Time `gorm:"column:captured_at"`
|
||||
FrpsConnections int32 `gorm:"column:frps_connections"`
|
||||
FrpsProxyCount int32 `gorm:"column:frps_proxy_count"`
|
||||
FrpsClientCount int32 `gorm:"column:frps_client_count"`
|
||||
FrpsProxies string `gorm:"column:frps_proxies"`
|
||||
CreatedAt time.Time `gorm:"column:created_at"`
|
||||
}
|
||||
|
||||
// TableName returns the ClickHouse table name.
|
||||
func (NodeObsFrps) TableName() string {
|
||||
return nodeObsFrpsTableName
|
||||
}
|
||||
|
||||
// InsertColumns returns comma-separated column names for batch insert.
|
||||
func (NodeObsFrps) InsertColumns() string {
|
||||
return nodeObsFrpsInsertColumns
|
||||
}
|
||||
|
||||
// BatchInsertSQL returns the INSERT prefix used by native batch writers.
|
||||
func (NodeObsFrps) BatchInsertSQL() string {
|
||||
return fmt.Sprintf("INSERT INTO %s (%s)", nodeObsFrpsTableName, nodeObsFrpsInsertColumns)
|
||||
}
|
||||
|
||||
// NodeObsFrpc stores FRPC observability snapshots in ClickHouse.
|
||||
type NodeObsFrpc struct {
|
||||
ID uint64 `gorm:"column:id"`
|
||||
NodeID string `gorm:"column:node_id"`
|
||||
CapturedAt time.Time `gorm:"column:captured_at"`
|
||||
TunnelStatus string `gorm:"column:tunnel_status"`
|
||||
ConnectedRelaysCount int32 `gorm:"column:connected_relays_count"`
|
||||
CreatedAt time.Time `gorm:"column:created_at"`
|
||||
}
|
||||
|
||||
// TableName returns the ClickHouse table name.
|
||||
func (NodeObsFrpc) TableName() string {
|
||||
return nodeObsFrpcTableName
|
||||
}
|
||||
|
||||
// InsertColumns returns comma-separated column names for batch insert.
|
||||
func (NodeObsFrpc) InsertColumns() string {
|
||||
return nodeObsFrpcInsertColumns
|
||||
}
|
||||
|
||||
// BatchInsertSQL returns the INSERT prefix used by native batch writers.
|
||||
func (NodeObsFrpc) BatchInsertSQL() string {
|
||||
return fmt.Sprintf("INSERT INTO %s (%s)", nodeObsFrpcTableName, nodeObsFrpcInsertColumns)
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
risklogstore "Wavelet/plugins/domain/risk_control/logstore"
|
||||
)
|
||||
|
||||
// UserAccessLog is Wavelet risk_control's w_user_access_logs entity.
|
||||
type UserAccessLog = risklogstore.UserAccessLog
|
||||
@@ -0,0 +1,124 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import "strings"
|
||||
|
||||
// User-Agent 浏览器/OS/设备分类(纯函数,无 IO)。
|
||||
// 与 internal/repository/analytics/browser.go 的判定逻辑保持一致(Task 4 复制,
|
||||
// 因为 model 不得 import analyticsrepo);后续若移除旧 CH 实现,可让 analyticsrepo 改以别名复用本包。
|
||||
|
||||
const (
|
||||
uaLabelUnknown = "Unknown"
|
||||
uaLabelBot = "Bot"
|
||||
uaLabelOther = "Other"
|
||||
uaTokenBot = "bot"
|
||||
uaTokenAndroid = "android"
|
||||
uaTokenSpider = "spider"
|
||||
uaTokenCrawler = "crawler"
|
||||
)
|
||||
|
||||
type uaMatchRule struct {
|
||||
label string
|
||||
contains []string
|
||||
allOf []string
|
||||
noneOf []string
|
||||
}
|
||||
|
||||
func matchUARules(uaLower string, rules []uaMatchRule, fallback string) string {
|
||||
if uaLower == "" {
|
||||
return uaLabelUnknown
|
||||
}
|
||||
for _, rule := range rules {
|
||||
matched := false
|
||||
for _, token := range rule.contains {
|
||||
if strings.Contains(uaLower, token) {
|
||||
matched = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !matched && len(rule.allOf) > 0 {
|
||||
matched = true
|
||||
for _, token := range rule.allOf {
|
||||
if !strings.Contains(uaLower, token) {
|
||||
matched = false
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if !matched {
|
||||
continue
|
||||
}
|
||||
excluded := false
|
||||
for _, token := range rule.noneOf {
|
||||
if strings.Contains(uaLower, token) {
|
||||
excluded = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if excluded {
|
||||
continue
|
||||
}
|
||||
return rule.label
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
var browserRules = []uaMatchRule{
|
||||
{label: "WeChat", contains: []string{"micromessenger"}},
|
||||
{label: "Postman", contains: []string{"postman"}},
|
||||
{label: "CLI", contains: []string{"curl/", "wget/"}},
|
||||
{label: "Edge", contains: []string{"edg/", "edgios/", "edga/"}},
|
||||
{label: "Opera", contains: []string{"opr/", "opera"}},
|
||||
{label: "Firefox", contains: []string{"firefox", "fxios"}},
|
||||
{label: "Chrome", contains: []string{"crios", "chrome"}, noneOf: []string{"chromium"}},
|
||||
{label: "Chromium", contains: []string{"chromium"}},
|
||||
{label: "Safari", contains: []string{"safari"}},
|
||||
{label: uaLabelBot, contains: []string{uaTokenBot, uaTokenSpider, uaTokenCrawler, "slurp"}},
|
||||
}
|
||||
|
||||
var osRules = []uaMatchRule{
|
||||
{label: "Android", contains: []string{uaTokenAndroid}},
|
||||
{label: "iOS", contains: []string{"iphone", "ipad", "ipod", "ios"}},
|
||||
{label: "Windows", contains: []string{"windows"}},
|
||||
{label: "macOS", contains: []string{"mac os x", "macintosh", "macos"}},
|
||||
{label: "Chrome OS", contains: []string{"cros"}},
|
||||
{label: "Linux", contains: []string{"linux"}},
|
||||
{label: uaLabelBot, contains: []string{uaTokenBot, uaTokenSpider, uaTokenCrawler}},
|
||||
}
|
||||
|
||||
var deviceRules = []uaMatchRule{
|
||||
{
|
||||
label: uaLabelBot,
|
||||
contains: []string{uaTokenBot, uaTokenSpider, uaTokenCrawler, "slurp", "curl/", "wget/", "python-requests", "go-http-client", "postman"},
|
||||
},
|
||||
{
|
||||
label: "Tablet",
|
||||
contains: []string{"ipad", "tablet"},
|
||||
},
|
||||
{
|
||||
label: "Tablet",
|
||||
allOf: []string{uaTokenAndroid},
|
||||
noneOf: []string{"mobile"},
|
||||
},
|
||||
{
|
||||
label: "Mobile",
|
||||
contains: []string{"mobi", "iphone", "ipod", uaTokenAndroid},
|
||||
},
|
||||
}
|
||||
|
||||
// ParseBrowserName performs lightweight User-Agent browser identification.
|
||||
func ParseBrowserName(ua string) string {
|
||||
return matchUARules(strings.ToLower(ua), browserRules, uaLabelOther)
|
||||
}
|
||||
|
||||
// ParseOSName performs lightweight User-Agent OS identification.
|
||||
func ParseOSName(ua string) string {
|
||||
return matchUARules(strings.ToLower(ua), osRules, uaLabelOther)
|
||||
}
|
||||
|
||||
// ParseDeviceType performs lightweight User-Agent device type identification.
|
||||
func ParseDeviceType(ua string) string {
|
||||
return matchUARules(strings.ToLower(ua), deviceRules, "Desktop")
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package model
|
||||
|
||||
// Domain validation messages used by model.Validate and other no-IO rules.
|
||||
// Persistence / data-access messages belong in internal/repository (do not import repository).
|
||||
const (
|
||||
errTemplateKeyRequired = "模板标识符不能为空"
|
||||
errTemplateNameRequired = "模板名称不能为空"
|
||||
errTemplateContentRequired = "模板内容不能为空"
|
||||
errAuthSourceNameRequired = "认证源名称不能为空"
|
||||
errAuthSourceNameInvalid = "认证源名称只能包含字母、数字、短横线或下划线,且必须以字母或数字开头"
|
||||
errAuthSourceTypeUnsupported = "认证源类型仅支持 oidc"
|
||||
errAuthSourceDiscoveryURLRequired = "OIDC 认证源必须配置 Discovery URL"
|
||||
errAuthSourceClientCredentialsRequired = "启用认证源前必须配置 Client ID 和 Client Secret" //nolint:gosec // false positive: this is an error message, not hardcoded credentials
|
||||
)
|
||||
@@ -0,0 +1,28 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package model
|
||||
|
||||
// OpenFlareAccessLogTrafficSummary is a window-level traffic summary from access logs.
|
||||
type OpenFlareAccessLogTrafficSummary struct {
|
||||
RequestCount int64
|
||||
ErrorCount int64
|
||||
UniqueIPCount int64
|
||||
BytesSent int64
|
||||
RequestLength int64
|
||||
NodeCount int64
|
||||
}
|
||||
|
||||
// OpenFlareAccessLogValueCount is a dimension value count.
|
||||
type OpenFlareAccessLogValueCount struct {
|
||||
Value string
|
||||
Count int64
|
||||
}
|
||||
|
||||
// OpenFlareAccessLogNodeAggregate is per-node traffic over a window.
|
||||
type OpenFlareAccessLogNodeAggregate struct {
|
||||
NodeID string
|
||||
RequestCount int64
|
||||
ErrorCount int64
|
||||
UniqueIPCount int64
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package model
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// AcmeAccount OpenFlare ACME 账号实体。
|
||||
type AcmeAccount struct {
|
||||
ID uint `json:"id" gorm:"primaryKey;autoIncrement"`
|
||||
Email string `json:"email" gorm:"size:255"`
|
||||
URL string `json:"url" gorm:"size:255"`
|
||||
PrivateKey string `json:"-" gorm:"type:text;not null"`
|
||||
CreatedAt time.Time `json:"created_at" gorm:"autoCreateTime"`
|
||||
UpdatedAt time.Time `json:"updated_at" gorm:"autoUpdateTime"`
|
||||
}
|
||||
|
||||
// TableName 表名。
|
||||
func (AcmeAccount) TableName() string {
|
||||
return "of_acme_accounts"
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package model
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// OpenFlareApplyLogQuery filters apply logs for list queries.
|
||||
type OpenFlareApplyLogQuery struct {
|
||||
NodeID string
|
||||
PageNo int
|
||||
PageSize int
|
||||
}
|
||||
|
||||
// OpenFlareApplyLog stores node configuration apply results.
|
||||
type OpenFlareApplyLog struct {
|
||||
ID uint `json:"id" gorm:"primaryKey;autoIncrement"`
|
||||
NodeID string `json:"node_id" gorm:"index;size:64;not null"`
|
||||
Version string `json:"version" gorm:"size:32;not null"`
|
||||
Result string `json:"result" gorm:"size:32;not null"`
|
||||
Message string `json:"message" gorm:"type:text"`
|
||||
Checksum string `json:"checksum" gorm:"size:64;not null;default:''"`
|
||||
MainConfigChecksum string `json:"main_config_checksum" gorm:"size:64;not null;default:''"`
|
||||
RouteConfigChecksum string `json:"route_config_checksum" gorm:"size:64;not null;default:''"`
|
||||
SupportFileCount int `json:"support_file_count" gorm:"not null;default:0"`
|
||||
CreatedAt time.Time `json:"created_at" gorm:"autoCreateTime;index"`
|
||||
}
|
||||
|
||||
// TableName returns the GORM table name.
|
||||
func (OpenFlareApplyLog) TableName() string {
|
||||
return "of_apply_logs"
|
||||
}
|
||||
|
||||
// IsRepeatSuccessApplyLog reports whether the payload repeats an already-recorded success entry.
|
||||
func IsRepeatSuccessApplyLog(latest *OpenFlareApplyLog, version, checksum, result string) bool {
|
||||
if latest == nil || result != "success" {
|
||||
return false
|
||||
}
|
||||
return latest.Result == "success" &&
|
||||
strings.TrimSpace(latest.Version) == strings.TrimSpace(version) &&
|
||||
strings.TrimSpace(latest.Checksum) == strings.TrimSpace(checksum)
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package model
|
||||
|
||||
import "time"
|
||||
|
||||
const (
|
||||
// CFConnectionSourceDNSAccount imports credentials from an existing DNS account.
|
||||
CFConnectionSourceDNSAccount = "dns_account"
|
||||
// CFConnectionSourceStandalone stores an independent API token.
|
||||
CFConnectionSourceStandalone = "standalone"
|
||||
|
||||
// CFConnectionStatusReady indicates the credential passed verification.
|
||||
CFConnectionStatusReady = "ready"
|
||||
// CFConnectionStatusError indicates the latest verification failed.
|
||||
CFConnectionStatusError = "error"
|
||||
|
||||
// CFMemberSyncPending indicates synchronization is queued or required.
|
||||
CFMemberSyncPending = "pending"
|
||||
// CFMemberSyncing indicates a worker is reconciling the record.
|
||||
CFMemberSyncing = "syncing"
|
||||
// CFMemberSyncOK indicates the remote record matches the desired state.
|
||||
CFMemberSyncOK = "ok"
|
||||
// CFMemberSyncError indicates the latest reconciliation failed.
|
||||
CFMemberSyncError = "error"
|
||||
)
|
||||
|
||||
// CFConnection stores the single Cloudflare API credential source.
|
||||
type CFConnection struct {
|
||||
ID uint `json:"id" gorm:"primaryKey;autoIncrement"`
|
||||
Source string `json:"source" gorm:"size:32;not null;default:''"`
|
||||
DNSAccountID *uint `json:"dns_account_id" gorm:"index:idx_of_cf_connections_dns_account_id"`
|
||||
Authorization string `json:"-" gorm:"type:text;not null;default:''"`
|
||||
Status string `json:"status" gorm:"size:16;not null;default:''"`
|
||||
VerifiedAt *time.Time `json:"verified_at"`
|
||||
CreatedAt time.Time `json:"created_at" gorm:"autoCreateTime"`
|
||||
UpdatedAt time.Time `json:"updated_at" gorm:"autoUpdateTime"`
|
||||
}
|
||||
|
||||
// TableName returns the Cloudflare connection table name.
|
||||
func (CFConnection) TableName() string { return "of_cf_connections" }
|
||||
|
||||
// CFPointingGroup stores a reusable node target for DNS records.
|
||||
type CFPointingGroup struct {
|
||||
ID uint `json:"id" gorm:"primaryKey;autoIncrement"`
|
||||
Name string `json:"name" gorm:"size:128;not null"`
|
||||
PrimaryNodeID uint `json:"primary_node_id" gorm:"not null;index:idx_of_cf_pointing_groups_primary_node_id"`
|
||||
BackupNodeID *uint `json:"backup_node_id" gorm:"index:idx_of_cf_pointing_groups_backup_node_id"`
|
||||
ActiveNodeID uint `json:"active_node_id" gorm:"not null;index:idx_of_cf_pointing_groups_active_node_id"`
|
||||
DefaultProxied bool `json:"default_proxied" gorm:"not null;default:false"`
|
||||
Enabled bool `json:"enabled" gorm:"not null;default:false"`
|
||||
CreatedAt time.Time `json:"created_at" gorm:"autoCreateTime"`
|
||||
UpdatedAt time.Time `json:"updated_at" gorm:"autoUpdateTime"`
|
||||
}
|
||||
|
||||
// TableName returns the Cloudflare pointing group table name.
|
||||
func (CFPointingGroup) TableName() string { return "of_cf_pointing_groups" }
|
||||
|
||||
// CFPointingMember stores one managed ZoneDomain A record.
|
||||
type CFPointingMember struct {
|
||||
ID uint `json:"id" gorm:"primaryKey;autoIncrement"`
|
||||
GroupID uint `json:"group_id" gorm:"not null;index:idx_of_cf_pointing_members_group_id"`
|
||||
ZoneDomainID uint `json:"zone_domain_id" gorm:"not null;uniqueIndex:idx_of_cf_pointing_members_zone_domain_id"`
|
||||
Proxied bool `json:"proxied" gorm:"not null;default:false"`
|
||||
CFZoneID string `json:"cf_zone_id" gorm:"size:64;not null;default:''"`
|
||||
CFRecordID string `json:"cf_record_id" gorm:"size:64;not null;default:''"`
|
||||
DesiredIP string `json:"desired_ip" gorm:"size:64;not null;default:''"`
|
||||
SyncStatus string `json:"sync_status" gorm:"size:16;not null;default:'pending'"`
|
||||
LastError string `json:"last_error" gorm:"type:text;not null;default:''"`
|
||||
SyncedAt *time.Time `json:"synced_at"`
|
||||
CreatedAt time.Time `json:"created_at" gorm:"autoCreateTime"`
|
||||
UpdatedAt time.Time `json:"updated_at" gorm:"autoUpdateTime"`
|
||||
}
|
||||
|
||||
// TableName returns the Cloudflare pointing member table name.
|
||||
func (CFPointingMember) TableName() string { return "of_cf_pointing_members" }
|
||||
@@ -0,0 +1,57 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package model
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// ConfigVersionSummary is the list view for config versions.
|
||||
type ConfigVersionSummary struct {
|
||||
ID string `json:"id" gorm:"-"`
|
||||
Version string `json:"version" gorm:"primaryKey;column:version"`
|
||||
Checksum string `json:"checksum"`
|
||||
IsActive bool `json:"is_active"`
|
||||
CreatedBy string `json:"created_by"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
// AfterFind hook for ConfigVersionSummary.
|
||||
func (cvs *ConfigVersionSummary) AfterFind(_ *gorm.DB) (err error) {
|
||||
cvs.ID = cvs.Version
|
||||
return
|
||||
}
|
||||
|
||||
// ConfigVersion stores a published OpenResty configuration snapshot.
|
||||
type ConfigVersion struct {
|
||||
ID string `json:"id" gorm:"-"`
|
||||
Version string `json:"version" gorm:"primaryKey;size:32;not null"`
|
||||
SnapshotJSON string `json:"snapshot_json" gorm:"type:text;not null"`
|
||||
MainConfig string `json:"main_config" gorm:"type:text;not null;default:''"`
|
||||
RenderedConfig string `json:"rendered_config" gorm:"type:text;not null"`
|
||||
SupportFilesJSON string `json:"support_files_json" gorm:"type:text;not null;default:'[]'"`
|
||||
Checksum string `json:"checksum" gorm:"size:64;not null"`
|
||||
IsActive bool `json:"is_active" gorm:"not null;default:false;index"`
|
||||
CreatedBy string `json:"created_by" gorm:"size:64;not null"`
|
||||
CreatedAt time.Time `json:"created_at" gorm:"autoCreateTime"`
|
||||
}
|
||||
|
||||
// AfterFind hook for ConfigVersion.
|
||||
func (cv *ConfigVersion) AfterFind(_ *gorm.DB) (err error) {
|
||||
cv.ID = cv.Version
|
||||
return
|
||||
}
|
||||
|
||||
// AfterCreate hook for ConfigVersion.
|
||||
func (cv *ConfigVersion) AfterCreate(_ *gorm.DB) (err error) {
|
||||
cv.ID = cv.Version
|
||||
return
|
||||
}
|
||||
|
||||
// TableName returns the GORM table name.
|
||||
func (*ConfigVersion) TableName() string {
|
||||
return "of_config_versions"
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package model
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// DNSAccount OpenFlare DNS 账号实体。
|
||||
type DNSAccount struct {
|
||||
ID uint `json:"id" gorm:"primaryKey;autoIncrement"`
|
||||
Name string `json:"name" gorm:"size:255;not null"`
|
||||
Type string `json:"type" gorm:"size:64;not null"`
|
||||
Authorization string `json:"-" gorm:"type:text;not null"`
|
||||
CreatedAt time.Time `json:"created_at" gorm:"autoCreateTime"`
|
||||
UpdatedAt time.Time `json:"updated_at" gorm:"autoUpdateTime"`
|
||||
}
|
||||
|
||||
// TableName 表名。
|
||||
func (DNSAccount) TableName() string {
|
||||
return "of_dns_accounts"
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package model
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// OpenFlareNode stores an edge, relay, or tunnel client node.
|
||||
type OpenFlareNode struct {
|
||||
ID uint `json:"id" gorm:"primaryKey;autoIncrement"`
|
||||
NodeID string `json:"node_id" gorm:"uniqueIndex;size:64;not null"`
|
||||
Name string `json:"name" gorm:"size:128;not null"`
|
||||
IP string `json:"ip" gorm:"size:64;not null;default:''"`
|
||||
IPManualOverride bool `json:"ip_manual_override" gorm:"not null;default:false"`
|
||||
GeoName string `json:"geo_name" gorm:"size:128;not null;default:''"`
|
||||
GeoLatitude *float64 `json:"geo_latitude"`
|
||||
GeoLongitude *float64 `json:"geo_longitude"`
|
||||
GeoManualOverride bool `json:"geo_manual_override" gorm:"not null;default:false"`
|
||||
AccessToken string `json:"-" gorm:"column:access_token;size:128;index"`
|
||||
AutoUpdateEnabled bool `json:"auto_update_enabled" gorm:"not null;default:false"`
|
||||
UpdateRequested bool `json:"update_requested" gorm:"not null;default:false"`
|
||||
UpdateChannel string `json:"update_channel" gorm:"size:16;not null;default:'stable'"`
|
||||
UpdateTag string `json:"update_tag" gorm:"size:64;not null;default:''"`
|
||||
RestartOpenrestyRequested bool `json:"restart_openresty_requested" gorm:"not null;default:false"`
|
||||
Version string `json:"version" gorm:"size:64;not null;default:''"`
|
||||
ExtVersion string `json:"ext_version" gorm:"size:64;not null;default:''"`
|
||||
OpenrestyStatus string `json:"openresty_status" gorm:"size:16;not null;default:'unknown'"`
|
||||
OpenrestyMessage string `json:"openresty_message" gorm:"type:text"`
|
||||
Status string `json:"status" gorm:"size:16;not null;default:'offline'"`
|
||||
CurrentVersion string `json:"current_version" gorm:"size:32;not null;default:''"`
|
||||
LastSeenAt *time.Time `json:"last_seen_at"`
|
||||
LastError string `json:"last_error" gorm:"type:text"`
|
||||
CreatedAt time.Time `json:"created_at" gorm:"autoCreateTime"`
|
||||
UpdatedAt time.Time `json:"updated_at" gorm:"autoUpdateTime"`
|
||||
NodeType string `json:"node_type" gorm:"size:32;not null;default:'edge_node'"`
|
||||
RelayBindPort int `json:"relay_bind_port" gorm:"not null;default:0"`
|
||||
RelayVhostHTTPPort int `json:"relay_vhost_http_port" gorm:"not null;default:0"`
|
||||
RelayAuthToken string `json:"-" gorm:"size:128;not null;default:''"`
|
||||
RelayAgentAccessAddr string `json:"relay_agent_access_addr" gorm:"size:255;not null;default:''"`
|
||||
RelayClientAccessAddr string `json:"relay_client_access_addr" gorm:"size:255;not null;default:''"`
|
||||
RelayClientProxyURL string `json:"relay_client_proxy_url" gorm:"size:512;not null;default:''"`
|
||||
CapabilitiesJSON string `json:"capabilities_json" gorm:"type:text;not null;default:'[]'"`
|
||||
RelayStatus string `json:"relay_status" gorm:"size:16;not null;default:'unknown'"`
|
||||
RelayWebServerEnabled bool `json:"relay_web_server_enabled" gorm:"not null;default:false"`
|
||||
}
|
||||
|
||||
// TableName returns the GORM table name.
|
||||
func (OpenFlareNode) TableName() string {
|
||||
return "of_nodes"
|
||||
}
|
||||
@@ -0,0 +1,316 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package model
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// OpenFlareMetricSnapshot stores a node capacity snapshot in ClickHouse (database: openflare, table: of_node_metric_snapshots).
|
||||
// ClickHouse DDL is managed by goose; reads/writes go through internal/repository/analytics.
|
||||
type OpenFlareMetricSnapshot struct {
|
||||
ID uint `json:"id" gorm:"primaryKey;autoIncrement"`
|
||||
NodeID string `json:"node_id" gorm:"index;size:64;not null"`
|
||||
CapturedAt time.Time `json:"captured_at" gorm:"index"`
|
||||
CPUUsagePercent float64 `json:"cpu_usage_percent"`
|
||||
MemoryUsedBytes int64 `json:"memory_used_bytes"`
|
||||
MemoryTotalBytes int64 `json:"memory_total_bytes"`
|
||||
StorageUsedBytes int64 `json:"storage_used_bytes"`
|
||||
StorageTotalBytes int64 `json:"storage_total_bytes"`
|
||||
DiskReadBytes int64 `json:"disk_read_bytes"`
|
||||
DiskWriteBytes int64 `json:"disk_write_bytes"`
|
||||
NetworkRxBytes int64 `json:"network_rx_bytes"`
|
||||
NetworkTxBytes int64 `json:"network_tx_bytes"`
|
||||
CreatedAt time.Time `json:"created_at" gorm:"autoCreateTime"`
|
||||
}
|
||||
|
||||
// TableName returns the GORM table name.
|
||||
func (OpenFlareMetricSnapshot) TableName() string {
|
||||
return "of_node_metric_snapshots"
|
||||
}
|
||||
|
||||
// OpenFlareAccessLog stores a single access log row in ClickHouse (database: openflare, table: of_node_access_logs).
|
||||
// ClickHouse DDL is managed by goose; reads/writes go through internal/repository/analytics.
|
||||
type OpenFlareAccessLog struct {
|
||||
ID uint64 `json:"id,string" gorm:"column:id"`
|
||||
NodeID string `json:"node_id" gorm:"index;size:64;not null"`
|
||||
LoggedAt time.Time `json:"logged_at" gorm:"index"`
|
||||
RemoteAddr string `json:"remote_addr" gorm:"index;size:128"`
|
||||
Region string `json:"region" gorm:"size:128"`
|
||||
Host string `json:"host" gorm:"index;size:255"`
|
||||
Path string `json:"path" gorm:"size:2048"`
|
||||
UserAgent string `json:"user_agent" gorm:"column:user_agent;size:512"`
|
||||
CacheStatus string `json:"cache_status" gorm:"column:cache_status;size:32"`
|
||||
StatusCode int `json:"status_code" gorm:"index"`
|
||||
BytesSent int64 `json:"bytes_sent" gorm:"column:bytes_sent;not null;default:0"`
|
||||
RequestLength int64 `json:"request_length" gorm:"column:request_length;not null;default:0"`
|
||||
RequestTimeMs int64 `json:"request_time_ms" gorm:"column:request_time_ms;not null;default:0"`
|
||||
CreatedAt time.Time `json:"created_at" gorm:"autoCreateTime"`
|
||||
}
|
||||
|
||||
// TableName returns the GORM table name.
|
||||
func (OpenFlareAccessLog) TableName() string {
|
||||
return "of_node_access_logs"
|
||||
}
|
||||
|
||||
// OpenFlareAccessLogRegionCount aggregates access log regions.
|
||||
type OpenFlareAccessLogRegionCount struct {
|
||||
Region string `json:"region"`
|
||||
Count int64 `json:"count"`
|
||||
}
|
||||
|
||||
// OpenFlareHealthEvent stores node health alert events.
|
||||
type OpenFlareHealthEvent struct {
|
||||
ID uint `json:"id" gorm:"primaryKey;autoIncrement"`
|
||||
NodeID string `json:"node_id" gorm:"index;size:64;not null"`
|
||||
EventType string `json:"event_type" gorm:"index;size:64;not null"`
|
||||
Severity string `json:"severity" gorm:"size:16;not null"`
|
||||
Status string `json:"status" gorm:"index;size:16;not null"`
|
||||
Message string `json:"message" gorm:"type:text"`
|
||||
FirstTriggeredAt time.Time `json:"first_triggered_at" gorm:"index"`
|
||||
LastTriggeredAt time.Time `json:"last_triggered_at" gorm:"index"`
|
||||
ReportedAt time.Time `json:"reported_at" gorm:"index"`
|
||||
ResolvedAt *time.Time `json:"resolved_at" gorm:"index"`
|
||||
MetadataJSON string `json:"metadata_json" gorm:"type:text"`
|
||||
CreatedAt time.Time `json:"created_at" gorm:"autoCreateTime"`
|
||||
UpdatedAt time.Time `json:"updated_at" gorm:"autoUpdateTime"`
|
||||
}
|
||||
|
||||
// TableName returns the GORM table name.
|
||||
func (OpenFlareHealthEvent) TableName() string {
|
||||
return "of_node_health_events"
|
||||
}
|
||||
|
||||
// OpenFlareNodeSystemProfile stores the latest node system profile.
|
||||
type OpenFlareNodeSystemProfile struct {
|
||||
ID uint `json:"id" gorm:"primaryKey;autoIncrement"`
|
||||
NodeID string `json:"node_id" gorm:"uniqueIndex;size:64;not null"`
|
||||
Hostname string `json:"hostname" gorm:"size:255"`
|
||||
OSName string `json:"os_name" gorm:"size:128"`
|
||||
OSVersion string `json:"os_version" gorm:"size:128"`
|
||||
KernelVersion string `json:"kernel_version" gorm:"size:128"`
|
||||
Architecture string `json:"architecture" gorm:"size:64"`
|
||||
CPUModel string `json:"cpu_model" gorm:"size:255"`
|
||||
CPUCores int `json:"cpu_cores"`
|
||||
TotalMemoryBytes int64 `json:"total_memory_bytes"`
|
||||
TotalDiskBytes int64 `json:"total_disk_bytes"`
|
||||
UptimeSeconds int64 `json:"uptime_seconds"`
|
||||
ReportedAt time.Time `json:"reported_at" gorm:"index"`
|
||||
CreatedAt time.Time `json:"created_at" gorm:"autoCreateTime"`
|
||||
UpdatedAt time.Time `json:"updated_at" gorm:"autoUpdateTime"`
|
||||
}
|
||||
|
||||
// TableName returns the GORM table name.
|
||||
func (OpenFlareNodeSystemProfile) TableName() string {
|
||||
return "of_node_system_profiles"
|
||||
}
|
||||
|
||||
// OpenFlareEdgeHealth is L2 OpenResty health (of_node_edge_health).
|
||||
type OpenFlareEdgeHealth struct {
|
||||
ID uint `json:"id"`
|
||||
NodeID string `json:"node_id"`
|
||||
CapturedAt time.Time `json:"captured_at"`
|
||||
Status string `json:"status"`
|
||||
Connections int64 `json:"connections"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
// TableName returns the ClickHouse table name.
|
||||
func (OpenFlareEdgeHealth) TableName() string {
|
||||
return "of_node_edge_health"
|
||||
}
|
||||
|
||||
// OpenFlareNodeObservationFrpc stores tunnel client frpc observations in ClickHouse (database: openflare, table: of_node_obs_frpc).
|
||||
// ClickHouse DDL is managed by goose; reads/writes go through internal/repository/analytics.
|
||||
type OpenFlareNodeObservationFrpc struct {
|
||||
ID uint `json:"id" gorm:"primaryKey;autoIncrement"`
|
||||
NodeID string `json:"node_id" gorm:"index;size:64;not null"`
|
||||
CapturedAt time.Time `json:"captured_at" gorm:"index"`
|
||||
TunnelStatus string `json:"tunnel_status" gorm:"size:16"`
|
||||
ConnectedRelaysCount int `json:"connected_relays_count"`
|
||||
CreatedAt time.Time `json:"created_at" gorm:"autoCreateTime"`
|
||||
}
|
||||
|
||||
// TableName returns the GORM table name.
|
||||
func (OpenFlareNodeObservationFrpc) TableName() string {
|
||||
return "of_node_obs_frpc"
|
||||
}
|
||||
|
||||
// OpenFlareNodeObservationFrps stores tunnel relay frps observations in ClickHouse (database: openflare, table: of_node_obs_frps).
|
||||
// ClickHouse DDL is managed by goose; reads/writes go through internal/repository/analytics.
|
||||
type OpenFlareNodeObservationFrps struct {
|
||||
ID uint `json:"id" gorm:"primaryKey;autoIncrement"`
|
||||
NodeID string `json:"node_id" gorm:"index;size:64;not null"`
|
||||
CapturedAt time.Time `json:"captured_at" gorm:"index"`
|
||||
FrpsConnections int `json:"frps_connections"`
|
||||
FrpsProxyCount int `json:"frps_proxy_count"`
|
||||
FrpsClientCount int `json:"frps_client_count"`
|
||||
FrpsProxies string `json:"frps_proxies" gorm:"type:text"`
|
||||
CreatedAt time.Time `json:"created_at" gorm:"autoCreateTime"`
|
||||
}
|
||||
|
||||
// TableName returns the GORM table name.
|
||||
func (OpenFlareNodeObservationFrps) TableName() string {
|
||||
return "of_node_obs_frps"
|
||||
}
|
||||
|
||||
// OpenFlareAccessLogQuery filters access log list queries.
|
||||
type OpenFlareAccessLogQuery struct {
|
||||
NodeID string
|
||||
RemoteAddr string
|
||||
Host string
|
||||
// Hosts exact-matches any host (case-insensitive). Prefer over Host for multi-domain scopes.
|
||||
Hosts []string
|
||||
Path string
|
||||
// StatusCode filters by exact HTTP status code when > 0.
|
||||
StatusCode int
|
||||
Since time.Time
|
||||
Until time.Time
|
||||
Page int
|
||||
PageSize int
|
||||
SortBy string
|
||||
SortOrder string
|
||||
}
|
||||
|
||||
// OpenFlareAccessLogBucketQuery filters folded access log queries (v1 stub).
|
||||
type OpenFlareAccessLogBucketQuery struct {
|
||||
NodeID string
|
||||
RemoteAddr string
|
||||
Host string
|
||||
Hosts []string
|
||||
Path string
|
||||
Since time.Time
|
||||
Until time.Time
|
||||
Page int
|
||||
PageSize int
|
||||
SortBy string
|
||||
SortOrder string
|
||||
FoldMinutes int
|
||||
}
|
||||
|
||||
// OpenFlareAccessLogBucketRow is a folded access log bucket row (v1 stub).
|
||||
type OpenFlareAccessLogBucketRow struct {
|
||||
BucketEpoch int64 `json:"bucket_epoch"`
|
||||
RequestCount int64 `json:"request_count"`
|
||||
UniqueIPCount int64 `json:"unique_ip_count"`
|
||||
UniqueHostCount int64 `json:"unique_host_count"`
|
||||
SuccessCount int64 `json:"success_count"`
|
||||
ClientErrorCount int64 `json:"client_error_count"`
|
||||
ServerErrorCount int64 `json:"server_error_count"`
|
||||
Status2xxCount int64 `json:"status_2xx_count"`
|
||||
Status4xxCount int64 `json:"status_4xx_count"`
|
||||
Status5xxCount int64 `json:"status_5xx_count"`
|
||||
BytesSent int64 `json:"bytes_sent"`
|
||||
RequestLength int64 `json:"request_length"`
|
||||
}
|
||||
|
||||
// OpenFlareAccessLogBucketIPQuery filters folded IP summary queries (v1 stub).
|
||||
type OpenFlareAccessLogBucketIPQuery struct {
|
||||
NodeID string
|
||||
RemoteAddr string
|
||||
Host string
|
||||
Path string
|
||||
BucketStartedAt time.Time
|
||||
FoldMinutes int
|
||||
Page int
|
||||
PageSize int
|
||||
SortBy string
|
||||
SortOrder string
|
||||
}
|
||||
|
||||
// OpenFlareAccessLogBucketIPRow is a folded IP row (v1 stub).
|
||||
type OpenFlareAccessLogBucketIPRow struct {
|
||||
RemoteAddr string `json:"remote_addr"`
|
||||
RequestCount int64 `json:"request_count"`
|
||||
SuccessCount int64 `json:"success_count"`
|
||||
ClientErrorCount int64 `json:"client_error_count"`
|
||||
ServerErrorCount int64 `json:"server_error_count"`
|
||||
LastSeenEpoch int64 `json:"last_seen_epoch"`
|
||||
}
|
||||
|
||||
// OpenFlareAccessLogIPSummaryQuery filters IP summary list queries (v1 stub).
|
||||
type OpenFlareAccessLogIPSummaryQuery struct {
|
||||
NodeID string
|
||||
RemoteAddr string
|
||||
Host string
|
||||
Since time.Time
|
||||
Until time.Time
|
||||
Page int
|
||||
PageSize int
|
||||
SortBy string
|
||||
SortOrder string
|
||||
}
|
||||
|
||||
// OpenFlareAccessLogIPSummaryRow is an IP summary row (v1 stub).
|
||||
type OpenFlareAccessLogIPSummaryRow struct {
|
||||
RemoteAddr string `json:"remote_addr"`
|
||||
Region string `json:"region"`
|
||||
TotalRequests int64 `json:"total_requests"`
|
||||
Success2xxCount int64 `json:"success_2xx_count"`
|
||||
SuccessRatio float64 `json:"success_ratio"`
|
||||
BytesReceived int64 `json:"bytes_received"`
|
||||
BytesSent int64 `json:"bytes_sent"`
|
||||
// RecentRequests is deprecated and always 0.
|
||||
RecentRequests int64 `json:"recent_requests"`
|
||||
LastSeenEpoch int64 `json:"last_seen_epoch"`
|
||||
}
|
||||
|
||||
// OpenFlareAccessLogIPTrendQuery filters IP trend queries (v1 stub).
|
||||
type OpenFlareAccessLogIPTrendQuery struct {
|
||||
NodeID string
|
||||
RemoteAddr string
|
||||
Host string
|
||||
Since time.Time
|
||||
BucketMinutes int
|
||||
}
|
||||
|
||||
// OpenFlareAccessLogIPTrendRow is an IP trend bucket row (v1 stub).
|
||||
type OpenFlareAccessLogIPTrendRow struct {
|
||||
BucketEpoch int64 `json:"bucket_epoch"`
|
||||
RequestCount int64 `json:"request_count"`
|
||||
}
|
||||
|
||||
// OpenFlareAccessLogWAFIPAggregate is a per-IP aggregate row for WAF automatic rules.
|
||||
type OpenFlareAccessLogWAFIPAggregate struct {
|
||||
RemoteAddr string
|
||||
RequestCount int
|
||||
Status404Count int
|
||||
ClientErrorCount int
|
||||
ServerErrorCount int
|
||||
IPHostCount int
|
||||
LastSeenEpoch int64
|
||||
StatusCounts map[int]int
|
||||
}
|
||||
|
||||
// OpenFlareTrafficHourly is an hourly traffic rollup row.
|
||||
type OpenFlareTrafficHourly struct {
|
||||
NodeID string `json:"node_id"`
|
||||
Hour time.Time `json:"hour"`
|
||||
RequestCount int64 `json:"request_count"`
|
||||
ErrorCount int64 `json:"error_count"`
|
||||
UniqueVisitorCount int64 `json:"unique_visitor_count"`
|
||||
}
|
||||
|
||||
// OpenFlareAccessLogHourly is a per-node/host hourly access log rollup.
|
||||
type OpenFlareAccessLogHourly struct {
|
||||
NodeID string `json:"node_id"`
|
||||
Hour time.Time `json:"hour"`
|
||||
Host string `json:"host"`
|
||||
RequestCount int64 `json:"request_count"`
|
||||
ErrorCount int64 `json:"error_count"`
|
||||
BytesSent int64 `json:"bytes_sent"`
|
||||
RequestLength int64 `json:"request_length"`
|
||||
}
|
||||
|
||||
// OpenFlareMetricHourly is an hourly metric snapshot aggregation row.
|
||||
type OpenFlareMetricHourly struct {
|
||||
Hour time.Time `json:"hour"`
|
||||
AverageCPUUsagePercent float64 `json:"average_cpu_usage_percent"`
|
||||
AverageMemoryUsagePercent float64 `json:"average_memory_usage_percent"`
|
||||
NetworkRxBytes int64 `json:"network_rx_bytes"`
|
||||
NetworkTxBytes int64 `json:"network_tx_bytes"`
|
||||
DiskReadBytes int64 `json:"disk_read_bytes"`
|
||||
DiskWriteBytes int64 `json:"disk_write_bytes"`
|
||||
ReportedNodes int `json:"reported_nodes"`
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package model
|
||||
|
||||
import "time"
|
||||
|
||||
// OpenFlareOption is the key/value DTO used by the OpenFlare option API surface.
|
||||
// Key 与 w_system_configs.key 一致(snake_case)。此类型仅作为
|
||||
// /api/v1/d/option 接口的请求/响应载体,不再映射到独立的 of_options 表。
|
||||
type OpenFlareOption struct {
|
||||
Key string `json:"key"`
|
||||
Value string `json:"value"`
|
||||
}
|
||||
|
||||
// StartTime records process start time (seconds) for the public /status endpoint.
|
||||
var StartTime = time.Now().Unix()
|
||||
|
||||
// DefaultOpenRestyMainConfigTemplate 是 OpenResty 主配置模板的内置默认值。
|
||||
// 当 SystemConfig 中缺失 openresty_main_config_template 时作为兜底使用。
|
||||
const DefaultOpenRestyMainConfigTemplate = `# This file is generated by OpenFlare. Do not edit manually.
|
||||
user openflare;
|
||||
worker_processes {{OpenRestyWorkerProcesses}};
|
||||
worker_rlimit_nofile {{OpenRestyWorkerRlimitNofile}};
|
||||
pid __OPENFLARE_PID_PATH__;
|
||||
error_log {{OpenRestyErrorLogPath}} warn;
|
||||
|
||||
events {
|
||||
worker_connections {{OpenRestyWorkerConnections}};
|
||||
{{OpenRestyEventsUseDirective}}{{OpenRestyEventsMultiAcceptDirective}}}
|
||||
|
||||
http {
|
||||
include mime.types;
|
||||
default_type application/octet-stream;
|
||||
server_tokens off;
|
||||
client_body_temp_path __OPENFLARE_NGINX_CACHE_DIR__/client_temp;
|
||||
proxy_temp_path __OPENFLARE_NGINX_CACHE_DIR__/proxy_temp;
|
||||
fastcgi_temp_path __OPENFLARE_NGINX_CACHE_DIR__/fastcgi_temp;
|
||||
uwsgi_temp_path __OPENFLARE_NGINX_CACHE_DIR__/uwsgi_temp;
|
||||
scgi_temp_path __OPENFLARE_NGINX_CACHE_DIR__/scgi_temp;
|
||||
{{OpenRestyConnectionUpgradeMap}}{{OpenRestyDefaultServerBlock}} log_format openflare_json escape=json '{"ts":"$time_iso8601","host":"$host","path":"$request_uri","remote_addr":"$remote_addr","status":$status,"request_time":$request_time,"bytes_sent":$body_bytes_sent,"request_length":$request_length,"user_agent":"$http_user_agent","cache_status":"$upstream_cache_status"}';
|
||||
access_log {{OpenRestyAccessLogPath}} openflare_json;
|
||||
sendfile on;
|
||||
tcp_nopush on;
|
||||
tcp_nodelay on;
|
||||
keepalive_timeout {{OpenRestyKeepaliveTimeout}};
|
||||
keepalive_requests {{OpenRestyKeepaliveRequests}};
|
||||
client_header_timeout {{OpenRestyClientHeaderTimeout}};
|
||||
client_body_timeout {{OpenRestyClientBodyTimeout}};
|
||||
client_max_body_size {{OpenRestyClientMaxBodySize}};
|
||||
large_client_header_buffers {{OpenRestyLargeClientHeaderBuffers}};
|
||||
send_timeout {{OpenRestySendTimeout}};
|
||||
proxy_connect_timeout {{OpenRestyProxyConnectTimeout}};
|
||||
proxy_send_timeout {{OpenRestyProxySendTimeout}};
|
||||
proxy_read_timeout {{OpenRestyProxyReadTimeout}};
|
||||
proxy_request_buffering {{OpenRestyProxyRequestBuffering}};
|
||||
proxy_buffering {{OpenRestyProxyBuffering}};
|
||||
proxy_buffers {{OpenRestyProxyBuffers}};
|
||||
proxy_buffer_size {{OpenRestyProxyBufferSize}};
|
||||
proxy_busy_buffers_size {{OpenRestyProxyBusyBuffersSize}};
|
||||
gzip {{OpenRestyGzip}};
|
||||
gzip_min_length {{OpenRestyGzipMinLength}};
|
||||
gzip_comp_level {{OpenRestyGzipCompLevel}};
|
||||
{{OpenRestyResolverDirective}}{{OpenRestyCacheBlock}} include {{OpenRestyRouteConfigInclude}};
|
||||
}
|
||||
`
|
||||
@@ -0,0 +1,45 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package model
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// Origin OpenFlare 源站实体。
|
||||
type Origin struct {
|
||||
ID uint `json:"id" gorm:"primaryKey;autoIncrement"`
|
||||
Name string `json:"name" gorm:"size:255;not null"`
|
||||
Address string `json:"address" gorm:"uniqueIndex;size:255;not null"`
|
||||
Remark string `json:"remark" gorm:"size:255"`
|
||||
CreatedAt time.Time `json:"created_at" gorm:"autoCreateTime"`
|
||||
UpdatedAt time.Time `json:"updated_at" gorm:"autoUpdateTime"`
|
||||
}
|
||||
|
||||
// TableName 表名。
|
||||
func (Origin) TableName() string {
|
||||
return "of_origins"
|
||||
}
|
||||
|
||||
// OriginRouteCount 源站关联的代理规则数量。
|
||||
type OriginRouteCount struct {
|
||||
OriginID uint `json:"origin_id"`
|
||||
RouteCount int64 `json:"route_count"`
|
||||
}
|
||||
|
||||
// OriginProxyRoute 源站模块查询代理规则时使用的最小字段集。
|
||||
type OriginProxyRoute struct {
|
||||
ID uint `gorm:"column:id;primaryKey"`
|
||||
OriginID *uint `gorm:"column:origin_id"`
|
||||
Domain string `gorm:"column:domain"`
|
||||
OriginURL string `gorm:"column:origin_url"`
|
||||
Upstreams string `gorm:"column:upstreams"`
|
||||
Enabled bool `gorm:"column:enabled"`
|
||||
UpdatedAt time.Time `gorm:"column:updated_at"`
|
||||
}
|
||||
|
||||
// TableName 表名。
|
||||
func (OriginProxyRoute) TableName() string {
|
||||
return tableOfProxyRoutes
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package model
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// Pages deployment status constants.
|
||||
const (
|
||||
PagesDeploymentStatusUploaded = "uploaded"
|
||||
PagesDeploymentStatusActive = "active"
|
||||
)
|
||||
|
||||
// PagesProject OpenFlare Pages 静态托管项目。
|
||||
type PagesProject struct {
|
||||
ID uint `json:"id" gorm:"primaryKey;autoIncrement"`
|
||||
Name string `json:"name" gorm:"size:255;not null"`
|
||||
Slug string `json:"slug" gorm:"uniqueIndex;size:128;not null"`
|
||||
Description string `json:"description" gorm:"type:text;not null;default:''"`
|
||||
Enabled bool `json:"enabled" gorm:"not null;default:true"`
|
||||
SPAFallbackEnabled bool `json:"spa_fallback_enabled" gorm:"not null;default:false"`
|
||||
SPAFallbackPath string `json:"spa_fallback_path" gorm:"size:512;not null;default:'/index.html'"`
|
||||
APIProxyEnabled bool `json:"api_proxy_enabled" gorm:"not null;default:false"`
|
||||
APIProxyPath string `json:"api_proxy_path" gorm:"size:255;not null;default:''"`
|
||||
APIProxyPass string `json:"api_proxy_pass" gorm:"size:2048;not null;default:''"`
|
||||
APIProxyRewrite string `json:"api_proxy_rewrite" gorm:"size:255;not null;default:''"`
|
||||
ActiveDeploymentID *uint `json:"active_deployment_id" gorm:"index"`
|
||||
RootDir string `json:"root_dir" gorm:"size:512;not null;default:''"`
|
||||
EntryFile string `json:"entry_file" gorm:"size:512;not null;default:'index.html'"`
|
||||
ContentConfigVersion int `json:"-" gorm:"not null;default:0"`
|
||||
CreatedAt time.Time `json:"created_at" gorm:"autoCreateTime"`
|
||||
UpdatedAt time.Time `json:"updated_at" gorm:"autoUpdateTime"`
|
||||
}
|
||||
|
||||
// TableName 表名。
|
||||
func (PagesProject) TableName() string {
|
||||
return "of_pages_projects"
|
||||
}
|
||||
|
||||
// PagesDeployment OpenFlare Pages 不可变部署记录。
|
||||
type PagesDeployment struct {
|
||||
ID uint `json:"id" gorm:"primaryKey;autoIncrement"`
|
||||
ProjectID uint `json:"project_id" gorm:"not null;index;uniqueIndex:idx_of_pages_deployments_project_number,priority:1;uniqueIndex:idx_of_pages_deployments_source_revision,priority:1,where:source_identity IS NOT NULL AND source_revision IS NOT NULL"`
|
||||
DeploymentNumber int `json:"deployment_number" gorm:"not null;uniqueIndex:idx_of_pages_deployments_project_number,priority:2"`
|
||||
Checksum string `json:"checksum" gorm:"size:64;not null;index"`
|
||||
Status string `json:"status" gorm:"size:32;not null;default:'uploaded';index"`
|
||||
UploadID uint64 `json:"upload_id,string" gorm:"not null;default:0;index"`
|
||||
ArtifactPath string `json:"artifact_path,omitempty" gorm:"size:2048;not null;default:''"` // legacy only
|
||||
FileCount int `json:"file_count" gorm:"not null;default:0"`
|
||||
TotalSize int64 `json:"total_size" gorm:"not null;default:0"`
|
||||
CreatedBy string `json:"created_by" gorm:"size:64;not null;default:''"`
|
||||
SourceType string `json:"source_type" gorm:"size:32;not null;default:''"`
|
||||
SourceIdentity *string `json:"-" gorm:"type:char(64);uniqueIndex:idx_of_pages_deployments_source_revision,priority:2,where:source_identity IS NOT NULL AND source_revision IS NOT NULL"`
|
||||
SourceRevision *string `json:"-" gorm:"type:char(64);uniqueIndex:idx_of_pages_deployments_source_revision,priority:3,where:source_identity IS NOT NULL AND source_revision IS NOT NULL"`
|
||||
SourceLabel string `json:"source_label" gorm:"size:255;not null;default:''"`
|
||||
SourceMeta string `json:"-" gorm:"type:text;not null;default:''"`
|
||||
TriggerType string `json:"trigger_type" gorm:"size:32;not null;default:''"`
|
||||
CreatedAt time.Time `json:"created_at" gorm:"autoCreateTime"`
|
||||
ActivatedAt *time.Time `json:"activated_at"`
|
||||
}
|
||||
|
||||
// TableName 表名。
|
||||
func (PagesDeployment) TableName() string {
|
||||
return "of_pages_deployments"
|
||||
}
|
||||
|
||||
// PagesDeploymentFile OpenFlare Pages 部署文件清单。
|
||||
type PagesDeploymentFile struct {
|
||||
ID uint `json:"id" gorm:"primaryKey;autoIncrement"`
|
||||
DeploymentID uint `json:"deployment_id" gorm:"not null;index"`
|
||||
Path string `json:"path" gorm:"size:2048;not null"`
|
||||
Size int64 `json:"size" gorm:"not null;default:0"`
|
||||
Checksum string `json:"checksum" gorm:"size:64;not null"`
|
||||
CreatedAt time.Time `json:"created_at" gorm:"autoCreateTime"`
|
||||
}
|
||||
|
||||
// TableName 表名。
|
||||
func (PagesDeploymentFile) TableName() string {
|
||||
return "of_pages_deployment_files"
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package model
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
// PagesOrphanUploadCandidateLimit bounds one delayed Pages upload cleanup pass.
|
||||
PagesOrphanUploadCandidateLimit = 100
|
||||
// PagesOrphanMarkerPredicatePostgres is the Postgres JSON marker match SQL fragment.
|
||||
PagesOrphanMarkerPredicatePostgres = "w_uploads.metadata #>> '{extra,pages_ingest_marker}' = ?"
|
||||
// PagesOrphanMarkerPredicateSQLite is the SQLite JSON marker match SQL fragment.
|
||||
PagesOrphanMarkerPredicateSQLite = "CASE WHEN json_valid(w_uploads.metadata) THEN json_extract(w_uploads.metadata, '$.extra.pages_ingest_marker') ELSE NULL END = ?"
|
||||
)
|
||||
|
||||
// PagesOrphanUploadCandidateQuery describes the fail-closed SQL candidate set
|
||||
// for delayed Pages upload compensation.
|
||||
type PagesOrphanUploadCandidateQuery struct {
|
||||
SystemUserID uint64
|
||||
UploadType string
|
||||
Marker string
|
||||
CreatedBefore time.Time
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package model
|
||||
|
||||
import "time"
|
||||
|
||||
// PagesProjectSource 保存 Pages 项目的持久部署源配置。
|
||||
//
|
||||
// 对外接口必须映射到 pages 包内的 source view,避免直接序列化 model。
|
||||
type PagesProjectSource struct {
|
||||
ID uint `json:"-" gorm:"primaryKey;autoIncrement"`
|
||||
ProjectID uint `json:"-" gorm:"not null;uniqueIndex:idx_of_pages_project_sources_project_id"`
|
||||
SourceType string `json:"-" gorm:"size:32;not null;default:''"`
|
||||
RemoteURL string `json:"-" gorm:"type:text;not null;default:''"`
|
||||
AllowInsecure bool `json:"-" gorm:"not null;default:false"`
|
||||
GitHubRepository string `json:"-" gorm:"column:github_repository;size:255;not null;default:''"`
|
||||
ReleaseSelector string `json:"-" gorm:"size:16;not null;default:''"`
|
||||
ReleaseTag string `json:"-" gorm:"size:255;not null;default:''"`
|
||||
AssetName string `json:"-" gorm:"size:255;not null;default:''"`
|
||||
AutoUpdateEnabled bool `json:"-" gorm:"not null;default:false"`
|
||||
CheckIntervalMinutes int `json:"-" gorm:"not null;default:0"`
|
||||
ConfigVersion int `json:"-" gorm:"not null;default:0"`
|
||||
SourceIdentity string `json:"-" gorm:"type:char(64);not null;default:''"`
|
||||
CreatedAt time.Time `json:"-" gorm:"autoCreateTime"`
|
||||
UpdatedAt time.Time `json:"-" gorm:"autoUpdateTime"`
|
||||
}
|
||||
|
||||
// TableName 返回 Pages 项目部署源配置表名。
|
||||
func (PagesProjectSource) TableName() string {
|
||||
return "of_pages_project_sources"
|
||||
}
|
||||
|
||||
// PagesProjectSourceRuntime 保存 Pages 项目部署源的可变运行态。
|
||||
//
|
||||
// Runtime 不冗余 project_id;调用方通过 SourceID 关联配置,并在最终提交时
|
||||
// 同时校验 source config version 与 project content config version。
|
||||
type PagesProjectSourceRuntime struct {
|
||||
SourceID uint `json:"-" gorm:"primaryKey;autoIncrement:false"`
|
||||
ETag string `json:"-" gorm:"column:etag;size:512;not null;default:''"`
|
||||
LastSeenRevision string `json:"-" gorm:"type:char(64);not null;default:''"`
|
||||
LastSeenDetail string `json:"-" gorm:"type:text;not null;default:''"`
|
||||
LastAppliedRevision string `json:"-" gorm:"type:char(64);not null;default:''"`
|
||||
LastAppliedDetail string `json:"-" gorm:"type:text;not null;default:''"`
|
||||
SyncStatus string `json:"-" gorm:"size:32;not null;default:''"`
|
||||
LastError string `json:"-" gorm:"type:text;not null;default:''"`
|
||||
LastCheckedAt *time.Time `json:"-"`
|
||||
LastSyncedAt *time.Time `json:"-"`
|
||||
NextCheckAt *time.Time `json:"-" gorm:"index:idx_of_pages_project_source_runtime_next_check_at"`
|
||||
LeaseExpiresAt *time.Time `json:"-"`
|
||||
LeaseToken string `json:"-" gorm:"size:64;not null;default:''"`
|
||||
UpdatedAt time.Time `json:"-" gorm:"autoUpdateTime"`
|
||||
}
|
||||
|
||||
// TableName 返回 Pages 项目部署源运行态表名。
|
||||
func (PagesProjectSourceRuntime) TableName() string {
|
||||
return "of_pages_project_source_runtime"
|
||||
}
|
||||
|
||||
// PagesExpiredSourceLeaseCandidate is a scanner query DTO for expired runtime leases.
|
||||
type PagesExpiredSourceLeaseCandidate struct {
|
||||
SourceID uint
|
||||
LeaseToken string
|
||||
LeaseExpiresAt time.Time
|
||||
SyncStatus string
|
||||
SourceType string
|
||||
ReleaseSelector string
|
||||
}
|
||||
|
||||
// PagesDueGitHubSourceCandidate is a scanner query DTO for due GitHub latest checks.
|
||||
type PagesDueGitHubSourceCandidate struct {
|
||||
SourceID uint
|
||||
ConfigVersion int
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package model
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/glebarez/sqlite"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func TestPagesSourceModelsMatchMigrationSchema(t *testing.T) {
|
||||
gormDB, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{
|
||||
DisableForeignKeyConstraintWhenMigrating: true,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, gormDB.AutoMigrate(
|
||||
&PagesProject{},
|
||||
&PagesDeployment{},
|
||||
&PagesProjectSource{},
|
||||
&PagesProjectSourceRuntime{},
|
||||
))
|
||||
|
||||
assert.Equal(t, "of_pages_project_sources", (PagesProjectSource{}).TableName())
|
||||
assert.Equal(t, "of_pages_project_source_runtime", (PagesProjectSourceRuntime{}).TableName())
|
||||
assert.True(t, gormDB.Migrator().HasColumn(&PagesProjectSource{}, "github_repository"))
|
||||
assert.False(t, gormDB.Migrator().HasColumn(&PagesProjectSource{}, "git_hub_repository"))
|
||||
assert.True(t, gormDB.Migrator().HasColumn(&PagesProjectSourceRuntime{}, "etag"))
|
||||
assert.False(t, gormDB.Migrator().HasColumn(&PagesProjectSourceRuntime{}, "e_tag"))
|
||||
|
||||
var indexSQL string
|
||||
require.NoError(t, gormDB.Raw(
|
||||
"SELECT sql FROM sqlite_master WHERE type = 'index' AND name = ?",
|
||||
"idx_of_pages_deployments_source_revision",
|
||||
).Scan(&indexSQL).Error)
|
||||
assert.Contains(t, strings.ToUpper(indexSQL), "WHERE SOURCE_IDENTITY IS NOT NULL AND SOURCE_REVISION IS NOT NULL")
|
||||
}
|
||||
|
||||
func TestPagesSourceModelsDoNotSerializeSecretsOrFencingState(t *testing.T) {
|
||||
sourceJSON, err := json.Marshal(PagesProjectSource{
|
||||
ID: 1,
|
||||
ProjectID: 2,
|
||||
RemoteURL: "https://example.com/site.zip?token=secret",
|
||||
ConfigVersion: 3,
|
||||
SourceIdentity: strings.Repeat("a", 64),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.JSONEq(t, `{}`, string(sourceJSON))
|
||||
|
||||
runtimeJSON, err := json.Marshal(PagesProjectSourceRuntime{
|
||||
SourceID: 1,
|
||||
ETag: `"secret-etag"`,
|
||||
LeaseToken: "secret-lease",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.JSONEq(t, `{}`, string(runtimeJSON))
|
||||
|
||||
identity := strings.Repeat("b", 64)
|
||||
revision := strings.Repeat("c", 64)
|
||||
deploymentJSON, err := json.Marshal(PagesDeployment{
|
||||
SourceType: "remote_url",
|
||||
SourceIdentity: &identity,
|
||||
SourceRevision: &revision,
|
||||
SourceLabel: "site.zip",
|
||||
SourceMeta: `{"provider":"remote_url","private":"secret"}`,
|
||||
TriggerType: "manual_sync",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.NotContains(t, string(deploymentJSON), identity)
|
||||
assert.NotContains(t, string(deploymentJSON), revision)
|
||||
assert.NotContains(t, string(deploymentJSON), "private")
|
||||
assert.Contains(t, string(deploymentJSON), `"source_type":"remote_url"`)
|
||||
assert.Contains(t, string(deploymentJSON), `"source_label":"site.zip"`)
|
||||
assert.Contains(t, string(deploymentJSON), `"trigger_type":"manual_sync"`)
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package model
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
const tableOfProxyRoutes = "of_proxy_routes"
|
||||
|
||||
// ProxyRoute OpenFlare 代理规则实体。
|
||||
// 域名与证书仅通过 of_zone_domains 关联,不再持久化在本表。
|
||||
type ProxyRoute struct {
|
||||
ID uint `json:"id" gorm:"primaryKey;autoIncrement"`
|
||||
SiteName string `json:"site_name" gorm:"size:255;not null;default:''"`
|
||||
OriginID *uint `json:"origin_id" gorm:"index"`
|
||||
OriginURL string `json:"origin_url" gorm:"size:2048;not null"`
|
||||
OriginHost string `json:"origin_host" gorm:"size:255"`
|
||||
Upstreams string `json:"upstreams" gorm:"type:text;not null;default:'[]'"`
|
||||
Enabled bool `json:"enabled" gorm:"not null;default:true"`
|
||||
EnableHTTPS bool `json:"enable_https" gorm:"column:enable_https;not null;default:false"`
|
||||
RedirectHTTP bool `json:"redirect_http" gorm:"not null;default:false"`
|
||||
LimitConnPerServer int `json:"limit_conn_per_server" gorm:"not null;default:0"`
|
||||
LimitConnPerIP int `json:"limit_conn_per_ip" gorm:"not null;default:0"`
|
||||
LimitRate string `json:"limit_rate" gorm:"size:32;not null;default:''"`
|
||||
LimitReqPerIP string `json:"limit_req_per_ip" gorm:"size:32;not null;default:''"`
|
||||
CacheEnabled bool `json:"cache_enabled" gorm:"not null;default:false"`
|
||||
CachePolicy string `json:"cache_policy" gorm:"size:32;not null;default:''"`
|
||||
CacheRules string `json:"cache_rules" gorm:"type:text;not null;default:'[]'"`
|
||||
CustomHeaders string `json:"custom_headers" gorm:"type:text;not null;default:'[]'"`
|
||||
BasicAuthEnabled bool `json:"basic_auth_enabled" gorm:"not null;default:false"`
|
||||
BasicAuthUsername string `json:"basic_auth_username" gorm:"size:255;not null;default:''"`
|
||||
BasicAuthPassword string `json:"basic_auth_password" gorm:"size:255;not null;default:''"`
|
||||
UpstreamType string `json:"upstream_type" gorm:"size:32;not null;default:'direct'"`
|
||||
TunnelNodeID *uint `json:"tunnel_node_id" gorm:"index"`
|
||||
TunnelTargetAddr string `json:"tunnel_target_addr" gorm:"size:512"`
|
||||
TunnelTargetProtocol string `json:"tunnel_target_protocol" gorm:"size:16"`
|
||||
PagesProjectID *uint `json:"pages_project_id" gorm:"index"`
|
||||
CreatedAt time.Time `json:"created_at" gorm:"autoCreateTime"`
|
||||
UpdatedAt time.Time `json:"updated_at" gorm:"autoUpdateTime"`
|
||||
ZoneDomains []ZoneDomain `json:"zone_domains,omitempty" gorm:"foreignKey:ProxyRouteID"`
|
||||
}
|
||||
|
||||
// TableName 表名。
|
||||
func (ProxyRoute) TableName() string {
|
||||
return tableOfProxyRoutes
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package model
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// TLSCertificate OpenFlare TLS 证书实体。
|
||||
type TLSCertificate struct {
|
||||
ID uint `json:"id" gorm:"primaryKey;autoIncrement"`
|
||||
Name string `json:"name" gorm:"uniqueIndex;size:255;not null"`
|
||||
CertPEM string `json:"-" gorm:"type:text;not null"`
|
||||
KeyPEM string `json:"-" gorm:"type:text;not null"`
|
||||
NotBefore time.Time `json:"not_before"`
|
||||
NotAfter time.Time `json:"not_after"`
|
||||
Remark string `json:"remark" gorm:"size:255"`
|
||||
Provider string `json:"provider" gorm:"size:64;default:upload"`
|
||||
AcmeAccountID uint `json:"acme_account_id"`
|
||||
DNSAccountID uint `json:"dns_account_id"`
|
||||
KeyAlgorithm string `json:"key_algorithm" gorm:"size:32"`
|
||||
AutoRenew bool `json:"auto_renew"`
|
||||
PrimaryDomain string `json:"primary_domain" gorm:"size:255"`
|
||||
OtherDomains string `json:"other_domains" gorm:"type:text"`
|
||||
DisableCNAME bool `json:"disable_cname"`
|
||||
SkipDNS bool `json:"skip_dns"`
|
||||
DNS1 string `json:"dns1" gorm:"size:128"`
|
||||
DNS2 string `json:"dns2" gorm:"size:128"`
|
||||
ApplyStatus string `json:"apply_status" gorm:"size:64;default:ready"`
|
||||
ApplyMessage string `json:"apply_message" gorm:"type:text"`
|
||||
CreatedAt time.Time `json:"created_at" gorm:"autoCreateTime"`
|
||||
UpdatedAt time.Time `json:"updated_at" gorm:"autoUpdateTime"`
|
||||
}
|
||||
|
||||
// TableName 表名。
|
||||
func (TLSCertificate) TableName() string {
|
||||
return "of_tls_certificates"
|
||||
}
|
||||
|
||||
// TLSProxyRouteRef 删除证书时检查代理规则引用的最小字段集。
|
||||
type TLSProxyRouteRef struct {
|
||||
ID uint `gorm:"column:id;primaryKey"`
|
||||
CertID *uint `gorm:"column:cert_id"`
|
||||
CertIDs string `gorm:"column:cert_ids"`
|
||||
DomainCertIDs string `gorm:"column:domain_cert_ids"`
|
||||
}
|
||||
|
||||
// TableName 表名。
|
||||
func (TLSProxyRouteRef) TableName() string {
|
||||
return tableOfProxyRoutes
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package model
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"time"
|
||||
)
|
||||
|
||||
// OpenFlareWAFRuleGroup stores a WAF rule group.
|
||||
type OpenFlareWAFRuleGroup struct {
|
||||
ID uint `json:"id" gorm:"primaryKey;autoIncrement"`
|
||||
Name string `json:"name" gorm:"size:255;not null"`
|
||||
Enabled bool `json:"enabled" gorm:"not null;default:true"`
|
||||
IsGlobal bool `json:"is_global" gorm:"not null;default:false;index"`
|
||||
Graph string `json:"graph" gorm:"type:text;not null;default:''"`
|
||||
Revision uint64 `json:"revision" gorm:"not null;default:1"`
|
||||
CreatedAt time.Time `json:"created_at" gorm:"autoCreateTime"`
|
||||
UpdatedAt time.Time `json:"updated_at" gorm:"autoUpdateTime"`
|
||||
}
|
||||
|
||||
// TableName returns the GORM table name.
|
||||
func (OpenFlareWAFRuleGroup) TableName() string {
|
||||
return "of_waf_rule_groups"
|
||||
}
|
||||
|
||||
// OpenFlareWAFIPGroup stores a WAF IP group.
|
||||
type OpenFlareWAFIPGroup struct {
|
||||
ID uint `json:"id" gorm:"primaryKey;autoIncrement"`
|
||||
Name string `json:"name" gorm:"size:255;not null"`
|
||||
Type string `json:"type" gorm:"size:32;not null;index"`
|
||||
Enabled bool `json:"enabled" gorm:"not null;default:true"`
|
||||
IPList string `json:"ip_list" gorm:"type:text;not null;default:'[]'"`
|
||||
AutoConfig string `json:"auto_config" gorm:"type:text;not null;default:'{}'"`
|
||||
ExtIPs string `json:"ext_ips" gorm:"type:text;not null;default:'[]'"`
|
||||
SubscriptionURL string `json:"subscription_url" gorm:"size:2048;not null;default:''"`
|
||||
SubscriptionFormat string `json:"subscription_format" gorm:"size:32;not null;default:'text'"`
|
||||
SubscriptionMappingRule string `json:"subscription_mapping_rule" gorm:"size:255;not null;default:''"`
|
||||
SyncIntervalMinutes int `json:"sync_interval_minutes" gorm:"not null;default:1440"`
|
||||
LastSyncedAt *time.Time `json:"last_synced_at"`
|
||||
NextSyncAt *time.Time `json:"next_sync_at" gorm:"index"`
|
||||
LastSyncStatus string `json:"last_sync_status" gorm:"size:32;not null;default:''"`
|
||||
LastSyncMessage string `json:"last_sync_message" gorm:"type:text;not null;default:''"`
|
||||
CreatedAt time.Time `json:"created_at" gorm:"autoCreateTime"`
|
||||
UpdatedAt time.Time `json:"updated_at" gorm:"autoUpdateTime"`
|
||||
}
|
||||
|
||||
// TableName returns the GORM table name.
|
||||
func (OpenFlareWAFIPGroup) TableName() string {
|
||||
return "of_waf_ip_groups"
|
||||
}
|
||||
|
||||
// OpenFlareWAFRuleGroupBinding binds a rule group to a proxy route.
|
||||
type OpenFlareWAFRuleGroupBinding struct {
|
||||
ID uint `json:"id" gorm:"primaryKey;autoIncrement"`
|
||||
RuleGroupID uint `json:"rule_group_id" gorm:"not null;uniqueIndex:idx_of_waf_group_route"`
|
||||
ProxyRouteID uint `json:"proxy_route_id" gorm:"not null;uniqueIndex:idx_of_waf_group_route;index"`
|
||||
Sequence int `json:"sequence" gorm:"not null;default:0"`
|
||||
CreatedAt time.Time `json:"created_at" gorm:"autoCreateTime"`
|
||||
}
|
||||
|
||||
// ErrWAFRuleRevisionConflict indicates that a rule graph was updated from a stale revision.
|
||||
var ErrWAFRuleRevisionConflict = errors.New("waf rule revision conflict")
|
||||
|
||||
// TableName returns the GORM table name.
|
||||
func (OpenFlareWAFRuleGroupBinding) TableName() string {
|
||||
return "of_waf_rule_group_bindings"
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package model
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
tableOfZones = "of_zones"
|
||||
tableOfZoneDomains = "of_zone_domains"
|
||||
)
|
||||
|
||||
// Zone OpenFlare 注册根域实体。
|
||||
type Zone struct {
|
||||
ID uint `json:"id" gorm:"primaryKey;autoIncrement"`
|
||||
Domain string `json:"domain" gorm:"uniqueIndex:idx_of_zones_domain;size:255;not null"`
|
||||
CreatedAt time.Time `json:"created_at" gorm:"autoCreateTime"`
|
||||
UpdatedAt time.Time `json:"updated_at" gorm:"autoUpdateTime"`
|
||||
}
|
||||
|
||||
// TableName 表名。
|
||||
func (Zone) TableName() string {
|
||||
return tableOfZones
|
||||
}
|
||||
|
||||
// ZoneDomain OpenFlare Zone 下的明确域名实体。
|
||||
type ZoneDomain struct {
|
||||
ID uint `json:"id" gorm:"primaryKey;autoIncrement"`
|
||||
ZoneID uint `json:"zone_id" gorm:"not null;index:idx_of_zone_domains_zone_id"`
|
||||
ProxyRouteID *uint `json:"proxy_route_id" gorm:"index:idx_of_zone_domains_proxy_route_id"`
|
||||
Domain string `json:"domain" gorm:"uniqueIndex:idx_of_zone_domains_domain;size:255;not null"`
|
||||
CertID *uint `json:"cert_id" gorm:"index:idx_of_zone_domains_cert_id"`
|
||||
CreatedAt time.Time `json:"created_at" gorm:"autoCreateTime"`
|
||||
UpdatedAt time.Time `json:"updated_at" gorm:"autoUpdateTime"`
|
||||
}
|
||||
|
||||
// TableName 表名。
|
||||
func (ZoneDomain) TableName() string {
|
||||
return tableOfZoneDomains
|
||||
}
|
||||
|
||||
// ZoneDomainCount is the per-zone explicit domain count for list queries.
|
||||
type ZoneDomainCount struct {
|
||||
ZoneID uint `json:"zone_id" gorm:"column:zone_id"`
|
||||
Count int64 `json:"count" gorm:"column:count"`
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package model
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
|
||||
adminmodel "Wavelet/plugins/domain/admin/model"
|
||||
authmodel "Wavelet/plugins/domain/auth"
|
||||
uploadmodels "Wavelet/plugins/domain/upload/models"
|
||||
usermodel "Wavelet/plugins/domain/user"
|
||||
)
|
||||
|
||||
const (
|
||||
tokenByteLength = 24
|
||||
maskThreshold = 8
|
||||
)
|
||||
|
||||
// User is the Wavelet w_users entity.
|
||||
type User = usermodel.User
|
||||
|
||||
// AccessToken is the Wavelet w_access_tokens entity.
|
||||
type AccessToken = usermodel.AccessToken
|
||||
|
||||
// AuthSource is the Wavelet w_auth_sources entity.
|
||||
type AuthSource = authmodel.AuthSource
|
||||
|
||||
// ExternalAccount is the Wavelet w_external_accounts entity.
|
||||
type ExternalAccount = authmodel.ExternalAccount
|
||||
|
||||
// TaskExecution is the Wavelet w_task_executions entity.
|
||||
type TaskExecution = adminmodel.TaskExecution
|
||||
|
||||
// Template is the Wavelet w_templates entity.
|
||||
type Template = adminmodel.Template
|
||||
|
||||
// Schedule is the Wavelet w_schedules entity.
|
||||
type Schedule = adminmodel.Schedule
|
||||
|
||||
// Upload is the Wavelet w_uploads entity.
|
||||
type Upload = uploadmodels.Upload
|
||||
|
||||
// UploadMetadata is the Wavelet upload metadata JSON.
|
||||
type UploadMetadata = uploadmodels.UploadMetadata
|
||||
|
||||
// UploadStatus is the Wavelet upload status.
|
||||
type UploadStatus = uploadmodels.UploadStatus
|
||||
|
||||
// UploadStat is the Wavelet w_upload_stats entity.
|
||||
type UploadStat = uploadmodels.UploadStat
|
||||
|
||||
const (
|
||||
// UploadStatusPending is a newly stored unused upload.
|
||||
UploadStatusPending = uploadmodels.UploadStatusPending
|
||||
// UploadStatusUsed is an in-use upload.
|
||||
UploadStatusUsed = uploadmodels.UploadStatusUsed
|
||||
// UploadStatusDeleted is a soft-deleted upload.
|
||||
UploadStatusDeleted = uploadmodels.UploadStatusDeleted
|
||||
|
||||
// UploadStatDimensionTotal is the total stats dimension.
|
||||
UploadStatDimensionTotal = uploadmodels.UploadStatDimensionTotal
|
||||
// UploadStatDimensionType is the type stats dimension.
|
||||
UploadStatDimensionType = uploadmodels.UploadStatDimensionType
|
||||
// UploadStatDimensionCategory is the category stats dimension.
|
||||
UploadStatDimensionCategory = uploadmodels.UploadStatDimensionCategory
|
||||
// UploadStatDimensionTrend is the trend stats dimension.
|
||||
UploadStatDimensionTrend = uploadmodels.UploadStatDimensionTrend
|
||||
)
|
||||
|
||||
// GenerateTokenString 生成加密安全的随机 Token 值
|
||||
func GenerateTokenString() (string, error) {
|
||||
bytes := make([]byte, tokenByteLength)
|
||||
if _, err := rand.Read(bytes); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "at_" + hex.EncodeToString(bytes), nil
|
||||
}
|
||||
|
||||
// HashToken 计算 Token 的 SHA-256 哈希值用于数据库存储与查询
|
||||
func HashToken(token string) string {
|
||||
h := sha256.New()
|
||||
h.Write([]byte(token))
|
||||
return hex.EncodeToString(h.Sum(nil))
|
||||
}
|
||||
|
||||
// MaskTokenString 生成脱敏显示的 Token,仅保留前缀和最后四位
|
||||
func MaskTokenString(token string) string {
|
||||
if len(token) <= maskThreshold {
|
||||
return "at_****"
|
||||
}
|
||||
return fmt.Sprintf("%s...%s", token[:7], token[len(token)-4:])
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package model
|
||||
|
||||
import (
|
||||
adminmodel "Wavelet/plugins/domain/admin/model"
|
||||
)
|
||||
|
||||
// 配置键常量 - 所有系统配置的 key 定义
|
||||
const (
|
||||
ConfigKeyUploadAllowedExtensions = "upload_allowed_extensions" // 允许上传的文件扩展名,逗号分隔
|
||||
ConfigKeySiteName = "site_name" // 站点名称
|
||||
ConfigKeyPasswordLoginEnabled = "password_login_enabled" // 是否允许密码登录
|
||||
ConfigKeyRegistrationEnabled = "registration_enabled" // 是否允许注册
|
||||
ConfigKeyPasswordRegisterEnabled = "password_register_enabled" // 是否允许密码注册
|
||||
ConfigKeyOIDCLoginEnabled = "oidc_login_enabled" // 是否允许 OIDC 登录
|
||||
ConfigKeyMaxAPIKeysPerUser = "max_api_keys_per_user" //nolint:gosec // false positive: config key name. 每个用户最大 API Key 数量
|
||||
ConfigKeyCapLoginEnabled = "cap_login_enabled" // 是否启用登录人机验证
|
||||
ConfigKeyCapAutoSolve = "cap_auto_solve" // 打开页面后是否自动开始计算(false 则需用户手动点击)
|
||||
ConfigKeyCapChallengeCount = "cap_challenge_count" // 客户端需求解的 PoW 难题总数,默认 1,推荐 1~5
|
||||
ConfigKeyCapChallengeSize = "cap_challenge_size" // 人机验证盐值长度
|
||||
ConfigKeyCapChallengeDifficulty = "cap_challenge_difficulty" // 人机验证 PoW 难度(目标前缀长度)
|
||||
ConfigKeyCapChallengeTTL = "cap_challenge_ttl_seconds" // 人机验证难题有效时间(秒)
|
||||
ConfigKeyCapTokenTTL = "cap_token_ttl_seconds" //nolint:gosec // false positive: config key name. 人机验证兑换凭证有效时间(秒)
|
||||
ConfigKeyServerAddress = "server_address" // 服务器地址
|
||||
ConfigKeySMTPHost = "smtp_host" // SMTP 服务器地址
|
||||
ConfigKeySMTPPort = "smtp_port" // SMTP 端口
|
||||
ConfigKeySMTPUsername = "smtp_username" // SMTP 账户
|
||||
ConfigKeySMTPPassword = "smtp_password" // SMTP 访问凭证
|
||||
ConfigKeyEmailLoginVerificationEnabled = "email_login_verification_enabled" // 是否启用邮箱登录验证
|
||||
ConfigKeyEmailRegisterVerificationEnabled = "email_register_verification_enabled" // 是否启用邮箱注册验证
|
||||
ConfigKeyMenuDisplayConfig = "menu_display_config" // 目录显示配置 (JSON 字符串)
|
||||
ConfigKeySearchEngineIndexingEnabled = "search_engine_indexing_enabled" // 是否允许搜索引擎检索
|
||||
ConfigKeyFileAccessWhitelist = "file_access_whitelist" // 免登录访问的文件业务类型白名单 (JSON 数组格式)
|
||||
ConfigKeyDiskCacheMaxSizeMB = "disk_cache_max_size_mb" // 磁盘缓存最大空间大小 (MB)
|
||||
ConfigKeyDiskCacheTTLMinutes = "disk_cache_ttl_minutes" // 磁盘缓存默认有效期 (分钟)
|
||||
ConfigKeyDiskCacheLRUEnabled = "disk_cache_lru_enabled" // 是否启用 LRU 淘汰机制
|
||||
ConfigKeyLoginSessionTTLHours = "login_session_ttl_hours" // 登录会话过期时间 (小时,0表示浏览器关闭后自动退出登录,-1表示永不过期)
|
||||
ConfigKeyUpdateUpstreamRepository = "update_upstream_repository" // GitHub Actions Release 上游仓库
|
||||
ConfigKeyStorageConfig = "storage_config" // 文件存储配置 (JSON)
|
||||
ConfigKeyRelayFRPSWebUIEnabled = "relay_frps_web_ui_enabled" // 是否启用 FRPS 内置 Web 界面
|
||||
ConfigKeyRelayFRPSWebUIPort = "relay_frps_web_ui_port" // FRPS 内置 Web 界面端口
|
||||
|
||||
// OpenFlare 业务配置(从 of_options 迁移)
|
||||
ConfigKeyAgentDiscoveryToken = "agent_discovery_token" //nolint:gosec // false positive: config key name. Agent 发现令牌
|
||||
ConfigKeyAgentHeartbeatInterval = "agent_heartbeat_interval" // Agent 心跳间隔(毫秒)
|
||||
ConfigKeyAgentWebsocketUpgradeEnabled = "agent_websocket_upgrade_enabled" // Agent WebSocket 升级开关
|
||||
ConfigKeyNodeOfflineThreshold = "node_offline_threshold" // 节点离线阈值(毫秒)
|
||||
ConfigKeyAgentUpdateRepo = "agent_update_repo" // Agent 更新仓库
|
||||
ConfigKeyGeoIPProvider = "geoip_provider" // GeoIP 服务商
|
||||
|
||||
// Pages 静态托管配置
|
||||
ConfigKeyPagesMaxPackageSizeMB = "pages_max_package_size_mb" // Pages 部署包上传大小上限(MiB)
|
||||
ConfigKeyPagesMaxHistoryCount = "pages_max_history_count" // Pages 每个项目最大历史部署保留数(0 表示不限制)
|
||||
|
||||
// UptimeKuma 集成配置
|
||||
ConfigKeyUptimeKumaEnabled = "uptime_kuma_enabled" // UptimeKuma 集成开关
|
||||
ConfigKeyUptimeKumaURL = "uptime_kuma_url" // UptimeKuma URL
|
||||
ConfigKeyUptimeKumaUsername = "uptime_kuma_username" // UptimeKuma 用户名
|
||||
ConfigKeyUptimeKumaPassword = "uptime_kuma_password" //nolint:gosec // false positive: config key name. UptimeKuma 密码
|
||||
ConfigKeyUptimeKumaMonitorScope = "uptime_kuma_monitor_scope" // UptimeKuma 监控范围
|
||||
ConfigKeyUptimeKumaSelectedSites = "uptime_kuma_selected_sites" // UptimeKuma 选定站点
|
||||
ConfigKeyUptimeKumaSyncInterval = "uptime_kuma_sync_interval" // UptimeKuma 同步间隔(分钟)
|
||||
ConfigKeyUptimeKumaInterval = "uptime_kuma_interval" // UptimeKuma 监控间隔(秒)
|
||||
ConfigKeyUptimeKumaRetry = "uptime_kuma_retry" // UptimeKuma 重试次数
|
||||
ConfigKeyUptimeKumaRetryInterval = "uptime_kuma_retry_interval" // UptimeKuma 重试间隔(秒)
|
||||
ConfigKeyUptimeKumaTimeout = "uptime_kuma_timeout" // UptimeKuma 超时(秒)
|
||||
|
||||
// OpenResty 配置
|
||||
ConfigKeyOpenRestyDefaultServerReturnStatus = "openresty_default_server_return_status" // 默认服务器返回状态码
|
||||
ConfigKeyOpenRestyWorkerProcesses = "openresty_worker_processes" // Worker 进程数
|
||||
ConfigKeyOpenRestyWorkerConnections = "openresty_worker_connections" // Worker 连接数
|
||||
ConfigKeyOpenRestyWorkerRlimitNofile = "openresty_worker_rlimit_nofile" // Worker 文件描述符限制
|
||||
ConfigKeyOpenRestyEventsUse = "openresty_events_use" // 事件模型
|
||||
ConfigKeyOpenRestyEventsMultiAcceptEnabled = "openresty_events_multi_accept_enabled" // 多路接受开关
|
||||
ConfigKeyOpenRestyKeepaliveTimeout = "openresty_keepalive_timeout" // Keepalive 超时(秒)
|
||||
ConfigKeyOpenRestyKeepaliveRequests = "openresty_keepalive_requests" // Keepalive 请求数
|
||||
ConfigKeyOpenRestyClientHeaderTimeout = "openresty_client_header_timeout" // 客户端头超时(秒)
|
||||
ConfigKeyOpenRestyClientBodyTimeout = "openresty_client_body_timeout" // 客户端体超时(秒)
|
||||
ConfigKeyOpenRestyClientMaxBodySize = "openresty_client_max_body_size" // 客户端最大体大小
|
||||
ConfigKeyOpenRestyLargeClientHeaderBuffers = "openresty_large_client_header_buffers" // 大客户端头缓冲区
|
||||
ConfigKeyOpenRestySendTimeout = "openresty_send_timeout" // 发送超时(秒)
|
||||
ConfigKeyOpenRestyResolvers = "openresty_resolvers" // DNS 解析器
|
||||
ConfigKeyOpenRestyProxyConnectTimeout = "openresty_proxy_connect_timeout" // 代理连接超时(秒)
|
||||
ConfigKeyOpenRestyProxySendTimeout = "openresty_proxy_send_timeout" // 代理发送超时(秒)
|
||||
ConfigKeyOpenRestyProxyReadTimeout = "openresty_proxy_read_timeout" // 代理读取超时(秒)
|
||||
ConfigKeyOpenRestyWebsocketEnabled = "openresty_websocket_enabled" // WebSocket 支持开关
|
||||
ConfigKeyOpenRestyHTTP3Enabled = "openresty_http3_enabled" // HTTP/3 支持开关
|
||||
ConfigKeyOpenRestyProxyRequestBufferingEnabled = "openresty_proxy_request_buffering_enabled" // 代理请求缓冲开关
|
||||
ConfigKeyOpenRestyProxyBufferingEnabled = "openresty_proxy_buffering_enabled" // 代理响应缓冲开关
|
||||
ConfigKeyOpenRestyProxyBuffers = "openresty_proxy_buffers" // 代理缓冲区
|
||||
ConfigKeyOpenRestyProxyBufferSize = "openresty_proxy_buffer_size" // 代理缓冲区大小
|
||||
ConfigKeyOpenRestyProxyBusyBuffersSize = "openresty_proxy_busy_buffers_size" // 代理繁忙缓冲区大小
|
||||
ConfigKeyOpenRestyGzipEnabled = "openresty_gzip_enabled" // Gzip 压缩开关
|
||||
ConfigKeyOpenRestyGzipMinLength = "openresty_gzip_min_length" // Gzip 最小长度
|
||||
ConfigKeyOpenRestyGzipCompLevel = "openresty_gzip_comp_level" // Gzip 压缩级别
|
||||
ConfigKeyOpenRestyCacheEnabled = "openresty_cache_enabled" // 缓存开关
|
||||
ConfigKeyOpenRestyCachePath = "openresty_cache_path" // 缓存路径
|
||||
ConfigKeyOpenRestyCacheLevels = "openresty_cache_levels" // 缓存层级
|
||||
ConfigKeyOpenRestyCacheInactive = "openresty_cache_inactive" // 缓存不活跃时间
|
||||
ConfigKeyOpenRestyCacheMaxSize = "openresty_cache_max_size" // 缓存最大大小
|
||||
ConfigKeyOpenRestyCacheKeyTemplate = "openresty_cache_key_template" // 缓存键模板
|
||||
ConfigKeyOpenRestyCacheLockEnabled = "openresty_cache_lock_enabled" // 缓存锁开关
|
||||
ConfigKeyOpenRestyCacheLockTimeout = "openresty_cache_lock_timeout" // 缓存锁超时
|
||||
ConfigKeyOpenRestyCacheUseStale = "openresty_cache_use_stale" // 缓存失效策略
|
||||
ConfigKeyOpenRestyMainConfigTemplate = "openresty_main_config_template" // 主配置模板
|
||||
ConfigKeyOpenRestyDefaultLimitConnPerServer = "openresty_default_limit_conn_per_server" // 默认站点并发连接
|
||||
ConfigKeyOpenRestyDefaultLimitConnPerIP = "openresty_default_limit_conn_per_ip" // 默认单 IP 并发连接
|
||||
ConfigKeyOpenRestyDefaultLimitRate = "openresty_default_limit_rate" // 默认单请求带宽
|
||||
ConfigKeyOpenRestyDefaultLimitReqPerIP = "openresty_default_limit_req_per_ip" // 默认单 IP 请求频率限制
|
||||
|
||||
// 源站错误页
|
||||
ConfigKeyOriginErrorPageEnabled = "origin_error_page_enabled" // 是否启用源站错误页
|
||||
ConfigKeyOriginErrorPageStatusCodes = "origin_error_page_status_codes" // 源站错误页触发状态码标签 JSON 数组
|
||||
ConfigKeyOriginErrorPageHTML = "origin_error_page_html" // 源站错误页自定义 HTML(空则内置默认)
|
||||
ConfigKeyOriginErrorPageGetOnly = "origin_error_page_get_only" // 是否仅对 GET 请求返回自定义错误页
|
||||
|
||||
// Service Worker 离线兜底
|
||||
ConfigKeySWOfflineEnabled = "sw_offline_enabled" // 是否启用 Service Worker 离线兜底
|
||||
ConfigKeySWOfflineHTML = "sw_offline_html" // 离线联系页自定义 HTML(空则内置默认)
|
||||
ConfigKeySWOfflineDomains = "sw_offline_domains" // 离线兜底生效域名列表(JSON 数组,空则仅总开关无效)
|
||||
)
|
||||
|
||||
// 日志数据库解耦
|
||||
const (
|
||||
ConfigKeyLogDatabase = "log_database" // 当前日志主库:postgres|sqlite|clickhouse(仅迁移任务写入)
|
||||
ConfigKeyLogDBMigration = "log_db_migration" // 迁移冻结标记:"migrating" 或空
|
||||
ConfigKeyLogRetentionDaysPostgres = "log_retention_days_postgres" // PostgreSQL 日志保留天数
|
||||
ConfigKeyLogRetentionDaysSQLite = "log_retention_days_sqlite" // SQLite 日志保留天数
|
||||
ConfigKeyLogRetentionDaysClickHouse = "log_retention_days_clickhouse" // ClickHouse 日志保留天数
|
||||
// ConfigKeyMetricRetentionDays 性能指标(CPU/内存/磁盘/网络)保留天数,三库共用;
|
||||
// 性能数据价值衰减快,默认短留存(3 天),不随访问日志保留配置。
|
||||
ConfigKeyMetricRetentionDays = "metric_retention_days"
|
||||
)
|
||||
|
||||
const (
|
||||
// ConfigVisibilityHidden 表示配置不通过公共配置接口暴露
|
||||
ConfigVisibilityHidden = adminmodel.ConfigVisibilityHidden
|
||||
// ConfigVisibilityVisible 表示配置通过公共配置接口暴露
|
||||
ConfigVisibilityVisible = adminmodel.ConfigVisibilityVisible
|
||||
)
|
||||
|
||||
// SystemConfig is the Wavelet w_system_configs entity.
|
||||
type SystemConfig = adminmodel.SystemConfig
|
||||
@@ -0,0 +1,13 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Package ofevents lists OpenFlare-specific push events registered onto Wavelet's PushRegistry.
|
||||
package ofevents
|
||||
|
||||
import "Wavelet/core/contracts"
|
||||
|
||||
// All returns OpenFlare-only built-in push events.
|
||||
// Platform events (admin login, etc.) stay in Wavelet message_gateway.
|
||||
func All() []contracts.PushEventMeta {
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Package ofupload wraps Wavelet upload ingest with Pages-specific helpers.
|
||||
package ofupload
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"Wavelet/core/contracts"
|
||||
waveletupload "Wavelet/plugins/domain/upload"
|
||||
"Wavelet/plugins/domain/upload/models"
|
||||
"Wavelet/plugins/infra/database"
|
||||
)
|
||||
|
||||
// ReservedPagesDeploymentType is managed exclusively by the Pages domain.
|
||||
const ReservedPagesDeploymentType = "openflare_pages_deployment"
|
||||
|
||||
const (
|
||||
// PolicyCreate always stores a new object and creates a new upload record.
|
||||
PolicyCreate = waveletupload.PolicyCreate
|
||||
// PolicyDedupNewRecord reuses an existing object path on hash match but creates a new record.
|
||||
PolicyDedupNewRecord = waveletupload.PolicyDedupNewRecord
|
||||
// PolicyResolveExisting returns an existing upload on hash match.
|
||||
PolicyResolveExisting = waveletupload.PolicyResolveExisting
|
||||
)
|
||||
|
||||
type (
|
||||
// IngestRequest is the programmatic upload ingest payload.
|
||||
IngestRequest = waveletupload.IngestRequest
|
||||
// IngestResult reports ingest side effects.
|
||||
IngestResult = waveletupload.IngestResult
|
||||
// IngestPolicy controls hash-collision behavior during ingest.
|
||||
IngestPolicy = waveletupload.IngestPolicy
|
||||
)
|
||||
|
||||
var (
|
||||
storageMu sync.RWMutex
|
||||
storageSvc contracts.StorageService
|
||||
)
|
||||
|
||||
// SetStorage injects the platform StorageService used to open stored objects.
|
||||
func SetStorage(s contracts.StorageService) {
|
||||
storageMu.Lock()
|
||||
defer storageMu.Unlock()
|
||||
storageSvc = s
|
||||
}
|
||||
|
||||
func currentStorage() contracts.StorageService {
|
||||
storageMu.RLock()
|
||||
defer storageMu.RUnlock()
|
||||
return storageSvc
|
||||
}
|
||||
|
||||
// IngestFromLocalPath ingests a local regular file through Wavelet upload ingest.
|
||||
func IngestFromLocalPath(ctx context.Context, localPath string, req IngestRequest) (IngestResult, error) {
|
||||
localPath = strings.TrimSpace(localPath)
|
||||
if localPath == "" {
|
||||
return IngestResult{}, errors.New("local path is required")
|
||||
}
|
||||
file, err := os.Open(localPath) //nolint:gosec // localPath is resolved from managed Pages artifacts
|
||||
if err != nil {
|
||||
return IngestResult{}, err
|
||||
}
|
||||
defer func() { _ = file.Close() }()
|
||||
|
||||
info, err := file.Stat()
|
||||
if err != nil {
|
||||
return IngestResult{}, err
|
||||
}
|
||||
if info.IsDir() {
|
||||
return IngestResult{}, errors.New("local path must be a regular file")
|
||||
}
|
||||
if req.Size <= 0 {
|
||||
req.Size = info.Size()
|
||||
}
|
||||
req.Reader = file
|
||||
return waveletupload.Ingest(ctx, req)
|
||||
}
|
||||
|
||||
// GetActiveUpload loads an active (non-deleted) upload by ID.
|
||||
func GetActiveUpload(ctx context.Context, id uint64) (models.Upload, error) {
|
||||
conn := database.DB(ctx)
|
||||
if conn == nil {
|
||||
return models.Upload{}, errors.New("database not initialized")
|
||||
}
|
||||
var upload models.Upload
|
||||
err := conn.Where("id = ? AND status <> ?", id, models.UploadStatusDeleted).First(&upload).Error
|
||||
return upload, err
|
||||
}
|
||||
|
||||
// OpenedUploadObject is a stored object stream plus the upload record.
|
||||
type OpenedUploadObject struct {
|
||||
Upload models.Upload
|
||||
Body io.ReadCloser
|
||||
ContentType string
|
||||
ContentLength int64
|
||||
}
|
||||
|
||||
// OpenStoredUpload opens the stored object for an active upload via StorageService.
|
||||
func OpenStoredUpload(ctx context.Context, id uint64) (*OpenedUploadObject, error) {
|
||||
upload, err := GetActiveUpload(ctx, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
svc := currentStorage()
|
||||
if svc == nil {
|
||||
return nil, errors.New("storage service not available")
|
||||
}
|
||||
obj, err := svc.Get(ctx, upload.FilePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &OpenedUploadObject{
|
||||
Upload: upload,
|
||||
Body: obj.Body,
|
||||
ContentType: obj.ContentType,
|
||||
ContentLength: obj.ContentLength,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// LocalFileCandidateRequest describes filesystem locations that may host a legacy blob.
|
||||
type LocalFileCandidateRequest struct {
|
||||
StoredPath string
|
||||
RelativePaths []string
|
||||
}
|
||||
|
||||
// ResolveLocalFile returns the first existing regular file among candidate paths.
|
||||
func ResolveLocalFile(_ context.Context, req LocalFileCandidateRequest) (string, int64, error) {
|
||||
candidates := append([]string{req.StoredPath}, req.RelativePaths...)
|
||||
for _, candidate := range candidates {
|
||||
candidate = strings.TrimSpace(candidate)
|
||||
if candidate == "" {
|
||||
continue
|
||||
}
|
||||
info, err := os.Stat(candidate) //nolint:gosec // candidate is resolved from managed Pages metadata
|
||||
if err != nil || info.IsDir() {
|
||||
continue
|
||||
}
|
||||
return candidate, info.Size(), nil
|
||||
}
|
||||
return "", 0, os.ErrNotExist
|
||||
}
|
||||
|
||||
// RebuildUploadStats rebuilds aggregate upload stats.
|
||||
func RebuildUploadStats(ctx context.Context) error {
|
||||
return waveletupload.RebuildUploadStats(ctx)
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package ofupload
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"Wavelet/plugins/domain/upload/cache"
|
||||
"Wavelet/plugins/domain/upload/models"
|
||||
uploadrepo "Wavelet/plugins/domain/upload/repository"
|
||||
uploadstats "Wavelet/plugins/domain/upload/stats"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// RemoveLockedTx performs the idempotent active-to-deleted transition for a row
|
||||
// that the caller has already locked in its surrounding transaction.
|
||||
func RemoveLockedTx(tx *gorm.DB, upload *models.Upload) (bool, error) {
|
||||
if upload == nil {
|
||||
return false, nil
|
||||
}
|
||||
if upload.Status == models.UploadStatusDeleted {
|
||||
return false, nil
|
||||
}
|
||||
snapshot := *upload
|
||||
if err := uploadrepo.SoftDeleteUploadTx(tx, upload); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := uploadstats.ApplyUploadStatsDeltaTx(tx, &snapshot, -1); err != nil {
|
||||
return false, err
|
||||
}
|
||||
upload.Status = models.UploadStatusDeleted
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// InvalidateUploadMetaCache evicts cached upload metadata.
|
||||
func InvalidateUploadMetaCache(ctx context.Context, id uint64) {
|
||||
cache.EvictUploadMeta(ctx, id)
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Package publicconfig implements contracts.PublicConfigProvider for OpenFlare.
|
||||
package publicconfig
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"Wavelet/core"
|
||||
"Wavelet/openflare/plugins/server/kernel/repository"
|
||||
)
|
||||
|
||||
// Provider returns visibility=1 system configs as a flat key/value map,
|
||||
// matching gold GetPublicConfig.
|
||||
type Provider struct{}
|
||||
|
||||
// New constructs a PublicConfigProvider. ctx is accepted for future binding
|
||||
// but the payload is loaded from the shared system config store.
|
||||
func New(_ *core.Context) *Provider {
|
||||
return &Provider{}
|
||||
}
|
||||
|
||||
// PublicConfig returns visibility=1 keys as map[string]string.
|
||||
func (p *Provider) PublicConfig(ctx context.Context) (any, error) {
|
||||
configs, err := repository.ListVisibleSystemConfigs(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp := make(map[string]string, len(configs))
|
||||
for _, config := range configs {
|
||||
resp[config.Key] = config.Value
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package publicconfig
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/model"
|
||||
"Wavelet/openflare/plugins/server/kernel/repository"
|
||||
"Wavelet/openflare/plugins/server/kernel/testhelper"
|
||||
)
|
||||
|
||||
func TestPublicConfigSeesSaveOrUpdateThroughAdminCache(t *testing.T) {
|
||||
_, _, cleanup := testhelper.SetupTestEnvironment(t)
|
||||
t.Cleanup(cleanup)
|
||||
|
||||
ctx := context.Background()
|
||||
provider := New(nil)
|
||||
|
||||
first, err := provider.PublicConfig(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("PublicConfig() warm error = %v", err)
|
||||
}
|
||||
firstMap, ok := first.(map[string]string)
|
||||
if !ok {
|
||||
t.Fatalf("PublicConfig() = %T, want map[string]string", first)
|
||||
}
|
||||
if got := firstMap[model.ConfigKeySiteName]; got != "OpenFlare" {
|
||||
t.Fatalf("PublicConfig()[%q] = %q, want %q", model.ConfigKeySiteName, got, "OpenFlare")
|
||||
}
|
||||
|
||||
if err := repository.SaveOrUpdateSystemConfig(ctx, model.ConfigKeySiteName, "Updated"); err != nil {
|
||||
t.Fatalf("SaveOrUpdateSystemConfig(%q) error = %v", model.ConfigKeySiteName, err)
|
||||
}
|
||||
|
||||
got, err := repository.GetSystemConfigByKey(ctx, model.ConfigKeySiteName)
|
||||
if err != nil {
|
||||
t.Fatalf("GetSystemConfigByKey(%q) error = %v", model.ConfigKeySiteName, err)
|
||||
}
|
||||
if got.Value != "Updated" {
|
||||
t.Fatalf("GetSystemConfigByKey(%q).Value = %q, want %q", model.ConfigKeySiteName, got.Value, "Updated")
|
||||
}
|
||||
|
||||
second, err := provider.PublicConfig(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("PublicConfig() after save error = %v", err)
|
||||
}
|
||||
secondMap, ok := second.(map[string]string)
|
||||
if !ok {
|
||||
t.Fatalf("PublicConfig() after save = %T, want map[string]string", second)
|
||||
}
|
||||
if got := secondMap[model.ConfigKeySiteName]; got == "OpenFlare" {
|
||||
t.Fatalf("PublicConfig() after save [%q] stayed stale at %q", model.ConfigKeySiteName, got)
|
||||
}
|
||||
if got := secondMap[model.ConfigKeySiteName]; got != "Updated" {
|
||||
t.Fatalf("PublicConfig() after save [%q] = %q, want %q", model.ConfigKeySiteName, got, "Updated")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"github.com/ClickHouse/clickhouse-go/v2/lib/column"
|
||||
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
|
||||
)
|
||||
|
||||
type mockConn struct {
|
||||
batch driver.Batch
|
||||
batchQuery string
|
||||
prepareCalled bool
|
||||
preparedQuery string
|
||||
queries []string
|
||||
queryArgs [][]any
|
||||
queryFn func(ctx context.Context, query string, args ...any) (driver.Rows, error)
|
||||
}
|
||||
|
||||
func (m *mockConn) Contributors() []string { return nil }
|
||||
|
||||
func (m *mockConn) ServerVersion() (*driver.ServerVersion, error) { return nil, nil }
|
||||
|
||||
func (m *mockConn) Select(_ context.Context, _ any, _ string, _ ...any) error { return nil }
|
||||
|
||||
func (m *mockConn) Query(ctx context.Context, query string, args ...any) (driver.Rows, error) {
|
||||
m.queries = append(m.queries, query)
|
||||
m.queryArgs = append(m.queryArgs, args)
|
||||
if m.queryFn != nil {
|
||||
return m.queryFn(ctx, query, args...)
|
||||
}
|
||||
return &mockRows{}, nil
|
||||
}
|
||||
|
||||
func (m *mockConn) QueryRow(_ context.Context, _ string, _ ...any) driver.Row { return nil }
|
||||
|
||||
func (m *mockConn) PrepareBatch(_ context.Context, query string, _ ...driver.PrepareBatchOption) (driver.Batch, error) {
|
||||
m.prepareCalled = true
|
||||
m.preparedQuery = query
|
||||
return m.batch, nil
|
||||
}
|
||||
|
||||
func (m *mockConn) Exec(_ context.Context, _ string, _ ...any) error { return nil }
|
||||
|
||||
func (m *mockConn) AsyncInsert(_ context.Context, _ string, _ bool, _ ...any) error { return nil }
|
||||
|
||||
func (m *mockConn) InsertFormat(_ context.Context, _ string, _ string, _ io.Reader) error { return nil }
|
||||
|
||||
func (m *mockConn) QueryFormat(_ context.Context, _ string, _ string, _ ...any) (io.ReadCloser, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (m *mockConn) Ping(_ context.Context) error { return nil }
|
||||
|
||||
func (m *mockConn) Stats() driver.Stats { return driver.Stats{} }
|
||||
|
||||
func (m *mockConn) Close() error { return nil }
|
||||
|
||||
type mockBatch struct {
|
||||
rows [][]any
|
||||
sendCalled bool
|
||||
}
|
||||
|
||||
func (m *mockBatch) Abort() error { return nil }
|
||||
|
||||
func (m *mockBatch) Append(v ...any) error {
|
||||
m.rows = append(m.rows, v)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *mockBatch) AppendStruct(_ any) error { return nil }
|
||||
|
||||
func (m *mockBatch) Column(_ int) driver.BatchColumn { return nil }
|
||||
|
||||
func (m *mockBatch) Flush() error { return nil }
|
||||
|
||||
func (m *mockBatch) Send() error {
|
||||
m.sendCalled = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *mockBatch) IsSent() bool { return m.sendCalled }
|
||||
|
||||
func (m *mockBatch) Rows() int { return len(m.rows) }
|
||||
|
||||
func (m *mockBatch) Columns() []column.Interface { return nil }
|
||||
|
||||
func (m *mockBatch) Close() error { return nil }
|
||||
|
||||
// mockRows is an empty driver.Rows implementation for query-path unit tests.
|
||||
type mockRows struct {
|
||||
index int
|
||||
data [][]any
|
||||
err error
|
||||
}
|
||||
|
||||
func (m *mockRows) Next() bool {
|
||||
if m.err != nil {
|
||||
return false
|
||||
}
|
||||
if m.index >= len(m.data) {
|
||||
return false
|
||||
}
|
||||
m.index++
|
||||
return true
|
||||
}
|
||||
|
||||
func (m *mockRows) Scan(dest ...any) error {
|
||||
if m.err != nil {
|
||||
return m.err
|
||||
}
|
||||
if m.index == 0 || m.index > len(m.data) {
|
||||
return nil
|
||||
}
|
||||
row := m.data[m.index-1]
|
||||
for i := range dest {
|
||||
if i >= len(row) {
|
||||
break
|
||||
}
|
||||
if err := assignMockScanValue(dest[i], row[i]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *mockRows) ScanStruct(_ any) error { return nil }
|
||||
|
||||
func (m *mockRows) ColumnTypes() []driver.ColumnType { return nil }
|
||||
|
||||
func (m *mockRows) Totals(_ ...any) error { return nil }
|
||||
|
||||
func (m *mockRows) Columns() []string { return nil }
|
||||
|
||||
func (m *mockRows) Close() error { return nil }
|
||||
|
||||
func (m *mockRows) Err() error { return m.err }
|
||||
|
||||
func (m *mockRows) HasData() bool { return len(m.data) > 0 }
|
||||
|
||||
func assignMockScanValue(dest any, value any) error {
|
||||
switch d := dest.(type) {
|
||||
case *string:
|
||||
if v, ok := value.(string); ok {
|
||||
*d = v
|
||||
}
|
||||
case *uint64:
|
||||
switch v := value.(type) {
|
||||
case uint64:
|
||||
*d = v
|
||||
case int:
|
||||
*d = uint64(v)
|
||||
case int64:
|
||||
*d = uint64(v)
|
||||
}
|
||||
case *int64:
|
||||
switch v := value.(type) {
|
||||
case int64:
|
||||
*d = v
|
||||
case int:
|
||||
*d = int64(v)
|
||||
case uint64:
|
||||
*d = int64(v)
|
||||
}
|
||||
case *float64:
|
||||
switch v := value.(type) {
|
||||
case float64:
|
||||
*d = v
|
||||
case float32:
|
||||
*d = float64(v)
|
||||
case int:
|
||||
*d = float64(v)
|
||||
}
|
||||
case *time.Time:
|
||||
if v, ok := value.(time.Time); ok {
|
||||
*d = v
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import "math"
|
||||
|
||||
func safeInt64Count(count uint64) int64 {
|
||||
if count > math.MaxInt64 {
|
||||
return math.MaxInt64
|
||||
}
|
||||
return int64(count)
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"math"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSafeInt64Count(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
count uint64
|
||||
want int64
|
||||
}{
|
||||
{name: "zero", count: 0, want: 0},
|
||||
{name: "small", count: 42, want: 42},
|
||||
{name: "max int64", count: math.MaxInt64, want: math.MaxInt64},
|
||||
{name: "overflow clamps", count: math.MaxUint64, want: math.MaxInt64},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
if got := safeInt64Count(tt.count); got != tt.want {
|
||||
t.Fatalf("safeInt64Count(%d) = %d, want %d", tt.count, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+120
@@ -0,0 +1,120 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
|
||||
)
|
||||
|
||||
// DDL TTL days for analytics tables (must match goose ClickHouse migrations).
|
||||
const (
|
||||
// TableTTLDaysNodeAccessLogs is the of_node_access_logs TTL (90 days).
|
||||
TableTTLDaysNodeAccessLogs = 90
|
||||
// TableTTLDaysNodeMetricSnapshots is the of_node_metric_snapshots TTL (30 days).
|
||||
TableTTLDaysNodeMetricSnapshots = 30
|
||||
// TableTTLDaysNodeObs is the of_node_edge_health / of_node_obs_frps / of_node_obs_frpc TTL (30 days).
|
||||
TableTTLDaysNodeObs = 30
|
||||
// TableTTLDaysUserAccessLogs is the w_user_access_logs TTL (180 days).
|
||||
TableTTLDaysUserAccessLogs = 180
|
||||
)
|
||||
|
||||
const (
|
||||
// CleanupModeTTLMaterialize expires rows via table TTL instead of ALTER DELETE mutations.
|
||||
// This is not a hard delete: deleted_count must stay 0; use EligibleCount as an estimate.
|
||||
CleanupModeTTLMaterialize = "ttl_materialize"
|
||||
// CleanupModeTruncate removes all rows via TRUNCATE TABLE (hard delete).
|
||||
CleanupModeTruncate = "truncate"
|
||||
)
|
||||
|
||||
// CleanupOutcome describes a non-mutation ClickHouse cleanup operation.
|
||||
//
|
||||
// For CleanupModeTruncate:
|
||||
// - DeletedCount and EligibleCount are the rows removed by TRUNCATE.
|
||||
//
|
||||
// For CleanupModeTTLMaterialize:
|
||||
// - DeletedCount is always 0 (MATERIALIZE TTL is async / not a counted hard delete).
|
||||
// - EligibleCount is an estimate of rows already past the table TTL policy (not an
|
||||
// arbitrary user cutoff younger than the DDL TTL).
|
||||
// - TableTTLDays is the DDL TTL used for the estimate and materialize.
|
||||
type CleanupOutcome struct {
|
||||
EligibleCount int64
|
||||
DeletedCount int64
|
||||
Mode string
|
||||
TableTTLDays int
|
||||
}
|
||||
|
||||
func countClickHouseRows(ctx context.Context, conn driver.Conn, countSQL string, countArgs []any) (int64, error) {
|
||||
var count uint64
|
||||
if err := conn.QueryRow(ctx, countSQL, countArgs...).Scan(&count); err != nil {
|
||||
return 0, fmt.Errorf("count clickhouse rows: %w", err)
|
||||
}
|
||||
return safeInt64Count(count), nil
|
||||
}
|
||||
|
||||
func materializeTableTTL(ctx context.Context, conn driver.Conn, tableName string) error {
|
||||
sql := fmt.Sprintf("ALTER TABLE %s MATERIALIZE TTL", tableName)
|
||||
if err := conn.Exec(ctx, sql); err != nil {
|
||||
return fmt.Errorf("materialize ttl on %s: %w", tableName, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// tableTTLCutoff returns the UTC instant at which rows become eligible under a fixed day TTL.
|
||||
func tableTTLCutoff(tableTTLDays int, now time.Time) time.Time {
|
||||
if tableTTLDays < 1 {
|
||||
tableTTLDays = 1
|
||||
}
|
||||
return now.UTC().Add(-time.Duration(tableTTLDays) * 24 * time.Hour)
|
||||
}
|
||||
|
||||
// materializeExpiredByTableTTL force-materializes table TTL and estimates rows past that policy.
|
||||
//
|
||||
// countSQL must count only rows older than the table TTL (callers pass tableTTLCutoff args).
|
||||
// Node-scoped filters may be used for the estimate only; MATERIALIZE is always table-global.
|
||||
func materializeExpiredByTableTTL(
|
||||
ctx context.Context,
|
||||
conn driver.Conn,
|
||||
tableName string,
|
||||
tableTTLDays int,
|
||||
countSQL string,
|
||||
countArgs []any,
|
||||
) (CleanupOutcome, error) {
|
||||
outcome := CleanupOutcome{
|
||||
Mode: CleanupModeTTLMaterialize,
|
||||
TableTTLDays: tableTTLDays,
|
||||
}
|
||||
count, err := countClickHouseRows(ctx, conn, countSQL, countArgs)
|
||||
if err != nil {
|
||||
return CleanupOutcome{}, err
|
||||
}
|
||||
outcome.EligibleCount = count
|
||||
// Always force materialize so ClickHouse applies the DDL TTL policy promptly.
|
||||
// EligibleCount is informational only; MATERIALIZE does not return a deleted row count.
|
||||
if err := materializeTableTTL(ctx, conn, tableName); err != nil {
|
||||
return CleanupOutcome{}, err
|
||||
}
|
||||
return outcome, nil
|
||||
}
|
||||
|
||||
func truncateClickHouseTable(ctx context.Context, conn driver.Conn, tableName string) (CleanupOutcome, error) {
|
||||
count, err := countClickHouseRows(ctx, conn, "SELECT count() FROM "+tableName, nil)
|
||||
if err != nil {
|
||||
return CleanupOutcome{}, err
|
||||
}
|
||||
if count == 0 {
|
||||
return CleanupOutcome{Mode: CleanupModeTruncate}, nil
|
||||
}
|
||||
if err := conn.Exec(ctx, "TRUNCATE TABLE "+tableName); err != nil {
|
||||
return CleanupOutcome{}, fmt.Errorf("truncate %s: %w", tableName, err)
|
||||
}
|
||||
return CleanupOutcome{
|
||||
EligibleCount: count,
|
||||
DeletedCount: count,
|
||||
Mode: CleanupModeTruncate,
|
||||
}, nil
|
||||
}
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestTableTTLCutoff(t *testing.T) {
|
||||
now := time.Date(2026, 7, 10, 12, 0, 0, 0, time.UTC)
|
||||
got := tableTTLCutoff(30, now)
|
||||
assert.Equal(t, now.Add(-30*24*time.Hour), got)
|
||||
|
||||
got = tableTTLCutoff(90, now)
|
||||
assert.Equal(t, now.Add(-90*24*time.Hour), got)
|
||||
|
||||
// Invalid TTL floors to 1 day.
|
||||
got = tableTTLCutoff(0, now)
|
||||
assert.Equal(t, now.Add(-24*time.Hour), got)
|
||||
}
|
||||
|
||||
func TestCleanupModeConstants(t *testing.T) {
|
||||
assert.Equal(t, "ttl_materialize", CleanupModeTTLMaterialize)
|
||||
assert.Equal(t, "truncate", CleanupModeTruncate)
|
||||
}
|
||||
|
||||
func TestTableTTLDaysMatchDDL(t *testing.T) {
|
||||
assert.Equal(t, 90, TableTTLDaysNodeAccessLogs)
|
||||
assert.Equal(t, 30, TableTTLDaysNodeMetricSnapshots)
|
||||
assert.Equal(t, 30, TableTTLDaysNodeObs)
|
||||
assert.Equal(t, 180, TableTTLDaysUserAccessLogs)
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
|
||||
"Wavelet/openflare/plugins/server/kernel/runtimeconfig"
|
||||
db "Wavelet/plugins/infra/database"
|
||||
)
|
||||
|
||||
// ClickHouseOperationalStats summarizes ClickHouse merge/mutation pressure
|
||||
// and in-process batch writer queue health.
|
||||
type ClickHouseOperationalStats = analyticsmodel.ClickHouseOperationalStats
|
||||
|
||||
// GetClickHouseOperationalStats returns operational metrics for the configured database.
|
||||
func GetClickHouseOperationalStats(ctx context.Context) (*ClickHouseOperationalStats, error) {
|
||||
if db.ChConn == nil {
|
||||
return nil, errors.New("clickhouse native connection is not initialized")
|
||||
}
|
||||
database := runtimeconfig.Get().ClickHouse.Database
|
||||
stats := &ClickHouseOperationalStats{Database: database}
|
||||
|
||||
partsSQL := `
|
||||
SELECT
|
||||
count() AS active_parts,
|
||||
ifNull(sum(rows), 0) AS total_rows
|
||||
FROM system.parts
|
||||
WHERE active AND database = ?`
|
||||
var activeParts, totalRows uint64
|
||||
if err := db.ChConn.QueryRow(ctx, partsSQL, database).Scan(&activeParts, &totalRows); err != nil {
|
||||
return nil, fmt.Errorf("query system.parts: %w", err)
|
||||
}
|
||||
stats.ActiveParts = safeInt64Count(activeParts)
|
||||
stats.TotalRows = safeInt64Count(totalRows)
|
||||
|
||||
mutationsSQL := `
|
||||
SELECT count()
|
||||
FROM system.mutations
|
||||
WHERE is_done = 0 AND database = ?`
|
||||
if err := db.ChConn.QueryRow(ctx, mutationsSQL, database).Scan(&stats.PendingMutations); err != nil {
|
||||
return nil, fmt.Errorf("query system.mutations: %w", err)
|
||||
}
|
||||
|
||||
asyncSQL := `
|
||||
SELECT
|
||||
count() AS queue_entries,
|
||||
ifNull(sum(bytes), 0) AS queue_bytes
|
||||
FROM system.asynchronous_inserts
|
||||
WHERE database = ?`
|
||||
var queueEntries, queueBytes uint64
|
||||
if err := db.ChConn.QueryRow(ctx, asyncSQL, database).Scan(&queueEntries, &queueBytes); err != nil {
|
||||
// Older ClickHouse versions may not expose asynchronous_inserts; treat as optional.
|
||||
stats.AsyncInsertQueue = 0
|
||||
stats.AsyncInsertBytes = 0
|
||||
} else {
|
||||
stats.AsyncInsertQueue = safeInt64Count(queueEntries)
|
||||
stats.AsyncInsertBytes = safeInt64Count(queueBytes)
|
||||
}
|
||||
|
||||
return stats, nil
|
||||
}
|
||||
@@ -0,0 +1,302 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
|
||||
db "Wavelet/plugins/infra/database"
|
||||
|
||||
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
|
||||
)
|
||||
|
||||
// NodeAccessLogRegionCount aggregates access log regions.
|
||||
type NodeAccessLogRegionCount = analyticsmodel.NodeAccessLogRegionCount
|
||||
|
||||
func nodeAccessLogConn() (driver.Conn, error) {
|
||||
if db.ChConn == nil {
|
||||
return nil, errors.New("clickhouse connection is not initialized")
|
||||
}
|
||||
return db.ChConn, nil
|
||||
}
|
||||
|
||||
// ListNodeAccessLogs returns access logs matching filter.
|
||||
func ListNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter) ([]analyticsmodel.NodeAccessLog, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clause, args := buildNodeAccessLogFilterClause(filter)
|
||||
tableName := nodeAccessLogTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT id, node_id, logged_at, remote_addr, region, host, path, user_agent, cache_status, status_code, bytes_sent, request_length, request_time_ms, created_at
|
||||
FROM %s
|
||||
WHERE %s
|
||||
ORDER BY %s`, tableName, clause, nodeAccessLogOrderClause(filter.SortBy, filter.SortOrder))
|
||||
if filter.PageSize > 0 {
|
||||
if filter.Page < 0 {
|
||||
filter.Page = 0
|
||||
}
|
||||
sql += clickHouseLimitOffsetClause
|
||||
args = append(args, filter.PageSize, filter.Page*filter.PageSize)
|
||||
}
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list node access logs: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
return scanNodeAccessLogRows(rows)
|
||||
}
|
||||
|
||||
//nolint:dupl // scan shapes differ by model fields; shared helper would obscure CH column mapping
|
||||
func scanNodeAccessLogRows(rows driver.Rows) ([]analyticsmodel.NodeAccessLog, error) {
|
||||
var result []analyticsmodel.NodeAccessLog
|
||||
for rows.Next() {
|
||||
var item analyticsmodel.NodeAccessLog
|
||||
if err := rows.Scan(
|
||||
&item.ID,
|
||||
&item.NodeID,
|
||||
&item.LoggedAt,
|
||||
&item.RemoteAddr,
|
||||
&item.Region,
|
||||
&item.Host,
|
||||
&item.Path,
|
||||
&item.UserAgent,
|
||||
&item.CacheStatus,
|
||||
&item.StatusCode,
|
||||
&item.BytesSent,
|
||||
&item.RequestLength,
|
||||
&item.RequestTimeMs,
|
||||
&item.CreatedAt,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan node access log row: %w", err)
|
||||
}
|
||||
item.LoggedAt = item.LoggedAt.UTC()
|
||||
item.CreatedAt = item.CreatedAt.UTC()
|
||||
result = append(result, item)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// CountNodeAccessLogs returns total records, distinct IPs, and total bytes sent matching filter.
|
||||
func CountNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter) (int64, int64, int64, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
return 0, 0, 0, err
|
||||
}
|
||||
clause, args := buildNodeAccessLogFilterClause(filter)
|
||||
tableName := nodeAccessLogTableName()
|
||||
|
||||
countSQL := fmt.Sprintf(`
|
||||
SELECT
|
||||
count() AS total_records,
|
||||
uniqExactIf(remote_addr, remote_addr != '') AS total_ips,
|
||||
sum(bytes_sent) AS total_bytes
|
||||
FROM %s
|
||||
WHERE %s`, tableName, clause)
|
||||
var totalRecords, totalIPs, totalBytes uint64
|
||||
if err := conn.QueryRow(ctx, countSQL, args...).Scan(&totalRecords, &totalIPs, &totalBytes); err != nil {
|
||||
return 0, 0, 0, fmt.Errorf("count node access logs: %w", err)
|
||||
}
|
||||
return safeInt64Count(totalRecords), safeInt64Count(totalIPs), safeInt64Count(totalBytes), nil
|
||||
}
|
||||
|
||||
// RegionCountsNodeAccessLogs returns region counts for a node since a time.
|
||||
func RegionCountsNodeAccessLogs(ctx context.Context, nodeID string, since time.Time, limit int) ([]NodeAccessLogRegionCount, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
filter := NodeAccessLogFilter{NodeID: nodeID, Since: since}
|
||||
clause, args := buildNodeAccessLogFilterClause(filter)
|
||||
tableName := nodeAccessLogTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT trim(region) AS trimmed_region, count() AS count
|
||||
FROM %s
|
||||
WHERE %s AND trim(region) != ''
|
||||
GROUP BY trimmed_region
|
||||
ORDER BY count DESC, trimmed_region ASC`, tableName, clause)
|
||||
if limit > 0 {
|
||||
sql += clickHouseLimitClause
|
||||
args = append(args, limit)
|
||||
}
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("region counts node access logs: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
var result []NodeAccessLogRegionCount
|
||||
for rows.Next() {
|
||||
var (
|
||||
region string
|
||||
count uint64
|
||||
)
|
||||
if err := rows.Scan(®ion, &count); err != nil {
|
||||
return nil, fmt.Errorf("scan region count row: %w", err)
|
||||
}
|
||||
result = append(result, NodeAccessLogRegionCount{
|
||||
Region: region,
|
||||
Count: safeInt64Count(count),
|
||||
})
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// NodeAccessLogTrafficSummary is a window-level access log traffic summary.
|
||||
type NodeAccessLogTrafficSummary = analyticsmodel.NodeAccessLogTrafficSummary
|
||||
|
||||
// NodeAccessLogValueCount is a grouped value count (status_code, host, ...).
|
||||
type NodeAccessLogValueCount = analyticsmodel.NodeAccessLogValueCount
|
||||
|
||||
// NodeAccessLogNodeAggregate is per-node traffic over a window.
|
||||
type NodeAccessLogNodeAggregate = analyticsmodel.NodeAccessLogNodeAggregate
|
||||
|
||||
// TrafficSummaryNodeAccessLogs returns request/error/UV/bytes/node counts for the filter.
|
||||
func TrafficSummaryNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter) (NodeAccessLogTrafficSummary, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
return NodeAccessLogTrafficSummary{}, err
|
||||
}
|
||||
clause, args := buildNodeAccessLogFilterClause(filter)
|
||||
tableName := nodeAccessLogTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT
|
||||
count() AS request_count,
|
||||
countIf(status_code >= 500) AS error_count,
|
||||
uniqExactIf(remote_addr, remote_addr != '') AS unique_ips,
|
||||
sum(bytes_sent) AS bytes_sent,
|
||||
sum(request_length) AS request_length,
|
||||
uniqExactIf(node_id, node_id != '') AS node_count
|
||||
FROM %s
|
||||
WHERE %s`, tableName, clause)
|
||||
var requestCount, errorCount, uniqueIPs, bytesSent, requestLength, nodeCount uint64
|
||||
if err := conn.QueryRow(ctx, sql, args...).Scan(
|
||||
&requestCount, &errorCount, &uniqueIPs, &bytesSent, &requestLength, &nodeCount,
|
||||
); err != nil {
|
||||
return NodeAccessLogTrafficSummary{}, fmt.Errorf("traffic summary node access logs: %w", err)
|
||||
}
|
||||
return NodeAccessLogTrafficSummary{
|
||||
RequestCount: safeInt64Count(requestCount),
|
||||
ErrorCount: safeInt64Count(errorCount),
|
||||
UniqueIPCount: safeInt64Count(uniqueIPs),
|
||||
BytesSent: safeInt64Count(bytesSent),
|
||||
RequestLength: safeInt64Count(requestLength),
|
||||
NodeCount: safeInt64Count(nodeCount),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ValueCountsNodeAccessLogs groups logs by a single dimension column.
|
||||
// Allowed columns: status_code, host, path, remote_addr, user_agent.
|
||||
func ValueCountsNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter, column string, limit int) ([]NodeAccessLogValueCount, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
col := strings.TrimSpace(strings.ToLower(column))
|
||||
valueExpr, ok := nodeAccessLogValueCountExpr(col)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("unsupported value count column: %s", column)
|
||||
}
|
||||
clause, args := buildNodeAccessLogFilterClause(filter)
|
||||
tableName := nodeAccessLogTableName()
|
||||
filterExpr := valueExpr + " != ''"
|
||||
if col == nodeAccessLogColumnStatusCode {
|
||||
filterExpr = "status_code >= 0"
|
||||
}
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT %s AS value, count() AS count
|
||||
FROM %s
|
||||
WHERE %s AND %s
|
||||
GROUP BY value
|
||||
ORDER BY count DESC, value ASC`, valueExpr, tableName, clause, filterExpr)
|
||||
if limit > 0 {
|
||||
sql += clickHouseLimitClause
|
||||
args = append(args, limit)
|
||||
}
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("value counts node access logs: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
var result []NodeAccessLogValueCount
|
||||
for rows.Next() {
|
||||
var (
|
||||
value string
|
||||
count uint64
|
||||
)
|
||||
if err := rows.Scan(&value, &count); err != nil {
|
||||
return nil, fmt.Errorf("scan value count row: %w", err)
|
||||
}
|
||||
result = append(result, NodeAccessLogValueCount{
|
||||
Value: value,
|
||||
Count: safeInt64Count(count),
|
||||
})
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func nodeAccessLogValueCountExpr(column string) (string, bool) {
|
||||
switch column {
|
||||
case nodeAccessLogColumnStatusCode:
|
||||
return "toString(" + nodeAccessLogColumnStatusCode + ")", true
|
||||
case nodeAccessLogColumnHost:
|
||||
return "trim(" + nodeAccessLogColumnHost + ")", true
|
||||
case nodeAccessLogColumnPath:
|
||||
return "trim(" + nodeAccessLogColumnPath + ")", true
|
||||
case nodeAccessLogColumnRemoteAddr:
|
||||
return "trim(" + nodeAccessLogColumnRemoteAddr + ")", true
|
||||
case nodeAccessLogColumnUserAgent:
|
||||
return "trim(" + nodeAccessLogColumnUserAgent + ")", true
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
// NodeAggregatesNodeAccessLogs returns per-node request/error/UV aggregates.
|
||||
func NodeAggregatesNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter) ([]NodeAccessLogNodeAggregate, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clause, args := buildNodeAccessLogFilterClause(filter)
|
||||
tableName := nodeAccessLogTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT
|
||||
node_id,
|
||||
count() AS request_count,
|
||||
countIf(status_code >= 500) AS error_count,
|
||||
uniqExactIf(remote_addr, remote_addr != '') AS unique_ips
|
||||
FROM %s
|
||||
WHERE %s AND node_id != ''
|
||||
GROUP BY node_id
|
||||
ORDER BY request_count DESC, node_id ASC`, tableName, clause)
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("node aggregates node access logs: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
var result []NodeAccessLogNodeAggregate
|
||||
for rows.Next() {
|
||||
var (
|
||||
nodeID string
|
||||
requestCount, errorCount, uniqueIPs uint64
|
||||
)
|
||||
if err := rows.Scan(&nodeID, &requestCount, &errorCount, &uniqueIPs); err != nil {
|
||||
return nil, fmt.Errorf("scan node aggregate row: %w", err)
|
||||
}
|
||||
result = append(result, NodeAccessLogNodeAggregate{
|
||||
NodeID: nodeID,
|
||||
RequestCount: safeInt64Count(requestCount),
|
||||
ErrorCount: safeInt64Count(errorCount),
|
||||
UniqueIPCount: safeInt64Count(uniqueIPs),
|
||||
})
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
+87
@@ -0,0 +1,87 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// DeleteAllNodeAccessLogs hard-deletes all node access logs via TRUNCATE.
|
||||
func DeleteAllNodeAccessLogs(ctx context.Context) (int64, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
outcome, err := truncateClickHouseTable(ctx, conn, nodeAccessLogTableName())
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return outcome.DeletedCount, nil
|
||||
}
|
||||
|
||||
// DeleteNodeAccessLogsBefore force-materializes of_node_access_logs table TTL.
|
||||
//
|
||||
// The cutoff argument is kept for call-site compatibility and is not used to select rows:
|
||||
// ClickHouse MATERIALIZE TTL only enforces the DDL policy (TableTTLDaysNodeAccessLogs).
|
||||
// Returns an estimate of rows past table TTL as the int64 (not a hard-deleted count).
|
||||
// Callers that need honest API fields should prefer MaterializeNodeAccessLogsTTL.
|
||||
func DeleteNodeAccessLogsBefore(ctx context.Context, _ time.Time) (int64, error) {
|
||||
outcome, err := MaterializeNodeAccessLogsTTL(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return outcome.EligibleCount, nil
|
||||
}
|
||||
|
||||
// MaterializeNodeAccessLogsTTL force-materializes table TTL and reports an honest outcome.
|
||||
func MaterializeNodeAccessLogsTTL(ctx context.Context) (CleanupOutcome, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
return CleanupOutcome{}, err
|
||||
}
|
||||
tableName := nodeAccessLogTableName()
|
||||
ttlDays := TableTTLDaysNodeAccessLogs
|
||||
cutoff := tableTTLCutoff(ttlDays, time.Now())
|
||||
return materializeExpiredByTableTTL(
|
||||
ctx,
|
||||
conn,
|
||||
tableName,
|
||||
ttlDays,
|
||||
fmt.Sprintf("SELECT count() FROM %s WHERE logged_at < ?", tableName),
|
||||
[]any{cutoff},
|
||||
)
|
||||
}
|
||||
|
||||
// DeleteNodeAccessLogsByNodeBefore force-materializes table-global TTL.
|
||||
//
|
||||
// Node-scoped hard delete is not supported: MATERIALIZE TTL is table-global.
|
||||
// The returned count is an estimate of rows for nodeID past table TTL only.
|
||||
func DeleteNodeAccessLogsByNodeBefore(ctx context.Context, nodeID string, _ time.Time) (int64, error) {
|
||||
outcome, err := MaterializeNodeAccessLogsTTLByNode(ctx, nodeID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return outcome.EligibleCount, nil
|
||||
}
|
||||
|
||||
// MaterializeNodeAccessLogsTTLByNode materializes table-global TTL and estimates node-scoped rows past TTL.
|
||||
func MaterializeNodeAccessLogsTTLByNode(ctx context.Context, nodeID string) (CleanupOutcome, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
return CleanupOutcome{}, err
|
||||
}
|
||||
tableName := nodeAccessLogTableName()
|
||||
ttlDays := TableTTLDaysNodeAccessLogs
|
||||
cutoff := tableTTLCutoff(ttlDays, time.Now())
|
||||
return materializeExpiredByTableTTL(
|
||||
ctx,
|
||||
conn,
|
||||
tableName,
|
||||
ttlDays,
|
||||
fmt.Sprintf("SELECT count() FROM %s WHERE node_id = ? AND logged_at < ?", tableName),
|
||||
[]any{nodeID, cutoff},
|
||||
)
|
||||
}
|
||||
+191
@@ -0,0 +1,191 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
|
||||
"Wavelet/pkg/util"
|
||||
)
|
||||
|
||||
const (
|
||||
nodeAccessLogFilterClauseCapacity = 7
|
||||
|
||||
nodeAccessLogSortDesc = "DESC"
|
||||
nodeAccessLogSortAsc = "ASC"
|
||||
nodeAccessLogSortAscInput = "asc"
|
||||
|
||||
nodeAccessLogColumnRemoteAddr = "remote_addr"
|
||||
nodeAccessLogColumnStatusCode = "status_code"
|
||||
nodeAccessLogColumnHost = "host"
|
||||
nodeAccessLogColumnPath = "path"
|
||||
nodeAccessLogColumnUserAgent = "user_agent"
|
||||
nodeAccessLogColumnLoggedAt = "logged_at"
|
||||
)
|
||||
|
||||
// NodeAccessLogFilter scopes ClickHouse node access log queries.
|
||||
type NodeAccessLogFilter = analyticsmodel.NodeAccessLogFilter
|
||||
|
||||
func buildNodeAccessLogFilterClause(filter NodeAccessLogFilter) (string, []any) {
|
||||
parts := make([]string, 0, nodeAccessLogFilterClauseCapacity)
|
||||
args := make([]any, 0, nodeAccessLogFilterClauseCapacity)
|
||||
if trimmed := strings.TrimSpace(filter.NodeID); trimmed != "" {
|
||||
parts = append(parts, "node_id = ?")
|
||||
args = append(args, trimmed)
|
||||
}
|
||||
if trimmed := normalizeNodeAccessLogRemoteAddr(filter.RemoteAddr); trimmed != "" {
|
||||
parts = append(parts, "remote_addr LIKE ?")
|
||||
args = append(args, util.EscapeLike(trimmed)+"%")
|
||||
}
|
||||
hosts := normalizeNodeAccessLogHosts(filter.Hosts)
|
||||
if len(hosts) > 0 {
|
||||
placeholders := make([]string, 0, len(hosts))
|
||||
for _, host := range hosts {
|
||||
placeholders = append(placeholders, "?")
|
||||
args = append(args, host)
|
||||
}
|
||||
parts = append(parts, "lowerUTF8(trim(host)) IN ("+strings.Join(placeholders, ", ")+")")
|
||||
} else if trimmed := strings.TrimSpace(filter.Host); trimmed != "" {
|
||||
parts = append(parts, "host LIKE ?")
|
||||
args = append(args, util.EscapeLike(trimmed)+"%")
|
||||
}
|
||||
if trimmed := strings.TrimSpace(filter.Path); trimmed != "" {
|
||||
parts = append(parts, "path LIKE ?")
|
||||
args = append(args, util.EscapeLike(trimmed)+"%")
|
||||
}
|
||||
if filter.StatusCode > 0 {
|
||||
parts = append(parts, "status_code = ?")
|
||||
args = append(args, filter.StatusCode)
|
||||
}
|
||||
if !filter.Since.IsZero() {
|
||||
parts = append(parts, "logged_at >= ?")
|
||||
args = append(args, filter.Since.UTC())
|
||||
}
|
||||
if !filter.Until.IsZero() {
|
||||
parts = append(parts, "logged_at < ?")
|
||||
args = append(args, filter.Until.UTC())
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return "1", nil
|
||||
}
|
||||
return strings.Join(parts, " AND "), args
|
||||
}
|
||||
|
||||
func combineNodeAccessLogSQLClauses(left string, right string) string {
|
||||
if strings.TrimSpace(left) == "" || left == "TRUE" || left == "1" {
|
||||
return right
|
||||
}
|
||||
return left + " AND " + right
|
||||
}
|
||||
|
||||
func nodeAccessLogOrderClause(sortBy string, sortOrder string) string {
|
||||
direction := nodeAccessLogSortDesc
|
||||
if normalizeNodeAccessLogSortOrder(sortOrder) == nodeAccessLogSortAscInput {
|
||||
direction = nodeAccessLogSortAsc
|
||||
}
|
||||
column := nodeAccessLogColumnLoggedAt
|
||||
switch strings.TrimSpace(sortBy) {
|
||||
case nodeAccessLogColumnStatusCode:
|
||||
column = nodeAccessLogColumnStatusCode
|
||||
case nodeAccessLogColumnRemoteAddr:
|
||||
column = nodeAccessLogColumnRemoteAddr
|
||||
case nodeAccessLogColumnHost:
|
||||
column = nodeAccessLogColumnHost
|
||||
case nodeAccessLogColumnPath:
|
||||
column = nodeAccessLogColumnPath
|
||||
}
|
||||
if column == nodeAccessLogColumnLoggedAt {
|
||||
return column + " " + direction + ", id " + direction
|
||||
}
|
||||
return column + " " + direction + ", " + nodeAccessLogColumnLoggedAt + " " + direction + ", id " + direction
|
||||
}
|
||||
|
||||
func normalizeNodeAccessLogRemoteAddr(value string) string {
|
||||
return strings.TrimSpace(value)
|
||||
}
|
||||
|
||||
func normalizeNodeAccessLogHosts(hosts []string) []string {
|
||||
if len(hosts) == 0 {
|
||||
return nil
|
||||
}
|
||||
seen := make(map[string]struct{}, len(hosts))
|
||||
result := make([]string, 0, len(hosts))
|
||||
for _, host := range hosts {
|
||||
trimmed := strings.ToLower(strings.TrimSpace(host))
|
||||
if trimmed == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[trimmed]; ok {
|
||||
continue
|
||||
}
|
||||
seen[trimmed] = struct{}{}
|
||||
result = append(result, trimmed)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func normalizeNodeAccessLogSortOrder(sortOrder string) string {
|
||||
if strings.EqualFold(strings.TrimSpace(sortOrder), "asc") {
|
||||
return "asc"
|
||||
}
|
||||
return "desc"
|
||||
}
|
||||
|
||||
func nodeAccessLogBucketEpochExpr(bucketSeconds int64) string {
|
||||
return fmt.Sprintf("toInt64(intDiv(toUnixTimestamp(logged_at), %d) * %d)", bucketSeconds, bucketSeconds)
|
||||
}
|
||||
|
||||
func nodeAccessLogEpochExpr() string {
|
||||
return "toInt64(toUnixTimestamp(logged_at))"
|
||||
}
|
||||
|
||||
func nodeAccessLogHostIsIPLiteralExpr() string {
|
||||
return `(
|
||||
toIPv4OrNull(trim(if(position(trim(host), ':') > 0 AND NOT startsWith(trim(host), '['), splitByChar(':', trim(host))[1], replaceRegexpAll(trim(host), '\\[|\\]', '')))) IS NOT NULL
|
||||
OR toIPv6OrNull(trim(if(position(trim(host), ':') > 0 AND NOT startsWith(trim(host), '['), splitByChar(':', trim(host))[1], replaceRegexpAll(trim(host), '\\[|\\]', '')))) IS NOT NULL
|
||||
)`
|
||||
}
|
||||
|
||||
func nodeAccessLogBucketOrderClause(sortBy string, sortOrder string) string {
|
||||
direction := nodeAccessLogSortDesc
|
||||
if normalizeNodeAccessLogSortOrder(sortOrder) == nodeAccessLogSortAscInput {
|
||||
direction = nodeAccessLogSortAsc
|
||||
}
|
||||
switch strings.TrimSpace(sortBy) {
|
||||
case "request_count":
|
||||
return "request_count " + direction + ", bucket_epoch DESC"
|
||||
default:
|
||||
return "bucket_epoch " + direction
|
||||
}
|
||||
}
|
||||
|
||||
func nodeAccessLogIPSummaryOrderClause(sortBy string, sortOrder string) string {
|
||||
direction := nodeAccessLogSortDesc
|
||||
if normalizeNodeAccessLogSortOrder(sortOrder) == nodeAccessLogSortAscInput {
|
||||
direction = nodeAccessLogSortAsc
|
||||
}
|
||||
column := "total_requests"
|
||||
switch strings.TrimSpace(sortBy) {
|
||||
case "request_length", "bytes_received":
|
||||
column = "request_length"
|
||||
case "bytes_sent":
|
||||
column = "bytes_sent"
|
||||
case "success_ratio":
|
||||
column = "success_ratio"
|
||||
case "last_seen_at":
|
||||
column = "last_seen_epoch"
|
||||
case "recent_requests":
|
||||
// Deprecated sort key; fall back to total_requests.
|
||||
column = "total_requests"
|
||||
case nodeAccessLogColumnRemoteAddr:
|
||||
column = nodeAccessLogColumnRemoteAddr
|
||||
}
|
||||
return column + " " + direction + ", last_seen_epoch DESC, remote_addr ASC"
|
||||
}
|
||||
|
||||
func nodeAccessLogTableName() string {
|
||||
return "of_node_access_logs"
|
||||
}
|
||||
+470
@@ -0,0 +1,470 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
|
||||
)
|
||||
|
||||
// NodeAccessLogBucketAggregate is a folded bucket aggregate row.
|
||||
type NodeAccessLogBucketAggregate = analyticsmodel.NodeAccessLogBucketAggregate
|
||||
|
||||
// NodeAccessLogWAFIPAggregate is a per-IP aggregate row for WAF automatic rules.
|
||||
type NodeAccessLogWAFIPAggregate = analyticsmodel.NodeAccessLogWAFIPAggregate
|
||||
|
||||
// NodeAccessLogBucketDimension is a bucket dimension value.
|
||||
type NodeAccessLogBucketDimension = analyticsmodel.NodeAccessLogBucketDimension
|
||||
|
||||
// NodeAccessLogIPAggregate is an IP aggregate row.
|
||||
type NodeAccessLogIPAggregate = analyticsmodel.NodeAccessLogIPAggregate
|
||||
|
||||
// NodeAccessLogIPSummary is an IP summary row.
|
||||
type NodeAccessLogIPSummary = analyticsmodel.NodeAccessLogIPSummary
|
||||
|
||||
// NodeAccessLogIPTrend is an IP trend bucket row.
|
||||
type NodeAccessLogIPTrend = analyticsmodel.NodeAccessLogIPTrend
|
||||
|
||||
// BucketAggregatesNodeAccessLogs returns folded bucket aggregates with unique IP/host counts.
|
||||
func BucketAggregatesNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter, bucketSeconds int64) ([]NodeAccessLogBucketAggregate, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clause, args := buildNodeAccessLogFilterClause(filter)
|
||||
bucketExpr := nodeAccessLogBucketEpochExpr(bucketSeconds)
|
||||
tableName := nodeAccessLogTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT
|
||||
%s AS bucket_epoch,
|
||||
count() AS request_count,
|
||||
countIf(status_code < 400) AS success_count,
|
||||
countIf(status_code >= 400 AND status_code < 500) AS client_error_count,
|
||||
countIf(status_code >= 500) AS server_error_count,
|
||||
countIf(status_code >= 200 AND status_code < 300) AS status_2xx_count,
|
||||
countIf(status_code >= 400 AND status_code < 500) AS status_4xx_count,
|
||||
countIf(status_code >= 500) AS status_5xx_count,
|
||||
uniqExactIf(remote_addr, remote_addr != '') AS unique_ip_count,
|
||||
uniqExactIf(host, host != '') AS unique_host_count,
|
||||
sum(bytes_sent) AS bytes_sent,
|
||||
sum(request_length) AS request_length
|
||||
FROM %s
|
||||
WHERE %s
|
||||
GROUP BY bucket_epoch
|
||||
ORDER BY %s`, bucketExpr, tableName, clause, nodeAccessLogBucketOrderClause(filter.SortBy, filter.SortOrder))
|
||||
if filter.PageSize > 0 {
|
||||
if filter.Page < 0 {
|
||||
filter.Page = 0
|
||||
}
|
||||
sql += clickHouseLimitOffsetClause
|
||||
args = append(args, filter.PageSize, filter.Page*filter.PageSize)
|
||||
}
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("bucket aggregates node access logs: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
var result []NodeAccessLogBucketAggregate
|
||||
for rows.Next() {
|
||||
var (
|
||||
bucketEpoch int64
|
||||
requestCount, successCount, clientErrorCount, serverErrorCount, status2xxCount, status4xxCount, status5xxCount, uniqueIPCount, uniqueHostCount, bytesSent, requestLength uint64
|
||||
)
|
||||
if err := rows.Scan(&bucketEpoch, &requestCount, &successCount, &clientErrorCount, &serverErrorCount, &status2xxCount, &status4xxCount, &status5xxCount, &uniqueIPCount, &uniqueHostCount, &bytesSent, &requestLength); err != nil {
|
||||
return nil, fmt.Errorf("scan bucket aggregate row: %w", err)
|
||||
}
|
||||
result = append(result, NodeAccessLogBucketAggregate{
|
||||
BucketEpoch: bucketEpoch,
|
||||
RequestCount: safeInt64Count(requestCount),
|
||||
SuccessCount: safeInt64Count(successCount),
|
||||
ClientErrorCount: safeInt64Count(clientErrorCount),
|
||||
ServerErrorCount: safeInt64Count(serverErrorCount),
|
||||
Status2xxCount: safeInt64Count(status2xxCount),
|
||||
Status4xxCount: safeInt64Count(status4xxCount),
|
||||
Status5xxCount: safeInt64Count(status5xxCount),
|
||||
UniqueIPCount: safeInt64Count(uniqueIPCount),
|
||||
UniqueHostCount: safeInt64Count(uniqueHostCount),
|
||||
BytesSent: safeInt64Count(bytesSent),
|
||||
RequestLength: safeInt64Count(requestLength),
|
||||
})
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// CountBucketAggregatesNodeAccessLogs returns the number of folded buckets matching filter.
|
||||
func CountBucketAggregatesNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter, bucketSeconds int64) (int64, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
clause, args := buildNodeAccessLogFilterClause(filter)
|
||||
bucketExpr := nodeAccessLogBucketEpochExpr(bucketSeconds)
|
||||
tableName := nodeAccessLogTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT count() FROM (
|
||||
SELECT 1
|
||||
FROM %s
|
||||
WHERE %s
|
||||
GROUP BY %s
|
||||
)`, tableName, clause, bucketExpr)
|
||||
var totalBuckets uint64
|
||||
if err := conn.QueryRow(ctx, sql, args...).Scan(&totalBuckets); err != nil {
|
||||
return 0, fmt.Errorf("count bucket aggregates node access logs: %w", err)
|
||||
}
|
||||
return safeInt64Count(totalBuckets), nil
|
||||
}
|
||||
|
||||
// BucketDimensionsNodeAccessLogs returns bucket dimension values.
|
||||
func BucketDimensionsNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter, column string, bucketSeconds int64) ([]NodeAccessLogBucketDimension, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clause, args := buildNodeAccessLogFilterClause(filter)
|
||||
bucketExpr := nodeAccessLogBucketEpochExpr(bucketSeconds)
|
||||
tableName := nodeAccessLogTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT
|
||||
%s AS bucket_epoch,
|
||||
trim(%s) AS value
|
||||
FROM %s
|
||||
WHERE %s AND trim(%s) != ''
|
||||
GROUP BY bucket_epoch, trim(%s)`, bucketExpr, column, tableName, clause, column, column)
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("bucket dimensions node access logs: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
var result []NodeAccessLogBucketDimension
|
||||
for rows.Next() {
|
||||
var item NodeAccessLogBucketDimension
|
||||
if err := rows.Scan(&item.BucketEpoch, &item.Value); err != nil {
|
||||
return nil, fmt.Errorf("scan bucket dimension row: %w", err)
|
||||
}
|
||||
result = append(result, item)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// IPAggregatesNodeAccessLogs returns IP aggregate rows.
|
||||
func IPAggregatesNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter, exactRemoteAddr bool) ([]NodeAccessLogIPAggregate, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clause, args := buildNodeAccessLogFilterClause(filter)
|
||||
queryClause := clause
|
||||
queryArgs := append([]any{}, args...)
|
||||
if exactRemoteAddr {
|
||||
trimmed := normalizeNodeAccessLogRemoteAddr(filter.RemoteAddr)
|
||||
if trimmed == "" {
|
||||
return []NodeAccessLogIPAggregate{}, nil
|
||||
}
|
||||
queryClause = combineNodeAccessLogSQLClauses(queryClause, "remote_addr = ?")
|
||||
queryArgs = append(queryArgs, trimmed)
|
||||
}
|
||||
lastSeenExpr := nodeAccessLogEpochExpr()
|
||||
tableName := nodeAccessLogTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT
|
||||
remote_addr,
|
||||
count() AS request_count,
|
||||
countIf(status_code < 400) AS success_count,
|
||||
countIf(status_code >= 400 AND status_code < 500) AS client_error_count,
|
||||
countIf(status_code >= 500) AS server_error_count,
|
||||
max(%s) AS last_seen_epoch
|
||||
FROM %s
|
||||
WHERE %s AND remote_addr != ''
|
||||
GROUP BY remote_addr`, lastSeenExpr, tableName, queryClause)
|
||||
rows, err := conn.Query(ctx, sql, queryArgs...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ip aggregates node access logs: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
var result []NodeAccessLogIPAggregate
|
||||
for rows.Next() {
|
||||
var (
|
||||
remoteAddr string
|
||||
lastSeenEpoch int64
|
||||
requestCount, successCount, clientErrorCount, serverErrorCount uint64
|
||||
)
|
||||
if err := rows.Scan(&remoteAddr, &requestCount, &successCount, &clientErrorCount, &serverErrorCount, &lastSeenEpoch); err != nil {
|
||||
return nil, fmt.Errorf("scan ip aggregate row: %w", err)
|
||||
}
|
||||
result = append(result, NodeAccessLogIPAggregate{
|
||||
RemoteAddr: remoteAddr,
|
||||
RequestCount: safeInt64Count(requestCount),
|
||||
SuccessCount: safeInt64Count(successCount),
|
||||
ClientErrorCount: safeInt64Count(clientErrorCount),
|
||||
ServerErrorCount: safeInt64Count(serverErrorCount),
|
||||
LastSeenEpoch: lastSeenEpoch,
|
||||
})
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// IPSummariesNodeAccessLogs returns paginated IP summary rows for the filter window.
|
||||
// recentSince is ignored (kept for call-site compatibility); recent_requests is always 0.
|
||||
func IPSummariesNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter, _ time.Time) ([]NodeAccessLogIPSummary, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clause, args := buildNodeAccessLogFilterClause(filter)
|
||||
lastSeenExpr := nodeAccessLogEpochExpr()
|
||||
queryArgs := append([]any{}, args...)
|
||||
tableName := nodeAccessLogTableName()
|
||||
// Outer query allows ORDER BY success_ratio without repeating countIf.
|
||||
innerSQL := fmt.Sprintf(`
|
||||
SELECT
|
||||
remote_addr,
|
||||
argMax(region, logged_at) AS region,
|
||||
count() AS total_requests,
|
||||
countIf(status_code >= 200 AND status_code < 300) AS success_2xx_count,
|
||||
sum(request_length) AS request_length,
|
||||
sum(bytes_sent) AS bytes_sent,
|
||||
max(%s) AS last_seen_epoch
|
||||
FROM %s
|
||||
WHERE %s AND remote_addr != ''
|
||||
GROUP BY remote_addr`, lastSeenExpr, tableName, clause)
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT
|
||||
remote_addr,
|
||||
region,
|
||||
total_requests,
|
||||
success_2xx_count,
|
||||
if(total_requests = 0, 0., toFloat64(success_2xx_count) / toFloat64(total_requests)) AS success_ratio,
|
||||
request_length,
|
||||
bytes_sent,
|
||||
last_seen_epoch
|
||||
FROM (%s)
|
||||
ORDER BY %s`, innerSQL, nodeAccessLogIPSummaryOrderClause(filter.SortBy, filter.SortOrder))
|
||||
if filter.PageSize > 0 {
|
||||
if filter.Page < 0 {
|
||||
filter.Page = 0
|
||||
}
|
||||
sql += clickHouseLimitOffsetClause
|
||||
queryArgs = append(queryArgs, filter.PageSize, filter.Page*filter.PageSize)
|
||||
}
|
||||
rows, err := conn.Query(ctx, sql, queryArgs...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ip summaries node access logs: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
var result []NodeAccessLogIPSummary
|
||||
for rows.Next() {
|
||||
var (
|
||||
remoteAddr, region string
|
||||
lastSeenEpoch int64
|
||||
successRatio float64
|
||||
totalRequests, success2xx, bytesReceived, bytes uint64
|
||||
)
|
||||
if err := rows.Scan(
|
||||
&remoteAddr,
|
||||
®ion,
|
||||
&totalRequests,
|
||||
&success2xx,
|
||||
&successRatio,
|
||||
&bytesReceived,
|
||||
&bytes,
|
||||
&lastSeenEpoch,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan ip summary row: %w", err)
|
||||
}
|
||||
result = append(result, NodeAccessLogIPSummary{
|
||||
RemoteAddr: remoteAddr,
|
||||
Region: region,
|
||||
TotalRequests: safeInt64Count(totalRequests),
|
||||
Success2xxCount: safeInt64Count(success2xx),
|
||||
SuccessRatio: successRatio,
|
||||
BytesReceived: safeInt64Count(bytesReceived),
|
||||
BytesSent: safeInt64Count(bytes),
|
||||
RecentRequests: 0,
|
||||
LastSeenEpoch: lastSeenEpoch,
|
||||
})
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// CountIPSummaryNodeAccessLogs returns the number of distinct IPs matching filter.
|
||||
func CountIPSummaryNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter) (int64, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
clause, args := buildNodeAccessLogFilterClause(filter)
|
||||
tableName := nodeAccessLogTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT count() FROM (
|
||||
SELECT 1
|
||||
FROM %s
|
||||
WHERE %s AND remote_addr != ''
|
||||
GROUP BY remote_addr
|
||||
)`, tableName, clause)
|
||||
var totalIPs uint64
|
||||
if err := conn.QueryRow(ctx, sql, args...).Scan(&totalIPs); err != nil {
|
||||
return 0, fmt.Errorf("count ip summary node access logs: %w", err)
|
||||
}
|
||||
return safeInt64Count(totalIPs), nil
|
||||
}
|
||||
|
||||
// IPAggregatesForWAFNodeAccessLogs returns per-IP aggregates for WAF automatic rules.
|
||||
func IPAggregatesForWAFNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter) ([]NodeAccessLogWAFIPAggregate, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clause, args := buildNodeAccessLogFilterClause(filter)
|
||||
lastSeenExpr := nodeAccessLogEpochExpr()
|
||||
hostIsIPExpr := nodeAccessLogHostIsIPLiteralExpr()
|
||||
tableName := nodeAccessLogTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT
|
||||
remote_addr,
|
||||
count() AS request_count,
|
||||
countIf(status_code = 404) AS status_404_count,
|
||||
countIf(status_code >= 400 AND status_code < 500) AS client_error_count,
|
||||
countIf(status_code >= 500) AS server_error_count,
|
||||
countIf(%s) AS ip_host_count,
|
||||
max(%s) AS last_seen_epoch
|
||||
FROM %s
|
||||
WHERE %s AND remote_addr != ''
|
||||
GROUP BY remote_addr`, hostIsIPExpr, lastSeenExpr, tableName, clause)
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ip aggregates for waf node access logs: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
aggregates := make(map[string]*NodeAccessLogWAFIPAggregate)
|
||||
order := make([]string, 0)
|
||||
for rows.Next() {
|
||||
var (
|
||||
remoteAddr string
|
||||
lastSeenEpoch int64
|
||||
requestCount, status404Count, clientErrorCount, serverErrorCount, ipHostCount uint64
|
||||
)
|
||||
if err := rows.Scan(&remoteAddr, &requestCount, &status404Count, &clientErrorCount, &serverErrorCount, &ipHostCount, &lastSeenEpoch); err != nil {
|
||||
return nil, fmt.Errorf("scan waf ip aggregate row: %w", err)
|
||||
}
|
||||
remoteAddr = strings.TrimSpace(remoteAddr)
|
||||
if remoteAddr == "" {
|
||||
continue
|
||||
}
|
||||
aggregates[remoteAddr] = &NodeAccessLogWAFIPAggregate{
|
||||
RemoteAddr: remoteAddr,
|
||||
RequestCount: safeInt64Count(requestCount),
|
||||
Status404Count: safeInt64Count(status404Count),
|
||||
ClientErrorCount: safeInt64Count(clientErrorCount),
|
||||
ServerErrorCount: safeInt64Count(serverErrorCount),
|
||||
IPHostCount: safeInt64Count(ipHostCount),
|
||||
LastSeenEpoch: lastSeenEpoch,
|
||||
StatusCounts: make(map[int]int64),
|
||||
}
|
||||
order = append(order, remoteAddr)
|
||||
}
|
||||
if err := mergeWAFIPStatusCodeCounts(ctx, filter, aggregates); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := make([]NodeAccessLogWAFIPAggregate, 0, len(order))
|
||||
for _, remoteAddr := range order {
|
||||
if aggregate := aggregates[remoteAddr]; aggregate != nil {
|
||||
result = append(result, *aggregate)
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func mergeWAFIPStatusCodeCounts(ctx context.Context, filter NodeAccessLogFilter, aggregates map[string]*NodeAccessLogWAFIPAggregate) error {
|
||||
if len(aggregates) == 0 {
|
||||
return nil
|
||||
}
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
clause, args := buildNodeAccessLogFilterClause(filter)
|
||||
tableName := nodeAccessLogTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT
|
||||
remote_addr,
|
||||
status_code,
|
||||
count() AS status_count
|
||||
FROM %s
|
||||
WHERE %s AND remote_addr != ''
|
||||
GROUP BY remote_addr, status_code`, tableName, clause)
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("waf ip status code counts: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
for rows.Next() {
|
||||
var (
|
||||
remoteAddr string
|
||||
statusCode int32
|
||||
statusCount uint64
|
||||
)
|
||||
if err := rows.Scan(&remoteAddr, &statusCode, &statusCount); err != nil {
|
||||
return fmt.Errorf("scan waf ip status code row: %w", err)
|
||||
}
|
||||
remoteAddr = strings.TrimSpace(remoteAddr)
|
||||
aggregate := aggregates[remoteAddr]
|
||||
if aggregate == nil {
|
||||
continue
|
||||
}
|
||||
if aggregate.StatusCounts == nil {
|
||||
aggregate.StatusCounts = make(map[int]int64)
|
||||
}
|
||||
aggregate.StatusCounts[int(statusCode)] = safeInt64Count(statusCount)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// IPTrendNodeAccessLogs returns IP trend bucket rows.
|
||||
func IPTrendNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter, bucketSeconds int64) ([]NodeAccessLogIPTrend, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clause, args := buildNodeAccessLogFilterClause(filter)
|
||||
bucketExpr := nodeAccessLogBucketEpochExpr(bucketSeconds)
|
||||
tableName := nodeAccessLogTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT
|
||||
%s AS bucket_epoch,
|
||||
count() AS request_count
|
||||
FROM %s
|
||||
WHERE %s
|
||||
GROUP BY bucket_epoch
|
||||
ORDER BY bucket_epoch ASC`, bucketExpr, tableName, clause)
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ip trend node access logs: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
var result []NodeAccessLogIPTrend
|
||||
for rows.Next() {
|
||||
var (
|
||||
bucketEpoch int64
|
||||
requestCount uint64
|
||||
)
|
||||
if err := rows.Scan(&bucketEpoch, &requestCount); err != nil {
|
||||
return nil, fmt.Errorf("scan ip trend row: %w", err)
|
||||
}
|
||||
result = append(result, NodeAccessLogIPTrend{
|
||||
BucketEpoch: bucketEpoch,
|
||||
RequestCount: safeInt64Count(requestCount),
|
||||
})
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
|
||||
"Wavelet/pkg/idgen"
|
||||
db "Wavelet/plugins/infra/database"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestBatchInsertNodeAccessLogs_Empty(t *testing.T) {
|
||||
err := BatchInsertNodeAccessLogs(context.Background(), nil)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestBatchInsertNodeAccessLogs_UsesModelBatchSQL(t *testing.T) {
|
||||
require.NoError(t, idgen.Init(1))
|
||||
ctx := context.Background()
|
||||
mockBatch := &mockBatch{}
|
||||
mockConn := &mockConn{
|
||||
batch: mockBatch,
|
||||
batchQuery: analyticsmodel.NodeAccessLog{}.BatchInsertSQL(),
|
||||
}
|
||||
db.SetChConnForTest(mockConn)
|
||||
t.Cleanup(func() { db.SetChConnForTest(nil) })
|
||||
|
||||
loggedAt := time.Now().UTC()
|
||||
err := BatchInsertNodeAccessLogs(ctx, []analyticsmodel.NodeAccessLog{
|
||||
{
|
||||
NodeID: "node-a",
|
||||
LoggedAt: loggedAt,
|
||||
RemoteAddr: "1.1.1.1",
|
||||
Region: "US",
|
||||
Host: "example.com",
|
||||
Path: "/alpha",
|
||||
StatusCode: 200,
|
||||
BytesSent: 2048,
|
||||
CreatedAt: loggedAt,
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.True(t, mockConn.prepareCalled)
|
||||
assert.Equal(t, analyticsmodel.NodeAccessLog{}.BatchInsertSQL(), mockConn.preparedQuery)
|
||||
assert.True(t, mockBatch.sendCalled)
|
||||
require.Len(t, mockBatch.rows, 1)
|
||||
assert.Equal(t, "node-a", mockBatch.rows[0][1])
|
||||
require.Len(t, mockBatch.rows[0], 14)
|
||||
assert.Empty(t, mockBatch.rows[0][7]) // user_agent
|
||||
assert.Empty(t, mockBatch.rows[0][8]) // cache_status
|
||||
assert.Equal(t, uint64(2048), mockBatch.rows[0][10]) // bytes_sent
|
||||
assert.Equal(t, uint64(0), mockBatch.rows[0][11]) // request_length
|
||||
assert.Equal(t, uint32(0), mockBatch.rows[0][12]) // request_time_ms
|
||||
}
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
|
||||
"Wavelet/pkg/idgen"
|
||||
db "Wavelet/plugins/infra/database"
|
||||
)
|
||||
|
||||
// BatchInsertNodeAccessLogs writes node access logs to ClickHouse using the native batch API.
|
||||
func BatchInsertNodeAccessLogs(ctx context.Context, logs []analyticsmodel.NodeAccessLog) error {
|
||||
if len(logs) == 0 {
|
||||
return nil
|
||||
}
|
||||
if db.ChConn == nil {
|
||||
return errors.New("clickhouse connection is not initialized")
|
||||
}
|
||||
|
||||
batch, err := db.ChConn.PrepareBatch(ctx, analyticsmodel.NodeAccessLog{}.BatchInsertSQL())
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare clickhouse batch: %w", err)
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
for _, logItem := range logs {
|
||||
id := logItem.ID
|
||||
if id == 0 {
|
||||
id = idgen.NextUint64ID()
|
||||
}
|
||||
createdAt := logItem.CreatedAt
|
||||
if createdAt.IsZero() {
|
||||
createdAt = now
|
||||
}
|
||||
if err := batch.Append(
|
||||
id,
|
||||
logItem.NodeID,
|
||||
logItem.LoggedAt.UTC(),
|
||||
strings.TrimSpace(logItem.RemoteAddr),
|
||||
logItem.Region,
|
||||
logItem.Host,
|
||||
logItem.Path,
|
||||
strings.TrimSpace(logItem.UserAgent),
|
||||
strings.TrimSpace(logItem.CacheStatus),
|
||||
logItem.StatusCode,
|
||||
logItem.BytesSent,
|
||||
logItem.RequestLength,
|
||||
logItem.RequestTimeMs,
|
||||
createdAt.UTC(),
|
||||
); err != nil {
|
||||
return fmt.Errorf("append node access log to batch: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := batch.Send(); err != nil {
|
||||
return fmt.Errorf("send clickhouse batch: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,512 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
|
||||
db "Wavelet/plugins/infra/database"
|
||||
|
||||
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
|
||||
)
|
||||
|
||||
func observabilityConn() (driver.Conn, error) {
|
||||
if db.ChConn == nil {
|
||||
return nil, errors.New("clickhouse connection is not initialized")
|
||||
}
|
||||
return db.ChConn, nil
|
||||
}
|
||||
|
||||
// ListNodeMetricSnapshots returns metric snapshots matching filter.
|
||||
func ListNodeMetricSnapshots(ctx context.Context, filter NodeObservabilityFilter) ([]analyticsmodel.NodeMetricSnapshot, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clause, args := buildNodeObservabilityFilterClause(filter, "captured_at")
|
||||
tableName := nodeMetricSnapshotTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT id, node_id, captured_at, cpu_usage_percent, memory_used_bytes, memory_total_bytes, storage_used_bytes, storage_total_bytes, disk_read_bytes, disk_write_bytes, network_rx_bytes, network_tx_bytes, created_at
|
||||
FROM %s
|
||||
WHERE %s
|
||||
ORDER BY %s`, tableName, clause, nodeObservabilityCapturedAtOrderClause())
|
||||
if filter.Limit > 0 {
|
||||
sql += clickHouseLimitClause
|
||||
args = append(args, filter.Limit)
|
||||
}
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list node metric snapshots: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
return scanNodeMetricSnapshotRows(rows)
|
||||
}
|
||||
|
||||
// ListLatestNodeMetricSnapshots returns the latest metric snapshot per node_id.
|
||||
// Uses ClickHouse LIMIT 1 BY so dashboard health does not depend on a global raw LIMIT.
|
||||
func ListLatestNodeMetricSnapshots(ctx context.Context, filter NodeObservabilityFilter) ([]analyticsmodel.NodeMetricSnapshot, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clause, args := buildNodeObservabilityFilterClause(filter, "captured_at")
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT id, node_id, captured_at, cpu_usage_percent, memory_used_bytes, memory_total_bytes, storage_used_bytes, storage_total_bytes, disk_read_bytes, disk_write_bytes, network_rx_bytes, network_tx_bytes, created_at
|
||||
FROM %s
|
||||
WHERE %s
|
||||
ORDER BY %s%s`, nodeMetricSnapshotTableName(), clause, nodeObservabilityCapturedAtOrderClause(), clickHouseLimit1ByNodeIDClause)
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list latest node metric snapshots: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
return scanNodeMetricSnapshotRows(rows)
|
||||
}
|
||||
|
||||
// ListNodeEdgeHealth returns L2 OpenResty health snapshots.
|
||||
func ListNodeEdgeHealth(ctx context.Context, filter NodeObservabilityFilter) ([]analyticsmodel.NodeEdgeHealth, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clause, args := buildNodeObservabilityFilterClause(filter, "captured_at")
|
||||
tableName := nodeEdgeHealthTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT id, node_id, captured_at, status, connections, created_at
|
||||
FROM %s
|
||||
WHERE %s
|
||||
ORDER BY %s`, tableName, clause, nodeObservabilityCapturedAtOrderClause())
|
||||
if filter.Limit > 0 {
|
||||
sql += clickHouseLimitClause
|
||||
args = append(args, filter.Limit)
|
||||
}
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list node edge health: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
var result []analyticsmodel.NodeEdgeHealth
|
||||
for rows.Next() {
|
||||
var item analyticsmodel.NodeEdgeHealth
|
||||
if err := rows.Scan(
|
||||
&item.ID,
|
||||
&item.NodeID,
|
||||
&item.CapturedAt,
|
||||
&item.Status,
|
||||
&item.Connections,
|
||||
&item.CreatedAt,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan node edge health row: %w", err)
|
||||
}
|
||||
item.CapturedAt = item.CapturedAt.UTC()
|
||||
item.CreatedAt = item.CreatedAt.UTC()
|
||||
result = append(result, item)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// ListNodeObsFrps returns FRPS observations matching filter.
|
||||
func ListNodeObsFrps(ctx context.Context, filter NodeObservabilityFilter) ([]analyticsmodel.NodeObsFrps, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clause, args := buildNodeObservabilityFilterClause(filter, "captured_at")
|
||||
tableName := nodeObsFrpsTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT id, node_id, captured_at, frps_connections, frps_proxy_count, frps_client_count, frps_proxies, created_at
|
||||
FROM %s
|
||||
WHERE %s
|
||||
ORDER BY %s`, tableName, clause, nodeObservabilityCapturedAtOrderClause())
|
||||
if filter.Limit > 0 {
|
||||
sql += clickHouseLimitClause
|
||||
args = append(args, filter.Limit)
|
||||
}
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list node frps observations: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
return scanNodeObsFrpsRows(rows)
|
||||
}
|
||||
|
||||
// ListNodeObsFrpc returns FRPC observations matching filter.
|
||||
func ListNodeObsFrpc(ctx context.Context, filter NodeObservabilityFilter) ([]analyticsmodel.NodeObsFrpc, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clause, args := buildNodeObservabilityFilterClause(filter, "captured_at")
|
||||
tableName := nodeObsFrpcTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT id, node_id, captured_at, tunnel_status, connected_relays_count, created_at
|
||||
FROM %s
|
||||
WHERE %s
|
||||
ORDER BY %s`, tableName, clause, nodeObservabilityCapturedAtOrderClause())
|
||||
if filter.Limit > 0 {
|
||||
sql += clickHouseLimitClause
|
||||
args = append(args, filter.Limit)
|
||||
}
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list node frpc observations: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
return scanNodeObsFrpcRows(rows)
|
||||
}
|
||||
|
||||
//nolint:dupl // scan shapes differ by model fields; shared helper would obscure CH column mapping
|
||||
func scanNodeMetricSnapshotRows(rows driver.Rows) ([]analyticsmodel.NodeMetricSnapshot, error) {
|
||||
var result []analyticsmodel.NodeMetricSnapshot
|
||||
for rows.Next() {
|
||||
var item analyticsmodel.NodeMetricSnapshot
|
||||
if err := rows.Scan(
|
||||
&item.ID,
|
||||
&item.NodeID,
|
||||
&item.CapturedAt,
|
||||
&item.CPUUsagePercent,
|
||||
&item.MemoryUsedBytes,
|
||||
&item.MemoryTotalBytes,
|
||||
&item.StorageUsedBytes,
|
||||
&item.StorageTotalBytes,
|
||||
&item.DiskReadBytes,
|
||||
&item.DiskWriteBytes,
|
||||
&item.NetworkRxBytes,
|
||||
&item.NetworkTxBytes,
|
||||
&item.CreatedAt,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan node metric snapshot row: %w", err)
|
||||
}
|
||||
item.CapturedAt = item.CapturedAt.UTC()
|
||||
item.CreatedAt = item.CreatedAt.UTC()
|
||||
result = append(result, item)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func scanNodeObsFrpsRows(rows driver.Rows) ([]analyticsmodel.NodeObsFrps, error) {
|
||||
var result []analyticsmodel.NodeObsFrps
|
||||
for rows.Next() {
|
||||
var item analyticsmodel.NodeObsFrps
|
||||
if err := rows.Scan(
|
||||
&item.ID,
|
||||
&item.NodeID,
|
||||
&item.CapturedAt,
|
||||
&item.FrpsConnections,
|
||||
&item.FrpsProxyCount,
|
||||
&item.FrpsClientCount,
|
||||
&item.FrpsProxies,
|
||||
&item.CreatedAt,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan node frps observation row: %w", err)
|
||||
}
|
||||
item.CapturedAt = item.CapturedAt.UTC()
|
||||
item.CreatedAt = item.CreatedAt.UTC()
|
||||
result = append(result, item)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// NodeTrafficHourly 为小时级流量汇总行(别名,定义见 model/analytics)。
|
||||
type NodeTrafficHourly = analyticsmodel.NodeTrafficHourly
|
||||
|
||||
// NodeMetricHourly 为小时级指标聚合行(别名,定义见 model/analytics)。
|
||||
type NodeMetricHourly = analyticsmodel.NodeMetricHourly
|
||||
|
||||
// ListNodeTrafficHourly returns hourly traffic from of_access_log_hourly (M5).
|
||||
// UniqueVisitorCount is always 0 here (UV requires raw uniqExact on access logs).
|
||||
func ListNodeTrafficHourly(ctx context.Context, filter NodeObservabilityFilter) ([]NodeTrafficHourly, error) {
|
||||
rows, err := ListAccessLogHourly(ctx, filter)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Aggregate across hosts per node/hour.
|
||||
type key struct {
|
||||
node string
|
||||
hour int64
|
||||
}
|
||||
merged := make(map[key]*NodeTrafficHourly)
|
||||
order := make([]key, 0)
|
||||
for _, row := range rows {
|
||||
k := key{node: row.NodeID, hour: row.Hour.UTC().Unix()}
|
||||
item := merged[k]
|
||||
if item == nil {
|
||||
item = &NodeTrafficHourly{NodeID: row.NodeID, Hour: row.Hour.UTC()}
|
||||
merged[k] = item
|
||||
order = append(order, k)
|
||||
}
|
||||
item.RequestCount += row.RequestCount
|
||||
item.ErrorCount += row.ErrorCount
|
||||
}
|
||||
result := make([]NodeTrafficHourly, 0, len(order))
|
||||
for _, k := range order {
|
||||
result = append(result, *merged[k])
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// ListAccessLogHourly returns Server-side access log hourly rollups.
|
||||
func ListAccessLogHourly(ctx context.Context, filter NodeObservabilityFilter) ([]analyticsmodel.AccessLogHourly, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clause, args := buildNodeObservabilityFilterClause(filter, "hour")
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT
|
||||
node_id,
|
||||
hour,
|
||||
host,
|
||||
sum(request_count) AS request_count,
|
||||
sum(error_count) AS error_count,
|
||||
sum(bytes_sent) AS bytes_sent,
|
||||
sum(request_length) AS request_length
|
||||
FROM %s
|
||||
WHERE %s
|
||||
GROUP BY node_id, hour, host
|
||||
ORDER BY hour ASC, node_id ASC, host ASC`, accessLogHourlyTableName(), clause)
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list access log hourly: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
var result []analyticsmodel.AccessLogHourly
|
||||
for rows.Next() {
|
||||
var (
|
||||
item analyticsmodel.AccessLogHourly
|
||||
requestCount, errorCount, bytesSent, requestLength uint64
|
||||
)
|
||||
if err := rows.Scan(
|
||||
&item.NodeID,
|
||||
&item.Hour,
|
||||
&item.Host,
|
||||
&requestCount,
|
||||
&errorCount,
|
||||
&bytesSent,
|
||||
&requestLength,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan access log hourly row: %w", err)
|
||||
}
|
||||
item.Hour = item.Hour.UTC()
|
||||
item.RequestCount = safeInt64Count(requestCount)
|
||||
item.ErrorCount = safeInt64Count(errorCount)
|
||||
item.BytesSent = safeInt64Count(bytesSent)
|
||||
item.RequestLength = safeInt64Count(requestLength)
|
||||
result = append(result, item)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// hourlyRollupMaxLead is how far after filter.Since the earliest rollup bucket may start
|
||||
// while still treating pre-aggregated tables as a complete window (skip raw query).
|
||||
const hourlyRollupMaxLead = 2 * time.Hour
|
||||
|
||||
// hourlyRollupCoversWindow reports whether rollup coverage starts near the requested window.
|
||||
// rows must be ordered by hour ascending.
|
||||
func hourlyRollupCoversWindow(earliestHour time.Time, since time.Time) bool {
|
||||
if since.IsZero() {
|
||||
return true
|
||||
}
|
||||
sinceHour := since.UTC().Truncate(time.Hour)
|
||||
earliest := earliestHour.UTC().Truncate(time.Hour)
|
||||
return !earliest.After(sinceHour.Add(hourlyRollupMaxLead))
|
||||
}
|
||||
|
||||
// ListNodeMetricHourly returns hourly metric snapshot aggregates matching filter.
|
||||
//
|
||||
// Strategy (optimal for correctness + cost):
|
||||
// 1. Load of_node_metric_capacity_hourly rollup.
|
||||
// 2. If rollup spans the window from filter.Since, return it alone (cheap path).
|
||||
// 3. Otherwise load raw lagInFrame aggregates and merge by hour: rollup wins on
|
||||
// overlap, raw fills historical gaps (MV never backfills pre-creation data).
|
||||
func ListNodeMetricHourly(ctx context.Context, filter NodeObservabilityFilter) ([]NodeMetricHourly, error) {
|
||||
rollup, rollupErr := listNodeMetricHourlyFromRollup(ctx, filter)
|
||||
if rollupErr == nil && len(rollup) > 0 && hourlyRollupCoversWindow(rollup[0].Hour, filter.Since) {
|
||||
return rollup, nil
|
||||
}
|
||||
|
||||
raw, rawErr := listNodeMetricHourlyFromRaw(ctx, filter)
|
||||
if rawErr != nil {
|
||||
if rollupErr == nil && len(rollup) > 0 {
|
||||
return rollup, nil
|
||||
}
|
||||
return nil, rawErr
|
||||
}
|
||||
if len(rollup) == 0 {
|
||||
return raw, nil
|
||||
}
|
||||
// Partial rollup (or rollupErr with empty slice): merge; raw fills historical gaps.
|
||||
return mergeNodeMetricHourlyPreferRollup(rollup, raw), nil
|
||||
}
|
||||
|
||||
// mergeNodeMetricHourlyPreferRollup unions two hour series (both ASC by Hour).
|
||||
// Rollup values replace raw for the same hour; raw supplies missing hours.
|
||||
func mergeNodeMetricHourlyPreferRollup(rollup, raw []NodeMetricHourly) []NodeMetricHourly {
|
||||
byHour := make(map[int64]NodeMetricHourly, len(raw)+len(rollup))
|
||||
order := make([]int64, 0, len(raw)+len(rollup))
|
||||
add := func(row NodeMetricHourly, overwrite bool) {
|
||||
key := row.Hour.UTC().Truncate(time.Hour).Unix()
|
||||
if _, exists := byHour[key]; !exists {
|
||||
order = append(order, key)
|
||||
byHour[key] = row
|
||||
return
|
||||
}
|
||||
if overwrite {
|
||||
byHour[key] = row
|
||||
}
|
||||
}
|
||||
for _, row := range raw {
|
||||
add(row, false)
|
||||
}
|
||||
for _, row := range rollup {
|
||||
add(row, true)
|
||||
}
|
||||
result := make([]NodeMetricHourly, 0, len(order))
|
||||
// Keep chronological order of first-seen keys; re-sort by hour for stability.
|
||||
slices.Sort(order)
|
||||
for _, key := range order {
|
||||
result = append(result, byHour[key])
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func listNodeMetricHourlyFromRollup(ctx context.Context, filter NodeObservabilityFilter) ([]NodeMetricHourly, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clause, args := buildNodeObservabilityFilterClause(filter, "hour")
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT
|
||||
hour,
|
||||
if(sum(cpu_usage_count) > 0, sum(cpu_usage_sum) / sum(cpu_usage_count), 0) AS average_cpu_usage_percent,
|
||||
if(sum(memory_usage_count) > 0, sum(memory_usage_sum) / sum(memory_usage_count), 0) AS average_memory_usage_percent,
|
||||
sum(greatest(network_rx_max - network_rx_min, 0)) AS network_rx_bytes,
|
||||
sum(greatest(network_tx_max - network_tx_min, 0)) AS network_tx_bytes,
|
||||
sum(greatest(disk_read_max - disk_read_min, 0)) AS disk_read_bytes,
|
||||
sum(greatest(disk_write_max - disk_write_min, 0)) AS disk_write_bytes,
|
||||
toUInt64(uniqExact(node_id)) AS reported_nodes
|
||||
FROM %s
|
||||
WHERE %s
|
||||
GROUP BY hour
|
||||
ORDER BY hour ASC`, nodeMetricCapacityHourlyTableName(), clause)
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list node metric hourly from rollup: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
return scanNodeMetricHourlyRows(rows)
|
||||
}
|
||||
|
||||
func listNodeMetricHourlyFromRaw(ctx context.Context, filter NodeObservabilityFilter) ([]NodeMetricHourly, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clause, args := buildNodeObservabilityFilterClause(filter, "captured_at")
|
||||
tableName := nodeMetricSnapshotTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT
|
||||
hour,
|
||||
avg(cpu_usage_percent) AS average_cpu_usage_percent,
|
||||
avg(memory_usage_percent) AS average_memory_usage_percent,
|
||||
sum(if(network_rx_delta >= 0, network_rx_delta, 0)) AS network_rx_bytes,
|
||||
sum(if(network_tx_delta >= 0, network_tx_delta, 0)) AS network_tx_bytes,
|
||||
sum(if(disk_read_delta >= 0, disk_read_delta, 0)) AS disk_read_bytes,
|
||||
sum(if(disk_write_delta >= 0, disk_write_delta, 0)) AS disk_write_bytes,
|
||||
toUInt64(uniqExact(node_id)) AS reported_nodes
|
||||
FROM (
|
||||
SELECT
|
||||
node_id,
|
||||
toStartOfHour(captured_at) AS hour,
|
||||
cpu_usage_percent,
|
||||
if(memory_total_bytes > 0, (memory_used_bytes * 100.0) / memory_total_bytes, 0) AS memory_usage_percent,
|
||||
network_rx_bytes - lagInFrame(network_rx_bytes, 1, network_rx_bytes) OVER (
|
||||
PARTITION BY node_id ORDER BY captured_at, id
|
||||
ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
|
||||
) AS network_rx_delta,
|
||||
network_tx_bytes - lagInFrame(network_tx_bytes, 1, network_tx_bytes) OVER (
|
||||
PARTITION BY node_id ORDER BY captured_at, id
|
||||
ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
|
||||
) AS network_tx_delta,
|
||||
disk_read_bytes - lagInFrame(disk_read_bytes, 1, disk_read_bytes) OVER (
|
||||
PARTITION BY node_id ORDER BY captured_at, id
|
||||
ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
|
||||
) AS disk_read_delta,
|
||||
disk_write_bytes - lagInFrame(disk_write_bytes, 1, disk_write_bytes) OVER (
|
||||
PARTITION BY node_id ORDER BY captured_at, id
|
||||
ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
|
||||
) AS disk_write_delta
|
||||
FROM %s
|
||||
WHERE %s
|
||||
)
|
||||
GROUP BY hour
|
||||
ORDER BY hour ASC`, tableName, clause)
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list node metric hourly: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
return scanNodeMetricHourlyRows(rows)
|
||||
}
|
||||
|
||||
func scanNodeMetricHourlyRows(rows driver.Rows) ([]NodeMetricHourly, error) {
|
||||
result := make([]NodeMetricHourly, 0)
|
||||
for rows.Next() {
|
||||
var (
|
||||
item NodeMetricHourly
|
||||
reportedNodes uint64
|
||||
networkRx int64
|
||||
networkTx int64
|
||||
diskRead int64
|
||||
diskWrite int64
|
||||
)
|
||||
if err := rows.Scan(
|
||||
&item.Hour,
|
||||
&item.AverageCPUUsagePercent,
|
||||
&item.AverageMemoryUsagePercent,
|
||||
&networkRx,
|
||||
&networkTx,
|
||||
&diskRead,
|
||||
&diskWrite,
|
||||
&reportedNodes,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan node metric hourly row: %w", err)
|
||||
}
|
||||
item.Hour = item.Hour.UTC()
|
||||
item.NetworkRxBytes = networkRx
|
||||
item.NetworkTxBytes = networkTx
|
||||
item.DiskReadBytes = diskRead
|
||||
item.DiskWriteBytes = diskWrite
|
||||
item.ReportedNodes = int(safeInt64Count(reportedNodes))
|
||||
result = append(result, item)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func scanNodeObsFrpcRows(rows driver.Rows) ([]analyticsmodel.NodeObsFrpc, error) {
|
||||
var result []analyticsmodel.NodeObsFrpc
|
||||
for rows.Next() {
|
||||
var item analyticsmodel.NodeObsFrpc
|
||||
if err := rows.Scan(
|
||||
&item.ID,
|
||||
&item.NodeID,
|
||||
&item.CapturedAt,
|
||||
&item.TunnelStatus,
|
||||
&item.ConnectedRelaysCount,
|
||||
&item.CreatedAt,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan node frpc observation row: %w", err)
|
||||
}
|
||||
item.CapturedAt = item.CapturedAt.UTC()
|
||||
item.CreatedAt = item.CreatedAt.UTC()
|
||||
result = append(result, item)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
+177
@@ -0,0 +1,177 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// DeleteAllNodeMetricSnapshots hard-deletes all node metric snapshots via TRUNCATE.
|
||||
func DeleteAllNodeMetricSnapshots(ctx context.Context) (int64, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
outcome, err := truncateClickHouseTable(ctx, conn, nodeMetricSnapshotTableName())
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return outcome.DeletedCount, nil
|
||||
}
|
||||
|
||||
// DeleteNodeMetricSnapshotsBefore force-materializes of_node_metric_snapshots table TTL.
|
||||
// cutoff is ignored; see MaterializeNodeMetricSnapshotsTTL.
|
||||
func DeleteNodeMetricSnapshotsBefore(ctx context.Context, _ time.Time) (int64, error) {
|
||||
outcome, err := MaterializeNodeMetricSnapshotsTTL(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return outcome.EligibleCount, nil
|
||||
}
|
||||
|
||||
// MaterializeNodeMetricSnapshotsTTL force-materializes table TTL and reports an honest outcome.
|
||||
func MaterializeNodeMetricSnapshotsTTL(ctx context.Context) (CleanupOutcome, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return CleanupOutcome{}, err
|
||||
}
|
||||
tableName := nodeMetricSnapshotTableName()
|
||||
ttlDays := TableTTLDaysNodeMetricSnapshots
|
||||
cutoff := tableTTLCutoff(ttlDays, time.Now())
|
||||
return materializeExpiredByTableTTL(
|
||||
ctx,
|
||||
conn,
|
||||
tableName,
|
||||
ttlDays,
|
||||
fmt.Sprintf("SELECT count() FROM %s WHERE captured_at < ?", tableName),
|
||||
[]any{cutoff},
|
||||
)
|
||||
}
|
||||
|
||||
// DeleteAllNodeEdgeHealth truncates of_node_edge_health.
|
||||
func DeleteAllNodeEdgeHealth(ctx context.Context) (int64, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
outcome, err := truncateClickHouseTable(ctx, conn, nodeEdgeHealthTableName())
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return outcome.DeletedCount, nil
|
||||
}
|
||||
|
||||
// DeleteNodeEdgeHealthBefore force-materializes of_node_edge_health TTL.
|
||||
func DeleteNodeEdgeHealthBefore(ctx context.Context, _ time.Time) (int64, error) {
|
||||
outcome, err := MaterializeNodeEdgeHealthTTL(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return outcome.EligibleCount, nil
|
||||
}
|
||||
|
||||
// MaterializeNodeEdgeHealthTTL force-materializes of_node_edge_health table TTL.
|
||||
func MaterializeNodeEdgeHealthTTL(ctx context.Context) (CleanupOutcome, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return CleanupOutcome{}, err
|
||||
}
|
||||
tableName := nodeEdgeHealthTableName()
|
||||
ttlDays := TableTTLDaysNodeObs
|
||||
cutoff := tableTTLCutoff(ttlDays, time.Now())
|
||||
return materializeExpiredByTableTTL(
|
||||
ctx,
|
||||
conn,
|
||||
tableName,
|
||||
ttlDays,
|
||||
fmt.Sprintf("SELECT count() FROM %s WHERE captured_at < ?", tableName),
|
||||
[]any{cutoff},
|
||||
)
|
||||
}
|
||||
|
||||
// DeleteAllNodeObsFrps hard-deletes all FRPS observations via TRUNCATE.
|
||||
func DeleteAllNodeObsFrps(ctx context.Context) (int64, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
outcome, err := truncateClickHouseTable(ctx, conn, nodeObsFrpsTableName())
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return outcome.DeletedCount, nil
|
||||
}
|
||||
|
||||
// DeleteNodeObsFrpsBefore force-materializes of_node_obs_frps table TTL.
|
||||
// cutoff is ignored; see MaterializeNodeObsFrpsTTL.
|
||||
func DeleteNodeObsFrpsBefore(ctx context.Context, _ time.Time) (int64, error) {
|
||||
outcome, err := MaterializeNodeObsFrpsTTL(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return outcome.EligibleCount, nil
|
||||
}
|
||||
|
||||
// MaterializeNodeObsFrpsTTL force-materializes table TTL and reports an honest outcome.
|
||||
func MaterializeNodeObsFrpsTTL(ctx context.Context) (CleanupOutcome, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return CleanupOutcome{}, err
|
||||
}
|
||||
tableName := nodeObsFrpsTableName()
|
||||
ttlDays := TableTTLDaysNodeObs
|
||||
cutoff := tableTTLCutoff(ttlDays, time.Now())
|
||||
return materializeExpiredByTableTTL(
|
||||
ctx,
|
||||
conn,
|
||||
tableName,
|
||||
ttlDays,
|
||||
fmt.Sprintf("SELECT count() FROM %s WHERE captured_at < ?", tableName),
|
||||
[]any{cutoff},
|
||||
)
|
||||
}
|
||||
|
||||
// DeleteAllNodeObsFrpc hard-deletes all FRPC observations via TRUNCATE.
|
||||
func DeleteAllNodeObsFrpc(ctx context.Context) (int64, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
outcome, err := truncateClickHouseTable(ctx, conn, nodeObsFrpcTableName())
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return outcome.DeletedCount, nil
|
||||
}
|
||||
|
||||
// DeleteNodeObsFrpcBefore force-materializes of_node_obs_frpc table TTL.
|
||||
// cutoff is ignored; see MaterializeNodeObsFrpcTTL.
|
||||
func DeleteNodeObsFrpcBefore(ctx context.Context, _ time.Time) (int64, error) {
|
||||
outcome, err := MaterializeNodeObsFrpcTTL(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return outcome.EligibleCount, nil
|
||||
}
|
||||
|
||||
// MaterializeNodeObsFrpcTTL force-materializes table TTL and reports an honest outcome.
|
||||
func MaterializeNodeObsFrpcTTL(ctx context.Context) (CleanupOutcome, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return CleanupOutcome{}, err
|
||||
}
|
||||
tableName := nodeObsFrpcTableName()
|
||||
ttlDays := TableTTLDaysNodeObs
|
||||
cutoff := tableTTLCutoff(ttlDays, time.Now())
|
||||
return materializeExpiredByTableTTL(
|
||||
ctx,
|
||||
conn,
|
||||
tableName,
|
||||
ttlDays,
|
||||
fmt.Sprintf("SELECT count() FROM %s WHERE captured_at < ?", tableName),
|
||||
[]any{cutoff},
|
||||
)
|
||||
}
|
||||
+63
@@ -0,0 +1,63 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
|
||||
)
|
||||
|
||||
const nodeObservabilityFilterClauseCapacity = 3
|
||||
|
||||
// NodeObservabilityFilter scopes ClickHouse node observability queries.
|
||||
type NodeObservabilityFilter = analyticsmodel.NodeObservabilityFilter
|
||||
|
||||
func buildNodeObservabilityFilterClause(filter NodeObservabilityFilter, sinceColumn string) (string, []any) {
|
||||
parts := make([]string, 0, nodeObservabilityFilterClauseCapacity)
|
||||
args := make([]any, 0, nodeObservabilityFilterClauseCapacity)
|
||||
if trimmed := strings.TrimSpace(filter.NodeID); trimmed != "" {
|
||||
parts = append(parts, "node_id = ?")
|
||||
args = append(args, trimmed)
|
||||
}
|
||||
if !filter.Since.IsZero() {
|
||||
parts = append(parts, sinceColumn+" >= ?")
|
||||
args = append(args, filter.Since.UTC())
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return "1", nil
|
||||
}
|
||||
return strings.Join(parts, " AND "), args
|
||||
}
|
||||
|
||||
func nodeObservabilityCapturedAtOrderClause() string {
|
||||
return "captured_at DESC, id DESC"
|
||||
}
|
||||
|
||||
func nodeMetricSnapshotTableName() string {
|
||||
return "of_node_metric_snapshots"
|
||||
}
|
||||
|
||||
func nodeEdgeHealthTableName() string {
|
||||
return "of_node_edge_health"
|
||||
}
|
||||
|
||||
func accessLogHourlyTableName() string {
|
||||
return "of_access_log_hourly"
|
||||
}
|
||||
|
||||
func nodeObsFrpsTableName() string {
|
||||
return "of_node_obs_frps"
|
||||
}
|
||||
|
||||
func nodeObsFrpcTableName() string {
|
||||
return "of_node_obs_frpc"
|
||||
}
|
||||
|
||||
func nodeMetricCapacityHourlyTableName() string {
|
||||
return "of_node_metric_capacity_hourly"
|
||||
}
|
||||
|
||||
// clickHouseLimit1ByNodeIDClause selects the first row per node_id after ORDER BY.
|
||||
const clickHouseLimit1ByNodeIDClause = " LIMIT 1 BY node_id"
|
||||
+156
@@ -0,0 +1,156 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
db "Wavelet/plugins/infra/database"
|
||||
|
||||
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestListLatestNodeMetricSnapshots_UsesLimit1ByNodeID(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
mock := &mockConn{}
|
||||
db.SetChConnForTest(mock)
|
||||
t.Cleanup(func() { db.SetChConnForTest(nil) })
|
||||
|
||||
since := time.Date(2026, 7, 10, 0, 0, 0, 0, time.UTC)
|
||||
_, err := ListLatestNodeMetricSnapshots(ctx, NodeObservabilityFilter{Since: since})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, mock.queries, 1)
|
||||
assert.Contains(t, mock.queries[0], "LIMIT 1 BY node_id")
|
||||
assert.Contains(t, mock.queries[0], nodeMetricSnapshotTableName())
|
||||
assert.Contains(t, mock.queries[0], "captured_at DESC")
|
||||
assert.NotContains(t, mock.queries[0], "LIMIT ?")
|
||||
require.Len(t, mock.queryArgs, 1)
|
||||
require.Len(t, mock.queryArgs[0], 1)
|
||||
assert.Equal(t, since, mock.queryArgs[0][0])
|
||||
}
|
||||
|
||||
func TestListNodeMetricHourly_PrefersRollup(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
hour := time.Date(2026, 7, 10, 12, 0, 0, 0, time.UTC)
|
||||
since := hour.Add(-1 * time.Hour)
|
||||
mock := &mockConn{
|
||||
queryFn: func(_ context.Context, query string, _ ...any) (driver.Rows, error) {
|
||||
if strings.Contains(query, nodeMetricCapacityHourlyTableName()) {
|
||||
return &mockRows{data: [][]any{{
|
||||
hour, 42.5, 60.0, int64(100), int64(200), int64(10), int64(20), uint64(2),
|
||||
}}}, nil
|
||||
}
|
||||
return nil, errors.New("raw path should not be used when rollup covers the window")
|
||||
},
|
||||
}
|
||||
db.SetChConnForTest(mock)
|
||||
t.Cleanup(func() { db.SetChConnForTest(nil) })
|
||||
|
||||
rows, err := ListNodeMetricHourly(ctx, NodeObservabilityFilter{Since: since})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, rows, 1)
|
||||
assert.InDelta(t, 42.5, rows[0].AverageCPUUsagePercent, 1e-9)
|
||||
assert.InDelta(t, 60.0, rows[0].AverageMemoryUsagePercent, 1e-9)
|
||||
assert.Equal(t, int64(100), rows[0].NetworkRxBytes)
|
||||
assert.Equal(t, 2, rows[0].ReportedNodes)
|
||||
require.Len(t, mock.queries, 1)
|
||||
assert.Contains(t, mock.queries[0], nodeMetricCapacityHourlyTableName())
|
||||
}
|
||||
|
||||
func TestListNodeMetricHourly_MergesRawGapsWithPartialRollup(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
// 24h window starts far before the only rollup bucket (last hour).
|
||||
since := time.Date(2026, 7, 9, 12, 0, 0, 0, time.UTC)
|
||||
rollupHour := time.Date(2026, 7, 10, 12, 0, 0, 0, time.UTC)
|
||||
rawHour := time.Date(2026, 7, 9, 15, 0, 0, 0, time.UTC)
|
||||
mock := &mockConn{
|
||||
queryFn: func(_ context.Context, query string, _ ...any) (driver.Rows, error) {
|
||||
if strings.Contains(query, nodeMetricCapacityHourlyTableName()) {
|
||||
return &mockRows{data: [][]any{{
|
||||
rollupHour, 99.0, 99.0, int64(1), int64(1), int64(1), int64(1), uint64(1),
|
||||
}}}, nil
|
||||
}
|
||||
if strings.Contains(query, nodeMetricSnapshotTableName()) {
|
||||
return &mockRows{data: [][]any{
|
||||
{rawHour, 12.0, 34.0, int64(5), int64(6), int64(7), int64(8), uint64(1)},
|
||||
{rollupHour, 50.0, 50.0, int64(9), int64(9), int64(9), int64(9), uint64(1)},
|
||||
}}, nil
|
||||
}
|
||||
return &mockRows{}, nil
|
||||
},
|
||||
}
|
||||
db.SetChConnForTest(mock)
|
||||
t.Cleanup(func() { db.SetChConnForTest(nil) })
|
||||
|
||||
rows, err := ListNodeMetricHourly(ctx, NodeObservabilityFilter{Since: since})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, rows, 2)
|
||||
assert.Equal(t, rawHour, rows[0].Hour)
|
||||
assert.InDelta(t, 12.0, rows[0].AverageCPUUsagePercent, 1e-9)
|
||||
// Overlapping hour prefers rollup (99) over raw (50).
|
||||
assert.Equal(t, rollupHour, rows[1].Hour)
|
||||
assert.InDelta(t, 99.0, rows[1].AverageCPUUsagePercent, 1e-9)
|
||||
require.GreaterOrEqual(t, len(mock.queries), 2)
|
||||
assert.Contains(t, mock.queries[1], "lagInFrame")
|
||||
}
|
||||
|
||||
func TestMergeNodeMetricHourlyPreferRollup(t *testing.T) {
|
||||
h1 := time.Date(2026, 7, 10, 10, 0, 0, 0, time.UTC)
|
||||
h2 := time.Date(2026, 7, 10, 11, 0, 0, 0, time.UTC)
|
||||
merged := mergeNodeMetricHourlyPreferRollup(
|
||||
[]NodeMetricHourly{{Hour: h2, AverageCPUUsagePercent: 80}},
|
||||
[]NodeMetricHourly{
|
||||
{Hour: h1, AverageCPUUsagePercent: 10},
|
||||
{Hour: h2, AverageCPUUsagePercent: 20},
|
||||
},
|
||||
)
|
||||
require.Len(t, merged, 2)
|
||||
assert.Equal(t, h1, merged[0].Hour)
|
||||
assert.InDelta(t, 10.0, merged[0].AverageCPUUsagePercent, 1e-9)
|
||||
assert.Equal(t, h2, merged[1].Hour)
|
||||
assert.InDelta(t, 80.0, merged[1].AverageCPUUsagePercent, 1e-9)
|
||||
}
|
||||
|
||||
func TestHourlyRollupCoversWindow(t *testing.T) {
|
||||
since := time.Date(2026, 7, 10, 0, 0, 0, 0, time.UTC)
|
||||
assert.True(t, hourlyRollupCoversWindow(since, since))
|
||||
assert.True(t, hourlyRollupCoversWindow(since.Add(2*time.Hour), since))
|
||||
assert.False(t, hourlyRollupCoversWindow(since.Add(3*time.Hour), since))
|
||||
assert.True(t, hourlyRollupCoversWindow(time.Date(2026, 7, 11, 0, 0, 0, 0, time.UTC), time.Time{}))
|
||||
}
|
||||
|
||||
func TestListNodeMetricHourly_FallsBackToRawOnRollupError(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
hour := time.Date(2026, 7, 10, 13, 0, 0, 0, time.UTC)
|
||||
mock := &mockConn{
|
||||
queryFn: func(_ context.Context, query string, _ ...any) (driver.Rows, error) {
|
||||
if strings.Contains(query, nodeMetricCapacityHourlyTableName()) {
|
||||
return nil, errors.New("rollup missing")
|
||||
}
|
||||
if strings.Contains(query, nodeMetricSnapshotTableName()) {
|
||||
return &mockRows{data: [][]any{{
|
||||
hour, 10.0, 20.0, int64(1), int64(2), int64(3), int64(4), uint64(1),
|
||||
}}}, nil
|
||||
}
|
||||
return &mockRows{}, nil
|
||||
},
|
||||
}
|
||||
db.SetChConnForTest(mock)
|
||||
t.Cleanup(func() { db.SetChConnForTest(nil) })
|
||||
|
||||
rows, err := ListNodeMetricHourly(ctx, NodeObservabilityFilter{})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, rows, 1)
|
||||
assert.InDelta(t, 10.0, rows[0].AverageCPUUsagePercent, 1e-9)
|
||||
assert.Equal(t, int64(3), rows[0].DiskReadBytes)
|
||||
require.GreaterOrEqual(t, len(mock.queries), 2)
|
||||
assert.Contains(t, mock.queries[0], nodeMetricCapacityHourlyTableName())
|
||||
assert.Contains(t, mock.queries[1], "lagInFrame")
|
||||
}
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
|
||||
db "Wavelet/plugins/infra/database"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestInsertNodeEdgeHealth_EmptyNodeID(t *testing.T) {
|
||||
err := InsertNodeEdgeHealth(context.Background(), analyticsmodel.NodeEdgeHealth{})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestInsertNodeEdgeHealth_UsesEdgeHealthBatchSQL(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
mockBatch := &mockBatch{}
|
||||
mockConn := &mockConn{
|
||||
batch: mockBatch,
|
||||
batchQuery: analyticsmodel.NodeEdgeHealth{}.BatchInsertSQL(),
|
||||
}
|
||||
db.SetChConnForTest(mockConn)
|
||||
t.Cleanup(func() { db.SetChConnForTest(nil) })
|
||||
|
||||
capturedAt := time.Now().UTC()
|
||||
err := InsertNodeEdgeHealth(ctx, analyticsmodel.NodeEdgeHealth{
|
||||
NodeID: "node-a",
|
||||
CapturedAt: capturedAt,
|
||||
Status: "",
|
||||
Connections: 3,
|
||||
CreatedAt: capturedAt,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.True(t, mockConn.prepareCalled)
|
||||
assert.Equal(t, analyticsmodel.NodeEdgeHealth{}.BatchInsertSQL(), mockConn.preparedQuery)
|
||||
assert.True(t, mockBatch.sendCalled)
|
||||
require.Len(t, mockBatch.rows, 1)
|
||||
assert.Equal(t, "node-a", mockBatch.rows[0][1])
|
||||
assert.Equal(t, "unknown", mockBatch.rows[0][3]) // status default
|
||||
assert.Equal(t, int64(3), mockBatch.rows[0][4]) // connections
|
||||
}
|
||||
+272
@@ -0,0 +1,272 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
|
||||
"Wavelet/pkg/idgen"
|
||||
db "Wavelet/plugins/infra/database"
|
||||
)
|
||||
|
||||
const edgeHealthStatusUnknown = "unknown"
|
||||
|
||||
// InsertNodeMetricSnapshot writes a single metric snapshot via the batch API.
|
||||
func InsertNodeMetricSnapshot(ctx context.Context, snapshot analyticsmodel.NodeMetricSnapshot) error {
|
||||
if strings.TrimSpace(snapshot.NodeID) == "" {
|
||||
return nil
|
||||
}
|
||||
return BatchInsertNodeMetricSnapshots(ctx, []analyticsmodel.NodeMetricSnapshot{snapshot})
|
||||
}
|
||||
|
||||
// BatchInsertNodeMetricSnapshots writes metric snapshots to ClickHouse.
|
||||
func BatchInsertNodeMetricSnapshots(ctx context.Context, snapshots []analyticsmodel.NodeMetricSnapshot) error {
|
||||
if len(snapshots) == 0 {
|
||||
return nil
|
||||
}
|
||||
if db.ChConn == nil {
|
||||
return errors.New("clickhouse connection is not initialized")
|
||||
}
|
||||
|
||||
batch, err := db.ChConn.PrepareBatch(ctx, analyticsmodel.NodeMetricSnapshot{}.BatchInsertSQL())
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare clickhouse batch: %w", err)
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
for _, snapshot := range snapshots {
|
||||
nodeID := strings.TrimSpace(snapshot.NodeID)
|
||||
if nodeID == "" {
|
||||
continue
|
||||
}
|
||||
id := snapshot.ID
|
||||
if id == 0 {
|
||||
id = idgen.NextUint64ID()
|
||||
}
|
||||
createdAt := snapshot.CreatedAt
|
||||
if createdAt.IsZero() {
|
||||
createdAt = now
|
||||
}
|
||||
if err := batch.Append(
|
||||
id,
|
||||
nodeID,
|
||||
snapshot.CapturedAt.UTC(),
|
||||
snapshot.CPUUsagePercent,
|
||||
snapshot.MemoryUsedBytes,
|
||||
snapshot.MemoryTotalBytes,
|
||||
snapshot.StorageUsedBytes,
|
||||
snapshot.StorageTotalBytes,
|
||||
snapshot.DiskReadBytes,
|
||||
snapshot.DiskWriteBytes,
|
||||
snapshot.NetworkRxBytes,
|
||||
snapshot.NetworkTxBytes,
|
||||
createdAt.UTC(),
|
||||
); err != nil {
|
||||
return fmt.Errorf("append node metric snapshot to batch: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if batch.Rows() == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := batch.Send(); err != nil {
|
||||
return fmt.Errorf("send clickhouse batch: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func normalizeEdgeHealthStatus(status string) string {
|
||||
status = strings.TrimSpace(status)
|
||||
if status == "" {
|
||||
return edgeHealthStatusUnknown
|
||||
}
|
||||
return status
|
||||
}
|
||||
|
||||
// InsertNodeEdgeHealth writes a single edge health snapshot.
|
||||
func InsertNodeEdgeHealth(ctx context.Context, row analyticsmodel.NodeEdgeHealth) error {
|
||||
if strings.TrimSpace(row.NodeID) == "" {
|
||||
return nil
|
||||
}
|
||||
return BatchInsertNodeEdgeHealth(ctx, []analyticsmodel.NodeEdgeHealth{row})
|
||||
}
|
||||
|
||||
// BatchInsertNodeEdgeHealth writes L2 OpenResty health snapshots to ClickHouse.
|
||||
func BatchInsertNodeEdgeHealth(ctx context.Context, rows []analyticsmodel.NodeEdgeHealth) error {
|
||||
if len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
if db.ChConn == nil {
|
||||
return errors.New("clickhouse connection is not initialized")
|
||||
}
|
||||
batch, err := db.ChConn.PrepareBatch(ctx, analyticsmodel.NodeEdgeHealth{}.BatchInsertSQL())
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare clickhouse batch: %w", err)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
for _, row := range rows {
|
||||
nodeID := strings.TrimSpace(row.NodeID)
|
||||
if nodeID == "" {
|
||||
continue
|
||||
}
|
||||
id := row.ID
|
||||
if id == 0 {
|
||||
id = idgen.NextUint64ID()
|
||||
}
|
||||
createdAt := row.CreatedAt
|
||||
if createdAt.IsZero() {
|
||||
createdAt = now
|
||||
}
|
||||
capturedAt := row.CapturedAt.UTC()
|
||||
if capturedAt.IsZero() {
|
||||
capturedAt = now
|
||||
}
|
||||
if err := batch.Append(
|
||||
id,
|
||||
nodeID,
|
||||
capturedAt,
|
||||
normalizeEdgeHealthStatus(row.Status),
|
||||
row.Connections,
|
||||
createdAt.UTC(),
|
||||
); err != nil {
|
||||
return fmt.Errorf("append node edge health to batch: %w", err)
|
||||
}
|
||||
}
|
||||
if batch.Rows() == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := batch.Send(); err != nil {
|
||||
return fmt.Errorf("send clickhouse batch: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// InsertNodeObsFrps writes a single FRPS observation via the batch API.
|
||||
func InsertNodeObsFrps(ctx context.Context, obs analyticsmodel.NodeObsFrps) error {
|
||||
if strings.TrimSpace(obs.NodeID) == "" {
|
||||
return nil
|
||||
}
|
||||
return BatchInsertNodeObsFrps(ctx, []analyticsmodel.NodeObsFrps{obs})
|
||||
}
|
||||
|
||||
// BatchInsertNodeObsFrps writes FRPS observations to ClickHouse.
|
||||
func BatchInsertNodeObsFrps(ctx context.Context, observations []analyticsmodel.NodeObsFrps) error {
|
||||
if len(observations) == 0 {
|
||||
return nil
|
||||
}
|
||||
if db.ChConn == nil {
|
||||
return errors.New("clickhouse connection is not initialized")
|
||||
}
|
||||
|
||||
batch, err := db.ChConn.PrepareBatch(ctx, analyticsmodel.NodeObsFrps{}.BatchInsertSQL())
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare clickhouse batch: %w", err)
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
for _, obs := range observations {
|
||||
nodeID := strings.TrimSpace(obs.NodeID)
|
||||
if nodeID == "" {
|
||||
continue
|
||||
}
|
||||
id := obs.ID
|
||||
if id == 0 {
|
||||
id = idgen.NextUint64ID()
|
||||
}
|
||||
createdAt := obs.CreatedAt
|
||||
if createdAt.IsZero() {
|
||||
createdAt = now
|
||||
}
|
||||
capturedAt := obs.CapturedAt.UTC()
|
||||
if capturedAt.IsZero() {
|
||||
capturedAt = now
|
||||
}
|
||||
if err := batch.Append(
|
||||
id,
|
||||
nodeID,
|
||||
capturedAt,
|
||||
obs.FrpsConnections,
|
||||
obs.FrpsProxyCount,
|
||||
obs.FrpsClientCount,
|
||||
obs.FrpsProxies,
|
||||
createdAt.UTC(),
|
||||
); err != nil {
|
||||
return fmt.Errorf("append node frps observation to batch: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if batch.Rows() == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := batch.Send(); err != nil {
|
||||
return fmt.Errorf("send clickhouse batch: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// InsertNodeObsFrpc writes a single FRPC observation via the batch API.
|
||||
func InsertNodeObsFrpc(ctx context.Context, obs analyticsmodel.NodeObsFrpc) error {
|
||||
if strings.TrimSpace(obs.NodeID) == "" {
|
||||
return nil
|
||||
}
|
||||
return BatchInsertNodeObsFrpc(ctx, []analyticsmodel.NodeObsFrpc{obs})
|
||||
}
|
||||
|
||||
// BatchInsertNodeObsFrpc writes FRPC observations to ClickHouse.
|
||||
func BatchInsertNodeObsFrpc(ctx context.Context, observations []analyticsmodel.NodeObsFrpc) error {
|
||||
if len(observations) == 0 {
|
||||
return nil
|
||||
}
|
||||
if db.ChConn == nil {
|
||||
return errors.New("clickhouse connection is not initialized")
|
||||
}
|
||||
|
||||
batch, err := db.ChConn.PrepareBatch(ctx, analyticsmodel.NodeObsFrpc{}.BatchInsertSQL())
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare clickhouse batch: %w", err)
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
for _, obs := range observations {
|
||||
nodeID := strings.TrimSpace(obs.NodeID)
|
||||
if nodeID == "" {
|
||||
continue
|
||||
}
|
||||
id := obs.ID
|
||||
if id == 0 {
|
||||
id = idgen.NextUint64ID()
|
||||
}
|
||||
createdAt := obs.CreatedAt
|
||||
if createdAt.IsZero() {
|
||||
createdAt = now
|
||||
}
|
||||
capturedAt := obs.CapturedAt.UTC()
|
||||
if capturedAt.IsZero() {
|
||||
capturedAt = now
|
||||
}
|
||||
if err := batch.Append(
|
||||
id,
|
||||
nodeID,
|
||||
capturedAt,
|
||||
obs.TunnelStatus,
|
||||
obs.ConnectedRelaysCount,
|
||||
createdAt.UTC(),
|
||||
); err != nil {
|
||||
return fmt.Errorf("append node frpc observation to batch: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if batch.Rows() == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := batch.Send(); err != nil {
|
||||
return fmt.Errorf("send clickhouse batch: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
const (
|
||||
clickHouseLimitClause = " LIMIT ?"
|
||||
clickHouseLimitOffsetClause = " LIMIT ? OFFSET ?"
|
||||
)
|
||||
@@ -0,0 +1,80 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
|
||||
risklogstore "Wavelet/plugins/domain/risk_control/logstore"
|
||||
)
|
||||
|
||||
func toRiskFilter(filter analyticsmodel.AccessLogFilter) risklogstore.AccessLogFilter {
|
||||
return risklogstore.AccessLogFilter{
|
||||
UserIDs: filter.UserIDs,
|
||||
Path: filter.Path,
|
||||
StartTime: filter.StartTime,
|
||||
EndTime: filter.EndTime,
|
||||
}
|
||||
}
|
||||
|
||||
// BatchInsert writes user access logs via Wavelet risk_control.
|
||||
func BatchInsert(ctx context.Context, logs []analyticsmodel.UserAccessLog) error {
|
||||
return risklogstore.BatchInsert(ctx, logs)
|
||||
}
|
||||
|
||||
// DeleteAllUserAccessLogs truncates user access logs via Wavelet risk_control.
|
||||
func DeleteAllUserAccessLogs(ctx context.Context) (int64, error) {
|
||||
return risklogstore.DeleteAllUserAccessLogs(ctx)
|
||||
}
|
||||
|
||||
// CountAccessLogs counts user access logs via Wavelet risk_control.
|
||||
func CountAccessLogs(ctx context.Context, filter analyticsmodel.AccessLogFilter) (uint64, error) {
|
||||
return risklogstore.CountAccessLogs(ctx, toRiskFilter(filter))
|
||||
}
|
||||
|
||||
// ListAccessLogs lists user access logs via Wavelet risk_control.
|
||||
func ListAccessLogs(ctx context.Context, filter analyticsmodel.AccessLogFilter, page, pageSize int) ([]analyticsmodel.UserAccessLog, uint64, error) {
|
||||
return risklogstore.ListAccessLogs(ctx, toRiskFilter(filter), page, pageSize)
|
||||
}
|
||||
|
||||
// GetDailyTrend returns the daily trend via Wavelet risk_control.
|
||||
func GetDailyTrend(ctx context.Context, days int) ([]analyticsmodel.DailyTrend, error) {
|
||||
src, err := risklogstore.GetDailyTrend(ctx, days)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]analyticsmodel.DailyTrend, len(src))
|
||||
for i, v := range src {
|
||||
out[i] = analyticsmodel.DailyTrend{Date: v.Date, Count: v.Count}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// GetBrowserDistribution returns browser share via Wavelet risk_control.
|
||||
func GetBrowserDistribution(ctx context.Context, startTime time.Time) ([]analyticsmodel.BrowserShare, error) {
|
||||
src, err := risklogstore.GetBrowserDistribution(ctx, startTime)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]analyticsmodel.BrowserShare, len(src))
|
||||
for i, v := range src {
|
||||
out[i] = analyticsmodel.BrowserShare{Browser: v.Browser, Count: v.Count}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// GetTopActiveUsers returns top users via Wavelet risk_control.
|
||||
func GetTopActiveUsers(ctx context.Context, startTime time.Time, limit int) ([]analyticsmodel.TopUser, error) {
|
||||
src, err := risklogstore.GetTopActiveUsers(ctx, startTime, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]analyticsmodel.TopUser, len(src))
|
||||
for i, v := range src {
|
||||
out[i] = analyticsmodel.TopUser{UserID: v.UserID, Count: v.Count}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package repository
|
||||
|
||||
// Persistence and repository-layer parameter messages live here (unexported).
|
||||
// Domain field validation used by model.Validate stays in internal/model/errs.go;
|
||||
// repository may call model.Validate and return those errors as-is.
|
||||
// Keep wording aligned with model where the same user-facing phrase applies,
|
||||
// but do not import or re-export model unexported consts (would require exporting).
|
||||
const (
|
||||
errDatabaseNotInitialized = "database not initialized"
|
||||
errConfigIntParseFailed = "配置 %s 的值 '%s' 无法转换为整数: %w"
|
||||
errConfigDecimalParseFailed = "配置 %s 的值 '%s' 无法转换为decimal: %w"
|
||||
errConfigBoolParseFailed = "配置 %s 的值 '%s' 无法转换为布尔值: %w"
|
||||
errParseMenuDisplayConfigFailed = "解析目录显示配置失败: %w"
|
||||
errAuthSourceNameRequired = "认证源名称不能为空"
|
||||
errAuthSourceIDRequired = "认证源 ID 不能为空"
|
||||
errExternalAccountBindingIncomplete = "外部账号绑定信息不完整"
|
||||
errExternalAccountAlreadyBoundToAnother = "该外部账号已绑定到其他用户"
|
||||
errUserIDRequired = "用户 ID 不能为空"
|
||||
errExternalAccountBindingIDRequired = "绑定记录 ID 不能为空"
|
||||
)
|
||||
|
||||
const colName = "name"
|
||||
|
||||
const colEnabled = "enabled"
|
||||
@@ -0,0 +1,207 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package logstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/model"
|
||||
"Wavelet/pkg/logger"
|
||||
)
|
||||
|
||||
// CleanupSummary 汇总本次清理结果。
|
||||
type CleanupSummary struct {
|
||||
ActiveDatabase string `json:"active_database"`
|
||||
// RetentionDays 访问日志(节点访问/用户访问)保留天数,按日志库读取。
|
||||
RetentionDays int `json:"retention_days"`
|
||||
// MetricRetentionDays 性能指标(CPU/内存/磁盘/网络)保留天数,三库共用短留存。
|
||||
MetricRetentionDays int `json:"metric_retention_days"`
|
||||
Deleted int64 `json:"deleted"`
|
||||
// Tables 记录本次清理的物理表简写名(去掉 of_ 前缀,如 node_access_logs 对应
|
||||
// of_node_access_logs;CH 侧物理表名相同,简写仅便于状态展示)。
|
||||
Tables []string `json:"tables"`
|
||||
}
|
||||
|
||||
// defaultLogRetentionDays 默认日志保留天数(配置缺失/非法时回退)。
|
||||
const defaultLogRetentionDays = 90
|
||||
|
||||
// defaultMetricRetentionDays 默认性能指标保留天数(配置缺失/非法时回退)。
|
||||
// 性能数据价值衰减快,默认短留存(3 天)。
|
||||
const defaultMetricRetentionDays = 3
|
||||
|
||||
// partitionLeadMonths 清理时确保「当前月 + 未来 2 个月」分区持续存在。
|
||||
const partitionLeadMonths = 2
|
||||
|
||||
// accessLogPartitionTables 按月分区的访问日志表(分区预建/空分区清理共用)。
|
||||
var accessLogPartitionTables = []string{"of_node_access_logs", "w_user_access_logs"}
|
||||
|
||||
// retentionDaysForDatabase 按给定日志库读取保留天数(默认 90)。
|
||||
func retentionDaysForDatabase(ctx context.Context, dbName string) int {
|
||||
key := model.ConfigKeyLogRetentionDaysPostgres
|
||||
switch dbName {
|
||||
case dbNameSQLite:
|
||||
key = model.ConfigKeyLogRetentionDaysSQLite
|
||||
case dbNameClickHouse:
|
||||
key = model.ConfigKeyLogRetentionDaysClickHouse
|
||||
}
|
||||
v, err := getConfig(ctx, key)
|
||||
if err != nil {
|
||||
if !errors.Is(err, errConfigReaderNotWired) {
|
||||
logger.ErrorF(ctx, "读取日志保留天数配置失败(key=%s),回退默认 %d 天: %v", key, defaultLogRetentionDays, err)
|
||||
}
|
||||
return defaultLogRetentionDays
|
||||
}
|
||||
days, perr := strconv.Atoi(v)
|
||||
if perr != nil || days <= 0 {
|
||||
logger.ErrorF(ctx, "日志保留天数配置非法(key=%s, value=%q),回退默认 %d 天", key, v, defaultLogRetentionDays)
|
||||
return defaultLogRetentionDays
|
||||
}
|
||||
return days
|
||||
}
|
||||
|
||||
// metricRetentionDays 读取性能指标保留天数(三库共用,默认 3 天)。
|
||||
func metricRetentionDays(ctx context.Context) int {
|
||||
v, err := getConfig(ctx, model.ConfigKeyMetricRetentionDays)
|
||||
if err != nil {
|
||||
if !errors.Is(err, errConfigReaderNotWired) {
|
||||
logger.ErrorF(ctx, "读取性能指标保留天数配置失败(key=%s),回退默认 %d 天: %v", model.ConfigKeyMetricRetentionDays, defaultMetricRetentionDays, err)
|
||||
}
|
||||
return defaultMetricRetentionDays
|
||||
}
|
||||
days, perr := strconv.Atoi(v)
|
||||
if perr != nil || days <= 0 {
|
||||
logger.ErrorF(ctx, "性能指标保留天数配置非法(key=%s, value=%q),回退默认 %d 天", model.ConfigKeyMetricRetentionDays, v, defaultMetricRetentionDays)
|
||||
return defaultMetricRetentionDays
|
||||
}
|
||||
return days
|
||||
}
|
||||
|
||||
// CleanupExpired 按当前激活库保留天数清理过期日志(每日由 system_cleanup 调用):
|
||||
// 访问日志(节点访问/用户访问)按 log_retention_days_* 清理;
|
||||
// 性能指标(CPU/内存/磁盘/网络)按三库共用的短留存 metric_retention_days 清理。
|
||||
func CleanupExpired(ctx context.Context) (*CleanupSummary, error) {
|
||||
dbName, err := resolveDatabase(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve active database: %w", err)
|
||||
}
|
||||
s, err := Active(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
days := retentionDaysForDatabase(ctx, dbName)
|
||||
metricDays := metricRetentionDays(ctx)
|
||||
cutoff := time.Now().AddDate(0, 0, -days)
|
||||
metricCutoff := time.Now().AddDate(0, 0, -metricDays)
|
||||
summary := &CleanupSummary{ActiveDatabase: dbName, RetentionDays: days, MetricRetentionDays: metricDays, Tables: []string{}}
|
||||
|
||||
// PG 分区表仅在迁移时预建「当前+2 月」分区,此处确保分区持续存在,
|
||||
// 否则跨月后新写入会报 "no partition of relation found"(SQLite/CH 为 no-op)。
|
||||
now := time.Now().UTC()
|
||||
if err := s.AccessLogs.EnsurePartitions(ctx, now, now.AddDate(0, partitionLeadMonths, 0)); err != nil {
|
||||
return nil, fmt.Errorf("ensure partitions: %w", err)
|
||||
}
|
||||
|
||||
// 先直接删除完全过期的整月分区(比逐行 DELETE 快几个数量级、无 MVCC/WAL 负担),
|
||||
// 再对边界月份执行 DeleteBefore(边界月仍可能含未过期数据,不可整表删)。
|
||||
if err := s.AccessLogs.DropExpiredPartitions(ctx, cutoff); err != nil {
|
||||
return nil, fmt.Errorf("drop expired partitions: %w", err)
|
||||
}
|
||||
|
||||
if err := cleanupTable("node_access_logs", func() (int64, error) {
|
||||
return s.AccessLogs.DeleteBefore(ctx, cutoff)
|
||||
}, summary); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 过期数据删除后清理旧月份空分区表,避免分区表无限累积;
|
||||
// 仅删「当前月之前」且无数据的分区(best-effort,失败不阻断数据保留清理)。
|
||||
if err := s.AccessLogs.DropEmptyPartitions(ctx, now); err != nil {
|
||||
logger.WarnF(ctx, "drop empty log partitions failed: %v", err)
|
||||
}
|
||||
|
||||
if err := cleanupTable("metric_snapshots", func() (int64, error) {
|
||||
return s.Observability.DeleteMetricSnapshotsBefore(ctx, metricCutoff)
|
||||
}, summary); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := cleanupTable("edge_health", func() (int64, error) {
|
||||
return s.Observability.DeleteEdgeHealthBefore(ctx, cutoff)
|
||||
}, summary); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := cleanupTable("obs_frps", func() (int64, error) {
|
||||
return s.Observability.DeleteNodeObservationFrpsBefore(ctx, cutoff)
|
||||
}, summary); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := cleanupTable("obs_frpc", func() (int64, error) {
|
||||
return s.Observability.DeleteNodeObservationFrpcBefore(ctx, cutoff)
|
||||
}, summary); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return summary, nil
|
||||
}
|
||||
|
||||
func cleanupTable(name string, fn func() (int64, error), summary *CleanupSummary) error {
|
||||
n, err := fn()
|
||||
if err != nil {
|
||||
return fmt.Errorf("cleanup %s: %w", name, err)
|
||||
}
|
||||
summary.Deleted += n
|
||||
summary.Tables = append(summary.Tables, name)
|
||||
return nil
|
||||
}
|
||||
|
||||
// partitionStatementsRange 生成覆盖 [from, to] 全部月份的两表分区 DDL,
|
||||
// 幂等 CREATE TABLE IF NOT EXISTS ... PARTITION OF ... FOR VALUES FROM ... TO ...。
|
||||
// 入参为任意时间点:按各自所在月份生成,含 from 月与 to 月(to 常用 max+1 月兜底)。
|
||||
func partitionStatementsRange(from, to time.Time) []string {
|
||||
var out []string
|
||||
start := time.Date(from.Year(), from.Month(), 1, 0, 0, 0, 0, time.UTC)
|
||||
end := time.Date(to.Year(), to.Month(), 1, 0, 0, 0, 0, time.UTC).AddDate(0, 1, 0)
|
||||
for ; start.Before(end); start = start.AddDate(0, 1, 0) {
|
||||
monthEnd := start.AddDate(0, 1, 0)
|
||||
suffix := start.Format("200601")
|
||||
fromDay := start.Format("2006-01-02")
|
||||
toDay := monthEnd.Format("2006-01-02")
|
||||
for _, table := range accessLogPartitionTables {
|
||||
out = append(out, fmt.Sprintf(
|
||||
"CREATE TABLE IF NOT EXISTS %s_%s PARTITION OF %s FOR VALUES FROM ('%s') TO ('%s')",
|
||||
table, suffix, table, fromDay, toDay))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// partitionNameMonth 解析按月分区表名 <table>_YYYYMM 的所属月份;命名不匹配返回 (零值, false)。
|
||||
func partitionNameMonth(table, name string) (time.Time, bool) {
|
||||
suffix, ok := strings.CutPrefix(name, table+"_")
|
||||
if !ok || len(suffix) != 6 {
|
||||
return time.Time{}, false
|
||||
}
|
||||
m, err := time.Parse("200601", suffix)
|
||||
if err != nil {
|
||||
return time.Time{}, false
|
||||
}
|
||||
return m, true
|
||||
}
|
||||
|
||||
// dropEligiblePartitionNames 返回 before 月份之前、命名合法的分区表名(是否为空由调用方校验)。
|
||||
func dropEligiblePartitionNames(table string, names []string, before time.Time) []string {
|
||||
beforeMonth := time.Date(before.Year(), before.Month(), 1, 0, 0, 0, 0, time.UTC)
|
||||
out := make([]string, 0, len(names))
|
||||
for _, name := range names {
|
||||
month, ok := partitionNameMonth(table, name)
|
||||
if !ok || !month.Before(beforeMonth) {
|
||||
continue // 非法命名或当月/未来月分区,必须保留
|
||||
}
|
||||
out = append(out, name)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,403 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package logstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/glebarez/sqlite"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/logger"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/model"
|
||||
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
|
||||
db "Wavelet/plugins/infra/database"
|
||||
)
|
||||
|
||||
// cleanupTestModels 清理涉及的 5 张日志/可观测表。
|
||||
func cleanupTestModels() []any {
|
||||
return []any{
|
||||
&analyticsmodel.NodeAccessLog{},
|
||||
&analyticsmodel.NodeMetricSnapshot{},
|
||||
&analyticsmodel.NodeEdgeHealth{},
|
||||
&analyticsmodel.NodeObsFrps{},
|
||||
&analyticsmodel.NodeObsFrpc{},
|
||||
}
|
||||
}
|
||||
|
||||
// newCleanupTestDB 构造内存 sqlite 库并注入 db.DB(CleanupExpired 经 Active → buildStore 使用)。
|
||||
func newCleanupTestDB(t *testing.T) *gorm.DB {
|
||||
t.Helper()
|
||||
dsn := fmt.Sprintf("file:logstore-cleanup-%d?mode=memory&cache=shared", atomic.AddInt64(&testGormStoreSeq, 1))
|
||||
gdb, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)})
|
||||
if err != nil {
|
||||
t.Fatalf("open sqlite: %v", err)
|
||||
}
|
||||
if err := gdb.AutoMigrate(cleanupTestModels()...); err != nil {
|
||||
t.Fatalf("automigrate: %v", err)
|
||||
}
|
||||
db.SetDB(gdb)
|
||||
t.Cleanup(func() { db.SetDB(nil) })
|
||||
return gdb
|
||||
}
|
||||
|
||||
// TestCleanupExpiredSQLite 验证 sqlite 激活库的过期日志清理:
|
||||
// 注入 log_retention_days_sqlite=30,40 天前的 5 表记录被删、昨天的保留。
|
||||
func TestCleanupExpiredSQLite(t *testing.T) {
|
||||
ResetForTest()
|
||||
SetConfigReader(func(_ context.Context, key string) (string, error) {
|
||||
switch key {
|
||||
case logDatabaseKey:
|
||||
return "sqlite", nil
|
||||
case model.ConfigKeyLogRetentionDaysSQLite:
|
||||
return "30", nil
|
||||
case model.ConfigKeyMetricRetentionDays:
|
||||
return "3", nil
|
||||
}
|
||||
return "", nil
|
||||
})
|
||||
defer ResetForTest()
|
||||
|
||||
gdb := newCleanupTestDB(t)
|
||||
ctx := context.Background()
|
||||
old := time.Now().AddDate(0, 0, -40).UTC()
|
||||
recent := time.Now().AddDate(0, 0, -1).UTC()
|
||||
|
||||
if err := gdb.Create([]analyticsmodel.NodeAccessLog{
|
||||
{ID: 1, NodeID: "n1", LoggedAt: old, RemoteAddr: "1.1.1.1"},
|
||||
{ID: 2, NodeID: "n1", LoggedAt: recent, RemoteAddr: "2.2.2.2"},
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("seed node access logs: %v", err)
|
||||
}
|
||||
if err := gdb.Create([]analyticsmodel.NodeMetricSnapshot{
|
||||
{ID: 1, NodeID: "n1", CapturedAt: old},
|
||||
{ID: 2, NodeID: "n1", CapturedAt: recent},
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("seed metric snapshots: %v", err)
|
||||
}
|
||||
if err := gdb.Create([]analyticsmodel.NodeEdgeHealth{
|
||||
{ID: 1, NodeID: "n1", CapturedAt: old},
|
||||
{ID: 2, NodeID: "n1", CapturedAt: recent},
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("seed edge health: %v", err)
|
||||
}
|
||||
if err := gdb.Create([]analyticsmodel.NodeObsFrps{
|
||||
{ID: 1, NodeID: "n1", CapturedAt: old},
|
||||
{ID: 2, NodeID: "n1", CapturedAt: recent},
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("seed obs frps: %v", err)
|
||||
}
|
||||
if err := gdb.Create([]analyticsmodel.NodeObsFrpc{
|
||||
{ID: 1, NodeID: "n1", CapturedAt: old},
|
||||
{ID: 2, NodeID: "n1", CapturedAt: recent},
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("seed obs frpc: %v", err)
|
||||
}
|
||||
|
||||
summary, err := CleanupExpired(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("CleanupExpired: %v", err)
|
||||
}
|
||||
if summary.ActiveDatabase != "sqlite" {
|
||||
t.Fatalf("ActiveDatabase = %q, want sqlite", summary.ActiveDatabase)
|
||||
}
|
||||
if summary.RetentionDays != 30 {
|
||||
t.Fatalf("RetentionDays = %d, want 30", summary.RetentionDays)
|
||||
}
|
||||
if summary.MetricRetentionDays != 3 {
|
||||
t.Fatalf("MetricRetentionDays = %d, want 3", summary.MetricRetentionDays)
|
||||
}
|
||||
if summary.Deleted != 5 {
|
||||
t.Fatalf("Deleted = %d, want 5", summary.Deleted)
|
||||
}
|
||||
if len(summary.Tables) != 5 {
|
||||
t.Fatalf("Tables = %v, want 5 tables", summary.Tables)
|
||||
}
|
||||
|
||||
assertCount := func(m any, want int64, label string) {
|
||||
t.Helper()
|
||||
var n int64
|
||||
if err := gdb.Model(m).Count(&n).Error; err != nil {
|
||||
t.Fatalf("count %s: %v", label, err)
|
||||
}
|
||||
if n != want {
|
||||
t.Fatalf("%s count = %d, want %d", label, n, want)
|
||||
}
|
||||
}
|
||||
assertCount(&analyticsmodel.NodeAccessLog{}, 1, "node_access_logs")
|
||||
assertCount(&analyticsmodel.NodeMetricSnapshot{}, 1, "metric_snapshots")
|
||||
assertCount(&analyticsmodel.NodeEdgeHealth{}, 1, "edge_health")
|
||||
assertCount(&analyticsmodel.NodeObsFrps{}, 1, "obs_frps")
|
||||
assertCount(&analyticsmodel.NodeObsFrpc{}, 1, "obs_frpc")
|
||||
|
||||
var kept analyticsmodel.NodeAccessLog
|
||||
if err := gdb.First(&kept).Error; err != nil {
|
||||
t.Fatalf("recent node access log missing: %v", err)
|
||||
}
|
||||
if kept.ID != 2 {
|
||||
t.Fatalf("kept log ID = %d, want 2 (recent)", kept.ID)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCleanupExpiredMetricShortRetention 回归:性能指标(CPU/内存/磁盘/网络)按三库共用
|
||||
// 的短留存(默认 3 天)清理,与访问日志保留天数(log_retention_days_*)解耦。
|
||||
// 10 天前的指标快照被删(> 3 天),同日期的访问日志保留(< 30 天)。
|
||||
func TestCleanupExpiredMetricShortRetention(t *testing.T) {
|
||||
ResetForTest()
|
||||
SetConfigReader(func(_ context.Context, key string) (string, error) {
|
||||
switch key {
|
||||
case logDatabaseKey:
|
||||
return "sqlite", nil
|
||||
case model.ConfigKeyLogRetentionDaysSQLite:
|
||||
return "30", nil
|
||||
case model.ConfigKeyMetricRetentionDays:
|
||||
return "3", nil
|
||||
}
|
||||
return "", nil
|
||||
})
|
||||
defer ResetForTest()
|
||||
|
||||
gdb := newCleanupTestDB(t)
|
||||
ctx := context.Background()
|
||||
mid := time.Now().AddDate(0, 0, -10).UTC() // 10 天前:超指标留存、未超日志留存
|
||||
|
||||
if err := gdb.Create(&analyticsmodel.NodeAccessLog{ID: 1, NodeID: "n1", LoggedAt: mid, RemoteAddr: "1.1.1.1"}).Error; err != nil {
|
||||
t.Fatalf("seed node access log: %v", err)
|
||||
}
|
||||
if err := gdb.Create(&analyticsmodel.NodeMetricSnapshot{ID: 1, NodeID: "n1", CapturedAt: mid}).Error; err != nil {
|
||||
t.Fatalf("seed metric snapshot: %v", err)
|
||||
}
|
||||
|
||||
summary, err := CleanupExpired(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("CleanupExpired: %v", err)
|
||||
}
|
||||
if summary.RetentionDays != 30 || summary.MetricRetentionDays != 3 {
|
||||
t.Fatalf("retention = (%d, %d), want (30, 3)", summary.RetentionDays, summary.MetricRetentionDays)
|
||||
}
|
||||
|
||||
var accessCount, metricCount int64
|
||||
if err := gdb.Model(&analyticsmodel.NodeAccessLog{}).Count(&accessCount).Error; err != nil {
|
||||
t.Fatalf("count access logs: %v", err)
|
||||
}
|
||||
if err := gdb.Model(&analyticsmodel.NodeMetricSnapshot{}).Count(&metricCount).Error; err != nil {
|
||||
t.Fatalf("count metric snapshots: %v", err)
|
||||
}
|
||||
if accessCount != 1 {
|
||||
t.Fatalf("node_access_logs count = %d, want 1 (10 天在 30 天日志留存内)", accessCount)
|
||||
}
|
||||
if metricCount != 0 {
|
||||
t.Fatalf("metric_snapshots count = %d, want 0 (10 天超 3 天指标留存)", metricCount)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMetricRetentionDays 覆盖性能指标保留天数读取:合法值、非法值回退默认 3。
|
||||
func TestMetricRetentionDays(t *testing.T) {
|
||||
ResetForTest()
|
||||
SetConfigReader(func(_ context.Context, key string) (string, error) {
|
||||
switch key {
|
||||
case model.ConfigKeyMetricRetentionDays:
|
||||
return "5", nil
|
||||
}
|
||||
return "", nil
|
||||
})
|
||||
if got := metricRetentionDays(context.Background()); got != 5 {
|
||||
t.Fatalf("metricRetentionDays = %d, want 5", got)
|
||||
}
|
||||
|
||||
// 非法值(非数字/<=0)回退默认 3。
|
||||
SetConfigReader(func(_ context.Context, key string) (string, error) {
|
||||
switch key {
|
||||
case model.ConfigKeyMetricRetentionDays:
|
||||
return "abc", nil
|
||||
}
|
||||
return "", nil
|
||||
})
|
||||
if got := metricRetentionDays(context.Background()); got != defaultMetricRetentionDays {
|
||||
t.Fatalf("metricRetentionDays invalid value = %d, want %d", got, defaultMetricRetentionDays)
|
||||
}
|
||||
|
||||
// reader 报错回退默认 3。
|
||||
SetConfigReader(func(_ context.Context, _ string) (string, error) {
|
||||
return "", fmt.Errorf("boom")
|
||||
})
|
||||
if got := metricRetentionDays(context.Background()); got != defaultMetricRetentionDays {
|
||||
t.Fatalf("metricRetentionDays reader error = %d, want %d", got, defaultMetricRetentionDays)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRetentionDaysForDatabase 覆盖保留天数读取:按激活库选 key、非法值回退默认 90。
|
||||
func TestRetentionDaysForDatabase(t *testing.T) {
|
||||
ResetForTest()
|
||||
SetConfigReader(func(_ context.Context, key string) (string, error) {
|
||||
switch key {
|
||||
case logDatabaseKey:
|
||||
return "sqlite", nil
|
||||
case model.ConfigKeyLogRetentionDaysSQLite:
|
||||
return "30", nil
|
||||
}
|
||||
return "", nil
|
||||
})
|
||||
if got := retentionDaysForDatabase(context.Background(), "sqlite"); got != 30 {
|
||||
t.Fatalf("retentionDaysForDatabase = %d, want 30", got)
|
||||
}
|
||||
|
||||
// 非法值(非数字/<=0)回退默认 90。
|
||||
SetConfigReader(func(_ context.Context, key string) (string, error) {
|
||||
switch key {
|
||||
case logDatabaseKey:
|
||||
return "postgres", nil
|
||||
case model.ConfigKeyLogRetentionDaysPostgres:
|
||||
return "abc", nil
|
||||
}
|
||||
return "", nil
|
||||
})
|
||||
if got := retentionDaysForDatabase(context.Background(), "postgres"); got != 90 {
|
||||
t.Fatalf("retentionDaysForDatabase invalid value = %d, want 90", got)
|
||||
}
|
||||
|
||||
// reader 报错回退默认 90。
|
||||
SetConfigReader(func(_ context.Context, _ string) (string, error) {
|
||||
return "", fmt.Errorf("boom")
|
||||
})
|
||||
if got := retentionDaysForDatabase(context.Background(), "postgres"); got != 90 {
|
||||
t.Fatalf("retentionDaysForDatabase reader error = %d, want 90", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPartitionStatements 验证 PG 分区 DDL 生成:当前月 + 未来 2 个月 × 2 表,
|
||||
// 幂等 PARTITION OF 语句与迁移 SQL 命名一致(含跨年)。
|
||||
func TestPartitionStatements(t *testing.T) {
|
||||
now := time.Date(2026, 8, 15, 10, 0, 0, 0, time.UTC)
|
||||
stmts := partitionStatementsRange(now, now.AddDate(0, 2, 0))
|
||||
if len(stmts) != 6 {
|
||||
t.Fatalf("partitionStatements len = %d, want 6", len(stmts))
|
||||
}
|
||||
want := []string{
|
||||
"CREATE TABLE IF NOT EXISTS of_node_access_logs_202608 PARTITION OF of_node_access_logs FOR VALUES FROM ('2026-08-01') TO ('2026-09-01')",
|
||||
"CREATE TABLE IF NOT EXISTS w_user_access_logs_202608 PARTITION OF w_user_access_logs FOR VALUES FROM ('2026-08-01') TO ('2026-09-01')",
|
||||
"CREATE TABLE IF NOT EXISTS of_node_access_logs_202609 PARTITION OF of_node_access_logs FOR VALUES FROM ('2026-09-01') TO ('2026-10-01')",
|
||||
"CREATE TABLE IF NOT EXISTS w_user_access_logs_202609 PARTITION OF w_user_access_logs FOR VALUES FROM ('2026-09-01') TO ('2026-10-01')",
|
||||
"CREATE TABLE IF NOT EXISTS of_node_access_logs_202610 PARTITION OF of_node_access_logs FOR VALUES FROM ('2026-10-01') TO ('2026-11-01')",
|
||||
"CREATE TABLE IF NOT EXISTS w_user_access_logs_202610 PARTITION OF w_user_access_logs FOR VALUES FROM ('2026-10-01') TO ('2026-11-01')",
|
||||
}
|
||||
for i, w := range want {
|
||||
if stmts[i] != w {
|
||||
t.Fatalf("stmt[%d] = %q, want %q", i, stmts[i], w)
|
||||
}
|
||||
}
|
||||
|
||||
// 跨年:2026-11 → 202611, 202612, 202701。
|
||||
nov := time.Date(2026, 11, 1, 0, 0, 0, 0, time.UTC)
|
||||
suffixes := []string{"202611", "202612", "202701"}
|
||||
for _, stmt := range partitionStatementsRange(nov, nov.AddDate(0, 2, 0)) {
|
||||
if !hasAnySuffix(stmt, suffixes) {
|
||||
t.Fatalf("statement lacks expected month suffix: %s", stmt)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func hasAnySuffix(stmt string, suffixes []string) bool {
|
||||
for _, table := range []string{"of_node_access_logs", "w_user_access_logs"} {
|
||||
for _, suf := range suffixes {
|
||||
if strings.Contains(stmt, table+"_"+suf) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// TestPartitionNameMonth 覆盖按月分区表名解析:合法命名返回所属月份,非法/其它表前缀返回 false。
|
||||
func TestPartitionNameMonth(t *testing.T) {
|
||||
cases := []struct {
|
||||
table string
|
||||
name string
|
||||
want string // 期望 "YYYY-MM";空串表示应解析失败
|
||||
}{
|
||||
{"of_node_access_logs", "of_node_access_logs_202608", "2026-08"},
|
||||
{"w_user_access_logs", "w_user_access_logs_202612", "2026-12"},
|
||||
{"of_node_access_logs", "w_user_access_logs_202608", ""}, // 其它表前缀
|
||||
{"of_node_access_logs", "of_node_access_logs_20268", ""}, // 位数不足
|
||||
{"of_node_access_logs", "of_node_access_logs_202613", ""}, // 非法月份
|
||||
{"of_node_access_logs", "of_node_access_logs_default", ""}, // 非数字后缀
|
||||
}
|
||||
for _, c := range cases {
|
||||
got, ok := partitionNameMonth(c.table, c.name)
|
||||
if c.want == "" {
|
||||
if ok {
|
||||
t.Fatalf("partitionNameMonth(%q, %q) ok = true, want false", c.table, c.name)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !ok || got.Format("2006-01") != c.want {
|
||||
t.Fatalf("partitionNameMonth(%q, %q) = %v, want %s", c.table, c.name, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestDropEligiblePartitionNames 覆盖空分区清理筛选:只保留 before 月份之前、命名合法的分区。
|
||||
func TestDropEligiblePartitionNames(t *testing.T) {
|
||||
before := time.Date(2026, 10, 15, 0, 0, 0, 0, time.UTC)
|
||||
names := []string{
|
||||
"of_node_access_logs_202608",
|
||||
"of_node_access_logs_202609",
|
||||
"of_node_access_logs_202610", // 当月:保留
|
||||
"of_node_access_logs_202611", // 未来:保留
|
||||
"of_node_access_logs_default", // 非法命名:忽略
|
||||
}
|
||||
got := dropEligiblePartitionNames("of_node_access_logs", names, before)
|
||||
want := []string{"of_node_access_logs_202608", "of_node_access_logs_202609"}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("eligible = %v, want %v", got, want)
|
||||
}
|
||||
for i, w := range want {
|
||||
if got[i] != w {
|
||||
t.Fatalf("eligible[%d] = %q, want %q", i, got[i], w)
|
||||
}
|
||||
}
|
||||
|
||||
// 月初边界:before 恰为当月 1 日 0 点,当月分区仍保留。
|
||||
first := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC)
|
||||
if got := dropEligiblePartitionNames("of_node_access_logs", []string{"of_node_access_logs_202610"}, first); len(got) != 0 {
|
||||
t.Fatalf("eligible at month boundary = %v, want empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDropExpiredPartitionsSQLiteNoop 验证 SQLite 下 DropExpiredPartitions 为 no-op:
|
||||
// 直接返回 nil、不触碰任何分区 SQL(SQLite 无分区),数据不受影响。
|
||||
func TestDropExpiredPartitionsSQLiteNoop(t *testing.T) {
|
||||
ResetForTest()
|
||||
SetConfigReader(func(_ context.Context, key string) (string, error) {
|
||||
if key == logDatabaseKey {
|
||||
return "sqlite", nil
|
||||
}
|
||||
return "", nil
|
||||
})
|
||||
defer ResetForTest()
|
||||
|
||||
gdb := newCleanupTestDB(t)
|
||||
ctx := context.Background()
|
||||
if err := gdb.Create(&analyticsmodel.NodeAccessLog{ID: 1, NodeID: "n1", LoggedAt: time.Now().AddDate(0, 0, -100).UTC(), RemoteAddr: "1.1.1.1"}).Error; err != nil {
|
||||
t.Fatalf("seed node access log: %v", err)
|
||||
}
|
||||
|
||||
store := newGormStore(gdb)
|
||||
if err := store.DropExpiredPartitions(ctx, time.Now().AddDate(0, 0, -90)); err != nil {
|
||||
t.Fatalf("DropExpiredPartitions on sqlite: %v", err)
|
||||
}
|
||||
|
||||
var n int64
|
||||
if err := gdb.Model(&analyticsmodel.NodeAccessLog{}).Count(&n).Error; err != nil {
|
||||
t.Fatalf("count node access logs: %v", err)
|
||||
}
|
||||
if n != 1 {
|
||||
t.Fatalf("node access log count = %d, want 1(no-op 不应删除任何行)", n)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,742 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package logstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"time"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/model"
|
||||
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
|
||||
analyticsrepo "Wavelet/openflare/plugins/server/kernel/repository/analytics"
|
||||
db "Wavelet/plugins/infra/database"
|
||||
|
||||
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
|
||||
)
|
||||
|
||||
// clickhouseLogStore 实现 AccessLogStore / ObservabilityStore / StatusStore,
|
||||
// 逐方法委托 analyticsrepo(CH 原生 batch 写入,零性能损耗)。
|
||||
// UserAccessLogStore 由 clickhouseUserAccessLogStore 实现(List/Count 方法名已被
|
||||
// AccessLogStore 占用,Go 不允许同名不同签名方法)。
|
||||
type clickhouseLogStore struct {
|
||||
// skipFreeze 为 true 时跳过迁移冻结检查(仅迁移目标 store 使用)。
|
||||
skipFreeze bool
|
||||
}
|
||||
|
||||
func newClickHouseStore() *clickhouseLogStore { return &clickhouseLogStore{} }
|
||||
|
||||
// 编译期断言。
|
||||
var (
|
||||
_ AccessLogStore = (*clickhouseLogStore)(nil)
|
||||
_ ObservabilityStore = (*clickhouseLogStore)(nil)
|
||||
_ StatusStore = (*clickhouseLogStore)(nil)
|
||||
_ UserAccessLogStore = (*clickhouseUserAccessLogStore)(nil)
|
||||
)
|
||||
|
||||
func chConnErr() error {
|
||||
if db.ChConn == nil {
|
||||
return errors.New("clickhouse connection is not initialized")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ensureWritable 迁移冻结期拒绝写入。
|
||||
func (s *clickhouseLogStore) ensureWritable(ctx context.Context) error {
|
||||
if !s.skipFreeze && Migrating(ctx) {
|
||||
return ErrMigrating
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ---- AccessLogStore ----
|
||||
|
||||
// InsertBatch 节点访问日志写入入口:冻结检查后经 hook 入队(异步),不直接落库。
|
||||
func (s *clickhouseLogStore) InsertBatch(ctx context.Context, records []*model.OpenFlareAccessLog) error {
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
rows := make([]analyticsmodel.NodeAccessLog, 0, len(records))
|
||||
for _, r := range records {
|
||||
if r == nil {
|
||||
continue
|
||||
}
|
||||
rows = append(rows, toAnalyticsNodeAccessLog(r))
|
||||
}
|
||||
if h := currentAccessLogHooks().QueueNodeAccessLogs; h != nil {
|
||||
h(rows)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// BatchInsertNodeAccessLogs 是 batchwriter flush 目标:CH 原生批量写入。
|
||||
func (s *clickhouseLogStore) BatchInsertNodeAccessLogs(ctx context.Context, rows []analyticsmodel.NodeAccessLog) error {
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
return analyticsrepo.BatchInsertNodeAccessLogs(ctx, rows)
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) List(ctx context.Context, query model.OpenFlareAccessLogQuery) ([]*model.OpenFlareAccessLog, error) {
|
||||
rows, err := analyticsrepo.ListNodeAccessLogs(ctx, toNodeAccessLogFilter(query))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return fromAnalyticsNodeAccessLogs(rows), nil
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) Count(ctx context.Context, query model.OpenFlareAccessLogQuery) (int64, int64, int64, error) {
|
||||
return analyticsrepo.CountNodeAccessLogs(ctx, toNodeAccessLogFilter(query))
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) RegionCounts(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareAccessLogRegionCount, error) {
|
||||
rows, err := analyticsrepo.RegionCountsNodeAccessLogs(ctx, nodeID, since, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]*model.OpenFlareAccessLogRegionCount, len(rows))
|
||||
for i, r := range rows {
|
||||
out[i] = &model.OpenFlareAccessLogRegionCount{Region: r.Region, Count: r.Count}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) BucketAggregates(ctx context.Context, query model.OpenFlareAccessLogQuery, bucketSeconds int64) ([]analyticsmodel.NodeAccessLogBucketAggregate, error) {
|
||||
return analyticsrepo.BucketAggregatesNodeAccessLogs(ctx, toNodeAccessLogFilter(query), bucketSeconds)
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) CountBuckets(ctx context.Context, query model.OpenFlareAccessLogQuery, bucketSeconds int64) (int64, error) {
|
||||
return analyticsrepo.CountBucketAggregatesNodeAccessLogs(ctx, toNodeAccessLogFilter(query), bucketSeconds)
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) BucketDimensions(ctx context.Context, query model.OpenFlareAccessLogQuery, column string, bucketSeconds int64) ([]analyticsmodel.NodeAccessLogBucketDimension, error) {
|
||||
return analyticsrepo.BucketDimensionsNodeAccessLogs(ctx, toNodeAccessLogFilter(query), column, bucketSeconds)
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) IPAggregates(ctx context.Context, query model.OpenFlareAccessLogQuery, exactRemoteAddr bool) ([]analyticsmodel.NodeAccessLogIPAggregate, error) {
|
||||
return analyticsrepo.IPAggregatesNodeAccessLogs(ctx, toNodeAccessLogFilter(query), exactRemoteAddr)
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) IPSummaries(ctx context.Context, query model.OpenFlareAccessLogQuery, recentSince time.Time) ([]analyticsmodel.NodeAccessLogIPSummary, error) {
|
||||
return analyticsrepo.IPSummariesNodeAccessLogs(ctx, toNodeAccessLogFilter(query), recentSince)
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) CountIPSummaries(ctx context.Context, query model.OpenFlareAccessLogQuery) (int64, error) {
|
||||
return analyticsrepo.CountIPSummaryNodeAccessLogs(ctx, toNodeAccessLogFilter(query))
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) WAFIPAggregates(ctx context.Context, query model.OpenFlareAccessLogQuery) ([]analyticsmodel.NodeAccessLogWAFIPAggregate, error) {
|
||||
return analyticsrepo.IPAggregatesForWAFNodeAccessLogs(ctx, toNodeAccessLogFilter(query))
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) IPTrend(ctx context.Context, query model.OpenFlareAccessLogQuery, bucketSeconds int64) ([]analyticsmodel.NodeAccessLogIPTrend, error) {
|
||||
return analyticsrepo.IPTrendNodeAccessLogs(ctx, toNodeAccessLogFilter(query), bucketSeconds)
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) TrafficSummary(ctx context.Context, query model.OpenFlareAccessLogQuery) (model.OpenFlareAccessLogTrafficSummary, error) {
|
||||
row, err := analyticsrepo.TrafficSummaryNodeAccessLogs(ctx, toNodeAccessLogFilter(query))
|
||||
if err != nil {
|
||||
return model.OpenFlareAccessLogTrafficSummary{}, err
|
||||
}
|
||||
return model.OpenFlareAccessLogTrafficSummary{
|
||||
RequestCount: row.RequestCount,
|
||||
ErrorCount: row.ErrorCount,
|
||||
UniqueIPCount: row.UniqueIPCount,
|
||||
BytesSent: row.BytesSent,
|
||||
RequestLength: row.RequestLength,
|
||||
NodeCount: row.NodeCount,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) ValueCounts(ctx context.Context, query model.OpenFlareAccessLogQuery, column string, limit int) ([]model.OpenFlareAccessLogValueCount, error) {
|
||||
rows, err := analyticsrepo.ValueCountsNodeAccessLogs(ctx, toNodeAccessLogFilter(query), column, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]model.OpenFlareAccessLogValueCount, len(rows))
|
||||
for i, r := range rows {
|
||||
out[i] = model.OpenFlareAccessLogValueCount{Value: r.Value, Count: r.Count}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) NodeAggregates(ctx context.Context, query model.OpenFlareAccessLogQuery) ([]model.OpenFlareAccessLogNodeAggregate, error) {
|
||||
rows, err := analyticsrepo.NodeAggregatesNodeAccessLogs(ctx, toNodeAccessLogFilter(query))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]model.OpenFlareAccessLogNodeAggregate, len(rows))
|
||||
for i, r := range rows {
|
||||
out[i] = model.OpenFlareAccessLogNodeAggregate{NodeID: r.NodeID, RequestCount: r.RequestCount, ErrorCount: r.ErrorCount, UniqueIPCount: r.UniqueIPCount}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) DeleteAll(ctx context.Context) (int64, error) {
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return analyticsrepo.DeleteAllNodeAccessLogs(ctx)
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) DeleteBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return analyticsrepo.DeleteNodeAccessLogsBefore(ctx, cutoff)
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) DeleteByNodeBefore(ctx context.Context, nodeID string, before time.Time) (int64, error) {
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return analyticsrepo.DeleteNodeAccessLogsByNodeBefore(ctx, nodeID, before)
|
||||
}
|
||||
|
||||
// ListForMigration 按 id 升序分页读取(迁移复制用):直接查询 CH 原生表。
|
||||
func (s *clickhouseLogStore) ListForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.NodeAccessLog, error) {
|
||||
if err := chConnErr(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows, err := db.ChConn.Query(ctx, `
|
||||
SELECT `+analyticsmodel.NodeAccessLog{}.InsertColumns()+`
|
||||
FROM `+analyticsmodel.NodeAccessLog{}.TableName()+`
|
||||
WHERE id > ?
|
||||
ORDER BY id ASC
|
||||
LIMIT ?`, afterID, limitOr(limit, migrationPageSize))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list node access logs for migration: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
var result []analyticsmodel.NodeAccessLog
|
||||
for rows.Next() {
|
||||
var item analyticsmodel.NodeAccessLog
|
||||
if err := rows.Scan(
|
||||
&item.ID,
|
||||
&item.NodeID,
|
||||
&item.LoggedAt,
|
||||
&item.RemoteAddr,
|
||||
&item.Region,
|
||||
&item.Host,
|
||||
&item.Path,
|
||||
&item.UserAgent,
|
||||
&item.CacheStatus,
|
||||
&item.StatusCode,
|
||||
&item.BytesSent,
|
||||
&item.RequestLength,
|
||||
&item.RequestTimeMs,
|
||||
&item.CreatedAt,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan node access log row: %w", err)
|
||||
}
|
||||
item.LoggedAt = item.LoggedAt.UTC()
|
||||
item.CreatedAt = item.CreatedAt.UTC()
|
||||
result = append(result, item)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// ---- ObservabilityStore ----
|
||||
|
||||
// InsertMetricSnapshot 写入入口:冻结检查 + 经 hook 入队(异步),不直接落库。
|
||||
func (s *clickhouseLogStore) InsertMetricSnapshot(ctx context.Context, record *model.OpenFlareMetricSnapshot) error {
|
||||
if record == nil {
|
||||
return nil
|
||||
}
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if h := currentObservabilityHooks().QueueMetricSnapshot; h != nil {
|
||||
h(toAnalyticsNodeMetricSnapshot(record))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) ListMetricSnapshots(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareMetricSnapshot, error) {
|
||||
rows, err := analyticsrepo.ListNodeMetricSnapshots(ctx, toNodeObservabilityFilter(nodeID, since, limit))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return fromAnalyticsNodeMetricSnapshots(rows), nil
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) DeleteAllMetricSnapshots(ctx context.Context) (int64, error) {
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return analyticsrepo.DeleteAllNodeMetricSnapshots(ctx)
|
||||
}
|
||||
|
||||
// ListTrafficHourly 委托 analyticsrepo 读 of_access_log_hourly rollup(M5 口径,UV 恒 0)。
|
||||
func (s *clickhouseLogStore) ListTrafficHourly(ctx context.Context, nodeID string, since time.Time) ([]analyticsmodel.NodeTrafficHourly, error) {
|
||||
return analyticsrepo.ListNodeTrafficHourly(ctx, toNodeObservabilitySince(nodeID, since))
|
||||
}
|
||||
|
||||
// ListAccessLogHourly 委托 analyticsrepo 读 of_access_log_hourly rollup。
|
||||
func (s *clickhouseLogStore) ListAccessLogHourly(ctx context.Context, nodeID string, since time.Time) ([]analyticsmodel.AccessLogHourly, error) {
|
||||
return analyticsrepo.ListAccessLogHourly(ctx, toNodeObservabilitySince(nodeID, since))
|
||||
}
|
||||
|
||||
// ListMetricHourly 委托 analyticsrepo ListNodeMetricHourly:rollup 覆盖窗口时读
|
||||
// of_node_metric_capacity_hourly,否则按 mergeNodeMetricHourlyPreferRollup 合并 raw 兜底。
|
||||
func (s *clickhouseLogStore) ListMetricHourly(ctx context.Context, nodeID string, since time.Time) ([]analyticsmodel.NodeMetricHourly, error) {
|
||||
return analyticsrepo.ListNodeMetricHourly(ctx, toNodeObservabilitySince(nodeID, since))
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) DeleteMetricSnapshotsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return analyticsrepo.DeleteNodeMetricSnapshotsBefore(ctx, cutoff)
|
||||
}
|
||||
|
||||
// BatchInsertNodeMetricSnapshots 是 batchwriter flush 目标:CH 原生批量写入。
|
||||
func (s *clickhouseLogStore) BatchInsertNodeMetricSnapshots(ctx context.Context, rows []analyticsmodel.NodeMetricSnapshot) error {
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
return analyticsrepo.BatchInsertNodeMetricSnapshots(ctx, rows)
|
||||
}
|
||||
|
||||
// InsertEdgeHealth 写入入口:冻结检查 + 经 hook 入队(异步),不直接落库。
|
||||
func (s *clickhouseLogStore) InsertEdgeHealth(ctx context.Context, record *model.OpenFlareEdgeHealth) error {
|
||||
if record == nil {
|
||||
return nil
|
||||
}
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if h := currentObservabilityHooks().QueueEdgeHealth; h != nil {
|
||||
h(toAnalyticsNodeEdgeHealth(record))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) ListEdgeHealth(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareEdgeHealth, error) {
|
||||
rows, err := analyticsrepo.ListNodeEdgeHealth(ctx, toNodeObservabilityFilter(nodeID, since, limit))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return fromAnalyticsNodeEdgeHealths(rows), nil
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) DeleteAllEdgeHealth(ctx context.Context) (int64, error) {
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return analyticsrepo.DeleteAllNodeEdgeHealth(ctx)
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) DeleteEdgeHealthBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return analyticsrepo.DeleteNodeEdgeHealthBefore(ctx, cutoff)
|
||||
}
|
||||
|
||||
// BatchInsertNodeEdgeHealth 是 batchwriter flush 目标:CH 原生批量写入。
|
||||
func (s *clickhouseLogStore) BatchInsertNodeEdgeHealth(ctx context.Context, rows []analyticsmodel.NodeEdgeHealth) error {
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
return analyticsrepo.BatchInsertNodeEdgeHealth(ctx, rows)
|
||||
}
|
||||
|
||||
// InsertNodeObservationFrps 写入入口:冻结检查 + 经 hook 入队(异步),不直接落库。
|
||||
func (s *clickhouseLogStore) InsertNodeObservationFrps(ctx context.Context, record *model.OpenFlareNodeObservationFrps) error {
|
||||
if record == nil {
|
||||
return nil
|
||||
}
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if h := currentObservabilityHooks().QueueNodeObsFrps; h != nil {
|
||||
h(toAnalyticsNodeObsFrps(record))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) ListNodeObservationFrps(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareNodeObservationFrps, error) {
|
||||
rows, err := analyticsrepo.ListNodeObsFrps(ctx, toNodeObservabilityFilter(nodeID, since, limit))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return fromAnalyticsNodeObsFrps(rows), nil
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) DeleteAllNodeObservationFrps(ctx context.Context) (int64, error) {
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return analyticsrepo.DeleteAllNodeObsFrps(ctx)
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) DeleteNodeObservationFrpsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return analyticsrepo.DeleteNodeObsFrpsBefore(ctx, cutoff)
|
||||
}
|
||||
|
||||
// BatchInsertNodeObsFrps 是 batchwriter flush 目标:CH 原生批量写入。
|
||||
func (s *clickhouseLogStore) BatchInsertNodeObsFrps(ctx context.Context, rows []analyticsmodel.NodeObsFrps) error {
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
return analyticsrepo.BatchInsertNodeObsFrps(ctx, rows)
|
||||
}
|
||||
|
||||
// InsertNodeObservationFrpc 写入入口:冻结检查 + 经 hook 入队(异步),不直接落库。
|
||||
func (s *clickhouseLogStore) InsertNodeObservationFrpc(ctx context.Context, record *model.OpenFlareNodeObservationFrpc) error {
|
||||
if record == nil {
|
||||
return nil
|
||||
}
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if h := currentObservabilityHooks().QueueNodeObsFrpc; h != nil {
|
||||
h(toAnalyticsNodeObsFrpc(record))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) ListNodeObservationFrpc(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareNodeObservationFrpc, error) {
|
||||
rows, err := analyticsrepo.ListNodeObsFrpc(ctx, toNodeObservabilityFilter(nodeID, since, limit))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return fromAnalyticsNodeObsFrpc(rows), nil
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) DeleteAllNodeObservationFrpc(ctx context.Context) (int64, error) {
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return analyticsrepo.DeleteAllNodeObsFrpc(ctx)
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) DeleteNodeObservationFrpcBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return analyticsrepo.DeleteNodeObsFrpcBefore(ctx, cutoff)
|
||||
}
|
||||
|
||||
// MigrationRange 返回 of_node_access_logs.logged_at 的最小/最大值(空表返回零值)。
|
||||
func (s *clickhouseLogStore) MigrationRange(ctx context.Context) (time.Time, time.Time, error) {
|
||||
return chMigrationRange(ctx, analyticsmodel.NodeAccessLog{}.TableName(), "logged_at")
|
||||
}
|
||||
|
||||
// EnsurePartitions 是 CH 分支 no-op(CH 无 PG 式分区)。
|
||||
func (s *clickhouseLogStore) EnsurePartitions(_ context.Context, _, _ time.Time) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// DropEmptyPartitions 是 CH 分支 no-op(CH 分区随数据删除自动消失,无独立分区表)。
|
||||
func (s *clickhouseLogStore) DropEmptyPartitions(_ context.Context, _ time.Time) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// DropExpiredPartitions 是 CH 分支 no-op(CH 无 PG 式分区,retention 仍走 DeleteBefore)。
|
||||
func (s *clickhouseLogStore) DropExpiredPartitions(_ context.Context, _ time.Time) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// chMigrationRange 查询 CH 表时间列 MIN/MAX;空表(NULL)返回零值。
|
||||
func chMigrationRange(ctx context.Context, table, column string) (time.Time, time.Time, error) {
|
||||
if err := chConnErr(); err != nil {
|
||||
return time.Time{}, time.Time{}, err
|
||||
}
|
||||
var minTime, maxTime *time.Time
|
||||
if err := db.ChConn.QueryRow(ctx,
|
||||
"SELECT min("+column+"), max("+column+") FROM "+table,
|
||||
).Scan(&minTime, &maxTime); err != nil {
|
||||
return time.Time{}, time.Time{}, fmt.Errorf("query migration range %s: %w", table, err)
|
||||
}
|
||||
if minTime == nil || maxTime == nil {
|
||||
return time.Time{}, time.Time{}, nil
|
||||
}
|
||||
return minTime.UTC(), maxTime.UTC(), nil
|
||||
}
|
||||
|
||||
// BatchInsertNodeObsFrpc 是 batchwriter flush 目标:CH 原生批量写入。
|
||||
func (s *clickhouseLogStore) BatchInsertNodeObsFrpc(ctx context.Context, rows []analyticsmodel.NodeObsFrpc) error {
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
return analyticsrepo.BatchInsertNodeObsFrpc(ctx, rows)
|
||||
}
|
||||
|
||||
// ListMetricSnapshotsForMigration 按 id 升序分页读取(迁移复制用)。
|
||||
func (s *clickhouseLogStore) ListMetricSnapshotsForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.NodeMetricSnapshot, error) {
|
||||
return chListForMigration(ctx, afterID, limit,
|
||||
analyticsmodel.NodeMetricSnapshot{}.TableName(),
|
||||
analyticsmodel.NodeMetricSnapshot{}.InsertColumns(),
|
||||
func(rows driver.Rows) ([]analyticsmodel.NodeMetricSnapshot, error) {
|
||||
var result []analyticsmodel.NodeMetricSnapshot
|
||||
for rows.Next() {
|
||||
var item analyticsmodel.NodeMetricSnapshot
|
||||
if err := rows.Scan(
|
||||
&item.ID,
|
||||
&item.NodeID,
|
||||
&item.CapturedAt,
|
||||
&item.CPUUsagePercent,
|
||||
&item.MemoryUsedBytes,
|
||||
&item.MemoryTotalBytes,
|
||||
&item.StorageUsedBytes,
|
||||
&item.StorageTotalBytes,
|
||||
&item.DiskReadBytes,
|
||||
&item.DiskWriteBytes,
|
||||
&item.NetworkRxBytes,
|
||||
&item.NetworkTxBytes,
|
||||
&item.CreatedAt,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan node metric snapshot row: %w", err)
|
||||
}
|
||||
item.CapturedAt = item.CapturedAt.UTC()
|
||||
item.CreatedAt = item.CreatedAt.UTC()
|
||||
result = append(result, item)
|
||||
}
|
||||
return result, nil
|
||||
})
|
||||
}
|
||||
|
||||
// chObsRow 迁移读取共用的双字段观测行(字符串状态 + 数值计数):
|
||||
// edge_health(status/connections)与 obs_frpc(tunnel_status/connected_relays_count)同形状。
|
||||
type chObsRow struct {
|
||||
ID uint64
|
||||
NodeID string
|
||||
CapturedAt time.Time
|
||||
Status string
|
||||
Count int64
|
||||
CreatedAt time.Time
|
||||
}
|
||||
|
||||
// countToInt32 将观测计数转为 int32(防御溢出;观测计数远小于 int32 上限)。
|
||||
func countToInt32(v int64) int32 {
|
||||
if v > math.MaxInt32 {
|
||||
return math.MaxInt32
|
||||
}
|
||||
if v < math.MinInt32 {
|
||||
return math.MinInt32
|
||||
}
|
||||
return int32(v)
|
||||
}
|
||||
|
||||
// scanChObsRow 扫描 chObsRow(含 UTC 归一化)。
|
||||
func scanChObsRow(rows driver.Rows) ([]chObsRow, error) {
|
||||
var result []chObsRow
|
||||
for rows.Next() {
|
||||
var item chObsRow
|
||||
if err := rows.Scan(
|
||||
&item.ID,
|
||||
&item.NodeID,
|
||||
&item.CapturedAt,
|
||||
&item.Status,
|
||||
&item.Count,
|
||||
&item.CreatedAt,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan observation row: %w", err)
|
||||
}
|
||||
item.CapturedAt = item.CapturedAt.UTC()
|
||||
item.CreatedAt = item.CreatedAt.UTC()
|
||||
result = append(result, item)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// ListEdgeHealthForMigration 按 id 升序分页读取(迁移复制用)。
|
||||
func (s *clickhouseLogStore) ListEdgeHealthForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.NodeEdgeHealth, error) {
|
||||
rows, err := chListForMigration(ctx, afterID, limit,
|
||||
analyticsmodel.NodeEdgeHealth{}.TableName(),
|
||||
analyticsmodel.NodeEdgeHealth{}.InsertColumns(),
|
||||
scanChObsRow)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]analyticsmodel.NodeEdgeHealth, len(rows))
|
||||
for i, r := range rows {
|
||||
out[i] = analyticsmodel.NodeEdgeHealth{ID: r.ID, NodeID: r.NodeID, CapturedAt: r.CapturedAt, Status: r.Status, Connections: r.Count, CreatedAt: r.CreatedAt}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ListNodeObsFrpsForMigration 按 id 升序分页读取(迁移复制用)。
|
||||
func (s *clickhouseLogStore) ListNodeObsFrpsForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.NodeObsFrps, error) {
|
||||
return chListForMigration(ctx, afterID, limit,
|
||||
analyticsmodel.NodeObsFrps{}.TableName(),
|
||||
analyticsmodel.NodeObsFrps{}.InsertColumns(),
|
||||
func(rows driver.Rows) ([]analyticsmodel.NodeObsFrps, error) {
|
||||
var result []analyticsmodel.NodeObsFrps
|
||||
for rows.Next() {
|
||||
var item analyticsmodel.NodeObsFrps
|
||||
if err := rows.Scan(
|
||||
&item.ID,
|
||||
&item.NodeID,
|
||||
&item.CapturedAt,
|
||||
&item.FrpsConnections,
|
||||
&item.FrpsProxyCount,
|
||||
&item.FrpsClientCount,
|
||||
&item.FrpsProxies,
|
||||
&item.CreatedAt,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan node frps observation row: %w", err)
|
||||
}
|
||||
item.CapturedAt = item.CapturedAt.UTC()
|
||||
item.CreatedAt = item.CreatedAt.UTC()
|
||||
result = append(result, item)
|
||||
}
|
||||
return result, nil
|
||||
})
|
||||
}
|
||||
|
||||
// ListNodeObsFrpcForMigration 按 id 升序分页读取(迁移复制用)。
|
||||
func (s *clickhouseLogStore) ListNodeObsFrpcForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.NodeObsFrpc, error) {
|
||||
rows, err := chListForMigration(ctx, afterID, limit,
|
||||
analyticsmodel.NodeObsFrpc{}.TableName(),
|
||||
analyticsmodel.NodeObsFrpc{}.InsertColumns(),
|
||||
scanChObsRow)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]analyticsmodel.NodeObsFrpc, len(rows))
|
||||
for i, r := range rows {
|
||||
out[i] = analyticsmodel.NodeObsFrpc{ID: r.ID, NodeID: r.NodeID, CapturedAt: r.CapturedAt, TunnelStatus: r.Status, ConnectedRelaysCount: countToInt32(r.Count), CreatedAt: r.CreatedAt}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// chListForMigration 执行按 id 升序分页的 CH 原生表查询,并交给 scanner 扫描。
|
||||
func chListForMigration[T any](ctx context.Context, afterID uint64, limit int, table, columns string, scanner func(driver.Rows) ([]T, error)) ([]T, error) {
|
||||
if err := chConnErr(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows, err := db.ChConn.Query(ctx, `
|
||||
SELECT `+columns+`
|
||||
FROM `+table+`
|
||||
WHERE id > ?
|
||||
ORDER BY id ASC
|
||||
LIMIT ?`, afterID, limitOr(limit, migrationPageSize))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list %s for migration: %w", table, err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
return scanner(rows)
|
||||
}
|
||||
|
||||
// ---- StatusStore ----
|
||||
|
||||
// ActiveDatabase 返回当前日志主库名(CH 分支固定 clickhouse)。
|
||||
func (s *clickhouseLogStore) ActiveDatabase(_ context.Context) (string, error) {
|
||||
return dbNameClickHouse, nil
|
||||
}
|
||||
|
||||
// ClickHouseOperationalStats 委托 analyticsrepo 汇总 CH 运行状态。
|
||||
func (s *clickhouseLogStore) ClickHouseOperationalStats(ctx context.Context) (*analyticsmodel.ClickHouseOperationalStats, error) {
|
||||
return analyticsrepo.GetClickHouseOperationalStats(ctx)
|
||||
}
|
||||
|
||||
// ---- UserAccessLogStore ----
|
||||
|
||||
// clickhouseUserAccessLogStore 实现 UserAccessLogStore。clickhouseLogStore 已占用
|
||||
// List/Count 方法名(AccessLogStore 接口),Go 不允许同名不同签名方法,故用户访问日志
|
||||
// 用独立类型嵌入同一 clickhouseLogStore(与 userAccessLogGormStore 同构),复用 ensureWritable。
|
||||
type clickhouseUserAccessLogStore struct {
|
||||
*clickhouseLogStore
|
||||
}
|
||||
|
||||
func newClickHouseUserAccessLogStore() *clickhouseUserAccessLogStore {
|
||||
return &clickhouseUserAccessLogStore{clickhouseLogStore: newClickHouseStore()}
|
||||
}
|
||||
|
||||
// BatchInsert 是 batchwriter flush 目标:CH 原生批量写入;冻结期拒绝写入,空批次直接返回。
|
||||
func (s *clickhouseUserAccessLogStore) BatchInsert(ctx context.Context, logs []analyticsmodel.UserAccessLog) error {
|
||||
if len(logs) == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
return analyticsrepo.BatchInsert(ctx, logs)
|
||||
}
|
||||
|
||||
// DeleteAll 清空全部用户访问日志(TRUNCATE 语义,迁移「覆盖目标库已有日志」幂等前提用)。
|
||||
func (s *clickhouseUserAccessLogStore) DeleteAll(ctx context.Context) (int64, error) {
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return analyticsrepo.DeleteAllUserAccessLogs(ctx)
|
||||
}
|
||||
|
||||
// ListForMigration 按 id 升序分页读取(迁移复制用)。
|
||||
func (s *clickhouseUserAccessLogStore) ListForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.UserAccessLog, error) {
|
||||
return chListForMigration(ctx, afterID, limit,
|
||||
analyticsmodel.UserAccessLog{}.TableName(),
|
||||
analyticsmodel.UserAccessLog{}.InsertColumns(),
|
||||
func(rows driver.Rows) ([]analyticsmodel.UserAccessLog, error) {
|
||||
var result []analyticsmodel.UserAccessLog
|
||||
for rows.Next() {
|
||||
var item analyticsmodel.UserAccessLog
|
||||
if err := rows.Scan(
|
||||
&item.ID,
|
||||
&item.UserID,
|
||||
&item.Path,
|
||||
&item.Method,
|
||||
&item.IP,
|
||||
&item.UserAgent,
|
||||
&item.Headers,
|
||||
&item.Status,
|
||||
&item.Latency,
|
||||
&item.CreatedAt,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan user access log row: %w", err)
|
||||
}
|
||||
item.CreatedAt = item.CreatedAt.UTC()
|
||||
result = append(result, item)
|
||||
}
|
||||
return result, nil
|
||||
})
|
||||
}
|
||||
|
||||
// MigrationRange 返回 w_user_access_logs.created_at 的最小/最大值(空表返回零值)。
|
||||
func (s *clickhouseUserAccessLogStore) MigrationRange(ctx context.Context) (time.Time, time.Time, error) {
|
||||
return chMigrationRange(ctx, analyticsmodel.UserAccessLog{}.TableName(), "created_at")
|
||||
}
|
||||
|
||||
func (s *clickhouseUserAccessLogStore) Count(ctx context.Context, filter analyticsmodel.AccessLogFilter) (uint64, error) {
|
||||
return analyticsrepo.CountAccessLogs(ctx, filter)
|
||||
}
|
||||
|
||||
func (s *clickhouseUserAccessLogStore) List(ctx context.Context, filter analyticsmodel.AccessLogFilter, page, pageSize int) ([]analyticsmodel.UserAccessLog, uint64, error) {
|
||||
return analyticsrepo.ListAccessLogs(ctx, filter, page, pageSize)
|
||||
}
|
||||
|
||||
func (s *clickhouseUserAccessLogStore) GetDailyTrend(ctx context.Context, days int) ([]analyticsmodel.DailyTrend, error) {
|
||||
return analyticsrepo.GetDailyTrend(ctx, days)
|
||||
}
|
||||
|
||||
func (s *clickhouseUserAccessLogStore) GetBrowserDistribution(ctx context.Context, startTime time.Time) ([]analyticsmodel.BrowserShare, error) {
|
||||
return analyticsrepo.GetBrowserDistribution(ctx, startTime)
|
||||
}
|
||||
|
||||
func (s *clickhouseUserAccessLogStore) GetTopActiveUsers(ctx context.Context, startTime time.Time, limit int) ([]analyticsmodel.TopUser, error) {
|
||||
return analyticsrepo.GetTopActiveUsers(ctx, startTime, limit)
|
||||
}
|
||||
|
||||
// toNodeObservabilityFilter 构造 CH 可观测查询过滤器(limit<=0 表示不限制)。
|
||||
func toNodeObservabilityFilter(nodeID string, since time.Time, limit int) analyticsmodel.NodeObservabilityFilter {
|
||||
return analyticsmodel.NodeObservabilityFilter{
|
||||
NodeID: nodeID,
|
||||
Since: since,
|
||||
Limit: limit,
|
||||
}
|
||||
}
|
||||
|
||||
// toNodeObservabilitySince 构造不带 limit 的可观测查询过滤器
|
||||
// (小时级聚合读无需分页,避免传无意义的 0)。
|
||||
func toNodeObservabilitySince(nodeID string, since time.Time) analyticsmodel.NodeObservabilityFilter {
|
||||
return analyticsmodel.NodeObservabilityFilter{NodeID: nodeID, Since: since}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package logstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
db "Wavelet/plugins/infra/database"
|
||||
)
|
||||
|
||||
// TestClickHouseHourlyDelegationRegression 验证 CH 后端小时级聚合读委托 analyticsrepo:
|
||||
// 未初始化 CH 连接时返回 analyticsrepo 的 "clickhouse connection is not initialized" 错误
|
||||
// (而非未实现/panic),证明 3 个方法都路由到 CH 原生查询。
|
||||
func TestClickHouseHourlyDelegationRegression(t *testing.T) {
|
||||
if db.ChConn != nil {
|
||||
t.Skip("clickhouse connection initialized; skipping delegation regression")
|
||||
}
|
||||
s := newClickHouseStore()
|
||||
ctx := context.Background()
|
||||
now := time.Now()
|
||||
check := func(name string, err error) {
|
||||
t.Helper()
|
||||
if err == nil {
|
||||
t.Fatalf("%s: want clickhouse-not-initialized error, got nil", name)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "clickhouse connection is not initialized") {
|
||||
t.Fatalf("%s: unexpected error %v", name, err)
|
||||
}
|
||||
}
|
||||
_, err := s.ListTrafficHourly(ctx, "n1", now)
|
||||
check("ListTrafficHourly", err)
|
||||
_, err = s.ListAccessLogHourly(ctx, "n1", now)
|
||||
check("ListAccessLogHourly", err)
|
||||
_, err = s.ListMetricHourly(ctx, "n1", now)
|
||||
check("ListMetricHourly", err)
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package logstore
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// timeBucketSQLPostgres 返回 PG 时间分桶表达式(epoch 秒 -> 分桶起点,int64)。
|
||||
func timeBucketSQLPostgres(column string, bucketSeconds int64) string {
|
||||
return "(floor(extract(epoch from " + column + ")/" + strconv.FormatInt(bucketSeconds, 10) + ")*" + strconv.FormatInt(bucketSeconds, 10) + ")::bigint"
|
||||
}
|
||||
|
||||
// dailyTrendDateSQLPostgres 返回 PG 按日聚合的日期表达式。
|
||||
func dailyTrendDateSQLPostgres() string {
|
||||
return "to_char(created_at, 'YYYY-MM-DD')"
|
||||
}
|
||||
|
||||
// epochSQLPostgres 返回 PG epoch 秒表达式(int64)。
|
||||
func epochSQLPostgres(column string) string {
|
||||
return "extract(epoch from " + column + ")::bigint"
|
||||
}
|
||||
|
||||
// textCastSQLPostgres 返回 PG 数值列转文本表达式。
|
||||
func textCastSQLPostgres(column string) string {
|
||||
return column + "::text"
|
||||
}
|
||||
|
||||
// distinctNonEmptyCountSQLPostgres 返回 PG 排除空串的 distinct 计数表达式。
|
||||
func distinctNonEmptyCountSQLPostgres(column string) string {
|
||||
return "COUNT(DISTINCT " + column + ") FILTER (WHERE " + column + " <> '')"
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package logstore
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// isPostgresDialect 判断 gorm 句柄是否为 PostgreSQL 方言(否则按 SQLite 处理)。
|
||||
// Dialector 经 gorm.Config 内嵌提升,Name() 可直接在 DB 上调用。
|
||||
func isPostgresDialect(db *gorm.DB) bool {
|
||||
return db != nil && db.Dialector != nil && db.Name() == "postgres"
|
||||
}
|
||||
|
||||
// timeBucketSQLSQLite 返回 SQLite 时间分桶表达式(epoch 秒 -> 分桶起点)。
|
||||
func timeBucketSQLSQLite(column string, bucketSeconds int64) string {
|
||||
return "(floor(unixepoch(" + column + ")/" + strconv.FormatInt(bucketSeconds, 10) + ")*" + strconv.FormatInt(bucketSeconds, 10) + ")"
|
||||
}
|
||||
|
||||
// dailyTrendDateSQLSQLite 返回 SQLite 按日聚合的日期表达式。
|
||||
func dailyTrendDateSQLSQLite() string {
|
||||
return "strftime('%Y-%m-%d', created_at)"
|
||||
}
|
||||
|
||||
// epochSQLSQLite 返回 SQLite epoch 秒表达式(unixepoch 整数秒)。
|
||||
func epochSQLSQLite(column string) string {
|
||||
return "unixepoch(" + column + ")"
|
||||
}
|
||||
|
||||
// textCastSQLSQLite 返回 SQLite 数值列转文本表达式。
|
||||
func textCastSQLSQLite(column string) string {
|
||||
return "CAST(" + column + " AS TEXT)"
|
||||
}
|
||||
|
||||
// distinctNonEmptyCountSQLSQLite 返回 SQLite 排除空串的 distinct 计数表达式
|
||||
// (SQLite 无 FILTER 语法,用 CASE 等价实现)。
|
||||
func distinctNonEmptyCountSQLSQLite(column string) string {
|
||||
return "COUNT(DISTINCT CASE WHEN " + column + " <> '' THEN " + column + " END)"
|
||||
}
|
||||
|
||||
// distinctNonEmptyCountSQL 按当前方言返回排除空串的 distinct 计数表达式
|
||||
// (运行时按 Dialector 分发,默认 SQLite)。
|
||||
func distinctNonEmptyCountSQL(db *gorm.DB, column string) string {
|
||||
if isPostgresDialect(db) {
|
||||
return distinctNonEmptyCountSQLPostgres(column)
|
||||
}
|
||||
return distinctNonEmptyCountSQLSQLite(column)
|
||||
}
|
||||
|
||||
// dailyTrendDateSQL 按当前方言返回按日聚合的日期表达式(运行时按 Dialector 分发,默认 SQLite)。
|
||||
func dailyTrendDateSQL(db *gorm.DB) string {
|
||||
if isPostgresDialect(db) {
|
||||
return dailyTrendDateSQLPostgres()
|
||||
}
|
||||
return dailyTrendDateSQLSQLite()
|
||||
}
|
||||
|
||||
// epochSQL 按当前方言返回 epoch 秒表达式(运行时按 Dialector 分发,默认 SQLite)。
|
||||
func epochSQL(db *gorm.DB, column string) string {
|
||||
if isPostgresDialect(db) {
|
||||
return epochSQLPostgres(column)
|
||||
}
|
||||
return epochSQLSQLite(column)
|
||||
}
|
||||
|
||||
// textCastSQL 按当前方言返回数值列转文本表达式(运行时按 Dialector 分发,默认 SQLite)。
|
||||
func textCastSQL(db *gorm.DB, column string) string {
|
||||
if isPostgresDialect(db) {
|
||||
return textCastSQLPostgres(column)
|
||||
}
|
||||
return textCastSQLSQLite(column)
|
||||
}
|
||||
|
||||
// timeBucketSQL 按当前方言返回时间分桶表达式。
|
||||
// brief 将 PG/SQLite 两版写为同名函数,同包无法共存;log_database 为运行时配置,
|
||||
// 不能使用编译期 build tag,故按 db.Dialector.Name() 运行时分发(默认 SQLite)。
|
||||
func timeBucketSQL(db *gorm.DB, column string, bucketSeconds int64) string {
|
||||
if isPostgresDialect(db) {
|
||||
return timeBucketSQLPostgres(column, bucketSeconds)
|
||||
}
|
||||
return timeBucketSQLSQLite(column, bucketSeconds)
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package logstore
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
|
||||
)
|
||||
|
||||
// AccessLogHooks 节点访问日志异步入队回调(由 chwriter 装配)。
|
||||
type AccessLogHooks struct {
|
||||
QueueNodeAccessLogs func(logs []analyticsmodel.NodeAccessLog)
|
||||
}
|
||||
|
||||
// ObservabilityHooks 可观测异步入队回调(由 chwriter 装配)。
|
||||
type ObservabilityHooks struct {
|
||||
QueueMetricSnapshot func(record analyticsmodel.NodeMetricSnapshot)
|
||||
QueueEdgeHealth func(record analyticsmodel.NodeEdgeHealth)
|
||||
QueueNodeObsFrps func(record analyticsmodel.NodeObsFrps)
|
||||
QueueNodeObsFrpc func(record analyticsmodel.NodeObsFrpc)
|
||||
}
|
||||
|
||||
var (
|
||||
hooksMu sync.RWMutex
|
||||
accessLogHooks AccessLogHooks
|
||||
observabilityHooks ObservabilityHooks
|
||||
)
|
||||
|
||||
// SetAccessLogHooks 注册节点访问日志异步入队回调。
|
||||
func SetAccessLogHooks(h AccessLogHooks) {
|
||||
hooksMu.Lock()
|
||||
accessLogHooks = h
|
||||
hooksMu.Unlock()
|
||||
}
|
||||
|
||||
// SetObservabilityHooks 注册可观测异步入队回调。
|
||||
func SetObservabilityHooks(h ObservabilityHooks) {
|
||||
hooksMu.Lock()
|
||||
observabilityHooks = h
|
||||
hooksMu.Unlock()
|
||||
}
|
||||
|
||||
// currentAccessLogHooks 返回当前 hooks 快照(未注册时为 zero value,调用方判空跳过)。
|
||||
func currentAccessLogHooks() AccessLogHooks {
|
||||
hooksMu.RLock()
|
||||
defer hooksMu.RUnlock()
|
||||
return accessLogHooks
|
||||
}
|
||||
|
||||
// currentObservabilityHooks 返回当前 hooks 快照(未注册时为 zero value,调用方判空跳过)。
|
||||
func currentObservabilityHooks() ObservabilityHooks {
|
||||
hooksMu.RLock()
|
||||
defer hooksMu.RUnlock()
|
||||
return observabilityHooks
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package logstore
|
||||
|
||||
import (
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// serverPkg 是下游 server 插件的包路径前缀。
|
||||
const serverPkg = "Wavelet/openflare/plugins/server"
|
||||
|
||||
// forbiddenImports 业务域禁止直接触碰的底层日志实现。
|
||||
var forbiddenImports = []string{
|
||||
serverPkg + "/kernel/repository/analytics",
|
||||
}
|
||||
|
||||
// allowedAnalyticsDelegation 允许直接依赖 analytics 仓储的委托层:
|
||||
// - repository:持久化门面,ListOpenFlareLatestMetricSnapshotsSince 的
|
||||
// CH 快速路径仍直连 analytics(LIMIT 1 BY node_id);小时级聚合读已改走 logstore;
|
||||
// - repository/logstore:CH 后端实现按设计委托 analytics。
|
||||
//
|
||||
// 除此之外,依赖闭包内任何包都禁止引入 analytics 仓储。
|
||||
var allowedAnalyticsDelegation = map[string]bool{
|
||||
serverPkg + "/kernel/repository": true,
|
||||
serverPkg + "/kernel/repository/logstore": true,
|
||||
}
|
||||
|
||||
// allowedInfraPersistence 允许业务域包引入的 infra/persistence 子包。
|
||||
var allowedInfraPersistence = []string{
|
||||
serverPkg + "/infra/persistence/batchwriter", // batchwriter 统计类型
|
||||
serverPkg + "/infra/persistence/idgen", // 雪花 ID 生成(无日志依赖)
|
||||
}
|
||||
|
||||
// domainScopes 是 server 插件内的业务域包(等价于改造前的 internal/apps/...)。
|
||||
// 持久化与基础设施层(repository/infra/model/…)不受本门禁约束。
|
||||
var domainScopes = []string{
|
||||
"domain/site", "domain/fleet", "domain/pages", "domain/waf", "domain/tls",
|
||||
"domain/cloudflare", "domain/observability", "domain/dashboard", "domain/option",
|
||||
"updater",
|
||||
}
|
||||
|
||||
func TestDomainsMustNotImportLogBackendDirectly(t *testing.T) {
|
||||
t.Chdir(moduleRoot(t))
|
||||
|
||||
wanted := make([]string, 0, len(domainScopes))
|
||||
patterns := make([]string, 0, len(domainScopes))
|
||||
for _, d := range domainScopes {
|
||||
wanted = append(wanted, serverPkg+"/"+d)
|
||||
patterns = append(patterns, "./openflare/plugins/server/"+d+"/...")
|
||||
}
|
||||
|
||||
args := append([]string{"list", "-test", "-f", `{{.ImportPath}} {{join .Imports " "}}`}, patterns...)
|
||||
//nolint:gosec // 固定参数,无外部输入
|
||||
out, err := exec.Command("go", args...).Output()
|
||||
if err != nil {
|
||||
t.Fatalf("go list: %v", err)
|
||||
}
|
||||
|
||||
scanned := 0
|
||||
for _, line := range strings.Split(string(out), "\n") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) == 0 {
|
||||
continue
|
||||
}
|
||||
pkg := fields[0]
|
||||
if !hasAnyPrefix(pkg, wanted) {
|
||||
continue
|
||||
}
|
||||
scanned++
|
||||
for _, imp := range fields[1:] {
|
||||
for _, forbidden := range forbiddenImports {
|
||||
if imp == forbidden && !allowedAnalyticsDelegation[pkg] {
|
||||
t.Errorf("%s must not import forbidden log backend %s", pkg, forbidden)
|
||||
}
|
||||
}
|
||||
if strings.HasPrefix(imp, serverPkg+"/infra/persistence/") {
|
||||
allowed := false
|
||||
for _, a := range allowedInfraPersistence {
|
||||
if imp == a || strings.HasPrefix(imp, a+"/") {
|
||||
allowed = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !allowed {
|
||||
t.Errorf("%s must not import infra/persistence subpackage directly: %s", pkg, imp)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// 扫描到 0 个包说明包路径已漂移,门禁会静默失效——必须报错而非给绿灯。
|
||||
if scanned == 0 {
|
||||
t.Fatalf("no domain package scanned; domainScopes is stale: %v", wanted)
|
||||
}
|
||||
}
|
||||
|
||||
func hasAnyPrefix(s string, prefixes []string) bool {
|
||||
for _, p := range prefixes {
|
||||
if s == p || strings.HasPrefix(s, p+"/") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// moduleRoot 向 go 查询模块根目录,避免依赖测试文件所在深度的相对路径。
|
||||
func moduleRoot(t *testing.T) string {
|
||||
t.Helper()
|
||||
cmd := exec.Command("go", "list", "-m", "-f", "{{.Dir}}")
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
t.Fatalf("resolve module root: %v", err)
|
||||
}
|
||||
return strings.TrimSpace(string(out))
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package logstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/model"
|
||||
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
|
||||
)
|
||||
|
||||
// TestGormAccessLogInsertBatchHooks 覆盖访问日志写入入口:
|
||||
// 冻结检查、hook 入队、不直接落库、flush 后可见(行为与旧 repository clickhouse 包装一致)。
|
||||
func TestGormAccessLogInsertBatchHooks(t *testing.T) {
|
||||
ResetForTest()
|
||||
SetConfigReader(func(_ context.Context, key string) (string, error) {
|
||||
return "", nil
|
||||
})
|
||||
defer ResetForTest()
|
||||
|
||||
s := newTestGormStore(t)
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
|
||||
var hooked []analyticsmodel.NodeAccessLog
|
||||
SetAccessLogHooks(AccessLogHooks{
|
||||
QueueNodeAccessLogs: func(logs []analyticsmodel.NodeAccessLog) {
|
||||
hooked = append(hooked, logs...)
|
||||
},
|
||||
})
|
||||
defer SetAccessLogHooks(AccessLogHooks{})
|
||||
|
||||
records := []*model.OpenFlareAccessLog{
|
||||
{NodeID: "n1", LoggedAt: now, RemoteAddr: "1.1.1.1", StatusCode: 200, BytesSent: 100},
|
||||
{NodeID: "n1", LoggedAt: now, RemoteAddr: "2.2.2.2", StatusCode: 404},
|
||||
}
|
||||
if err := s.InsertBatch(ctx, records); err != nil {
|
||||
t.Fatalf("insert batch: %v", err)
|
||||
}
|
||||
if len(hooked) != 2 || hooked[0].RemoteAddr != "1.1.1.1" || hooked[0].BytesSent != 100 || hooked[1].StatusCode != 404 {
|
||||
t.Fatalf("hook rows mismatch: %+v", hooked)
|
||||
}
|
||||
// 写入入口只入队、不直接落库。
|
||||
rows, err := s.List(ctx, model.OpenFlareAccessLogQuery{NodeID: "n1"})
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
if len(rows) != 0 {
|
||||
t.Fatalf("entry insert must not write rows, got %d", len(rows))
|
||||
}
|
||||
// flush 后可见。
|
||||
if err := s.BatchInsertNodeAccessLogs(ctx, hooked); err != nil {
|
||||
t.Fatalf("flush: %v", err)
|
||||
}
|
||||
rows, err = s.List(ctx, model.OpenFlareAccessLogQuery{NodeID: "n1"})
|
||||
if err != nil {
|
||||
t.Fatalf("list after flush: %v", err)
|
||||
}
|
||||
if len(rows) != 2 {
|
||||
t.Fatalf("list after flush want 2, got %d", len(rows))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Package logstore 提供日志/分析存储抽象:上层只面向本包接口,
|
||||
// 禁止直接 import internal/repository/analytics 或触碰 db.ChConn/db.ChDB。
|
||||
package logstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/model"
|
||||
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
|
||||
)
|
||||
|
||||
// ErrMigrating 表示日志数据库正在迁移,当前禁止写入。
|
||||
var ErrMigrating = errors.New("log database is migrating, writes are disabled")
|
||||
|
||||
// AccessLogStore 节点访问日志(of_node_access_logs)。
|
||||
type AccessLogStore interface {
|
||||
// InsertBatch 为写入入口:冻结检查 + 经 hook 入队(异步),不直接落库。
|
||||
InsertBatch(ctx context.Context, records []*model.OpenFlareAccessLog) error
|
||||
// BatchInsertNodeAccessLogs 为 batchwriter flush 目标:直接批量写入当前存储。
|
||||
BatchInsertNodeAccessLogs(ctx context.Context, rows []analyticsmodel.NodeAccessLog) error
|
||||
|
||||
List(ctx context.Context, query model.OpenFlareAccessLogQuery) ([]*model.OpenFlareAccessLog, error)
|
||||
Count(ctx context.Context, query model.OpenFlareAccessLogQuery) (int64, int64, int64, error)
|
||||
RegionCounts(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareAccessLogRegionCount, error)
|
||||
BucketAggregates(ctx context.Context, filter model.OpenFlareAccessLogQuery, bucketSeconds int64) ([]analyticsmodel.NodeAccessLogBucketAggregate, error)
|
||||
CountBuckets(ctx context.Context, filter model.OpenFlareAccessLogQuery, bucketSeconds int64) (int64, error)
|
||||
BucketDimensions(ctx context.Context, filter model.OpenFlareAccessLogQuery, column string, bucketSeconds int64) ([]analyticsmodel.NodeAccessLogBucketDimension, error)
|
||||
IPAggregates(ctx context.Context, filter model.OpenFlareAccessLogQuery, exactRemoteAddr bool) ([]analyticsmodel.NodeAccessLogIPAggregate, error)
|
||||
IPSummaries(ctx context.Context, filter model.OpenFlareAccessLogQuery, recentSince time.Time) ([]analyticsmodel.NodeAccessLogIPSummary, error)
|
||||
CountIPSummaries(ctx context.Context, filter model.OpenFlareAccessLogQuery) (int64, error)
|
||||
WAFIPAggregates(ctx context.Context, filter model.OpenFlareAccessLogQuery) ([]analyticsmodel.NodeAccessLogWAFIPAggregate, error)
|
||||
IPTrend(ctx context.Context, filter model.OpenFlareAccessLogQuery, bucketSeconds int64) ([]analyticsmodel.NodeAccessLogIPTrend, error)
|
||||
TrafficSummary(ctx context.Context, filter model.OpenFlareAccessLogQuery) (model.OpenFlareAccessLogTrafficSummary, error)
|
||||
ValueCounts(ctx context.Context, filter model.OpenFlareAccessLogQuery, column string, limit int) ([]model.OpenFlareAccessLogValueCount, error)
|
||||
NodeAggregates(ctx context.Context, filter model.OpenFlareAccessLogQuery) ([]model.OpenFlareAccessLogNodeAggregate, error)
|
||||
DeleteAll(ctx context.Context) (int64, error)
|
||||
DeleteBefore(ctx context.Context, cutoff time.Time) (int64, error)
|
||||
DeleteByNodeBefore(ctx context.Context, nodeID string, before time.Time) (int64, error)
|
||||
// ListForMigration 按 id 升序分页读取(迁移复制用)。
|
||||
ListForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.NodeAccessLog, error)
|
||||
// MigrationRange 返回源表 logged_at 的最小/最大值(空表返回零值),迁移预建分区用。
|
||||
MigrationRange(ctx context.Context) (from, to time.Time, err error)
|
||||
// EnsurePartitions 幂等预建 PG 分区(按月),覆盖 [from, to] 月份;CH/SQLite 为 no-op。
|
||||
// 目标为 PG 的迁移在复制前调用,避免历史数据写入报 "no partition of relation found"。
|
||||
EnsurePartitions(ctx context.Context, from, to time.Time) error
|
||||
// DropEmptyPartitions 幂等清理 PG 空分区表:删除 before 月份之前、且无任何数据的按月分区;
|
||||
// CH/SQLite 为 no-op(CH 分区随数据删除自动消失、SQLite 无分区)。
|
||||
DropEmptyPartitions(ctx context.Context, before time.Time) error
|
||||
// DropExpiredPartitions 直接删除完全过期的 PG 整月分区(候选为月份早于 cutoff 月的分区,
|
||||
// 删除前校验分区内无保留期内数据,避免时区偏移下误删;迁移冻结期间拒绝执行);CH/SQLite 为 no-op。
|
||||
DropExpiredPartitions(ctx context.Context, cutoff time.Time) error
|
||||
}
|
||||
|
||||
// ObservabilityStore 可观测 4 表(metric snapshots / edge health / frps / frpc)。
|
||||
type ObservabilityStore interface {
|
||||
InsertMetricSnapshot(ctx context.Context, record *model.OpenFlareMetricSnapshot) error
|
||||
ListMetricSnapshots(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareMetricSnapshot, error)
|
||||
DeleteAllMetricSnapshots(ctx context.Context) (int64, error)
|
||||
DeleteMetricSnapshotsBefore(ctx context.Context, cutoff time.Time) (int64, error)
|
||||
BatchInsertNodeMetricSnapshots(ctx context.Context, rows []analyticsmodel.NodeMetricSnapshot) error
|
||||
|
||||
// ListTrafficHourly 返回小时级流量汇总(按 node/hour 聚合,unique_visitor_count 恒 0)。
|
||||
// CH 后端读 of_access_log_hourly rollup;PG/SQLite 从 of_node_access_logs 实时聚合。
|
||||
ListTrafficHourly(ctx context.Context, nodeID string, since time.Time) ([]analyticsmodel.NodeTrafficHourly, error)
|
||||
// ListAccessLogHourly 返回按 node/hour/host 的小时级访问日志汇总。
|
||||
ListAccessLogHourly(ctx context.Context, nodeID string, since time.Time) ([]analyticsmodel.AccessLogHourly, error)
|
||||
// ListMetricHourly 返回小时级指标聚合(avg cpu/memory + 计数器增量,reported_nodes 去重节点数)。
|
||||
ListMetricHourly(ctx context.Context, nodeID string, since time.Time) ([]analyticsmodel.NodeMetricHourly, error)
|
||||
|
||||
InsertEdgeHealth(ctx context.Context, record *model.OpenFlareEdgeHealth) error
|
||||
ListEdgeHealth(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareEdgeHealth, error)
|
||||
DeleteAllEdgeHealth(ctx context.Context) (int64, error)
|
||||
DeleteEdgeHealthBefore(ctx context.Context, cutoff time.Time) (int64, error)
|
||||
BatchInsertNodeEdgeHealth(ctx context.Context, rows []analyticsmodel.NodeEdgeHealth) error
|
||||
|
||||
InsertNodeObservationFrps(ctx context.Context, record *model.OpenFlareNodeObservationFrps) error
|
||||
ListNodeObservationFrps(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareNodeObservationFrps, error)
|
||||
DeleteAllNodeObservationFrps(ctx context.Context) (int64, error)
|
||||
DeleteNodeObservationFrpsBefore(ctx context.Context, cutoff time.Time) (int64, error)
|
||||
BatchInsertNodeObsFrps(ctx context.Context, rows []analyticsmodel.NodeObsFrps) error
|
||||
|
||||
InsertNodeObservationFrpc(ctx context.Context, record *model.OpenFlareNodeObservationFrpc) error
|
||||
ListNodeObservationFrpc(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareNodeObservationFrpc, error)
|
||||
DeleteAllNodeObservationFrpc(ctx context.Context) (int64, error)
|
||||
DeleteNodeObservationFrpcBefore(ctx context.Context, cutoff time.Time) (int64, error)
|
||||
BatchInsertNodeObsFrpc(ctx context.Context, rows []analyticsmodel.NodeObsFrpc) error
|
||||
|
||||
// 迁移复制用:按 id 升序分页读取。
|
||||
ListMetricSnapshotsForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.NodeMetricSnapshot, error)
|
||||
ListEdgeHealthForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.NodeEdgeHealth, error)
|
||||
ListNodeObsFrpsForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.NodeObsFrps, error)
|
||||
ListNodeObsFrpcForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.NodeObsFrpc, error)
|
||||
}
|
||||
|
||||
// UserAccessLogStore 用户访问日志(w_user_access_logs)。
|
||||
type UserAccessLogStore interface {
|
||||
BatchInsert(ctx context.Context, logs []analyticsmodel.UserAccessLog) error
|
||||
// DeleteAll 清空全部用户访问日志(迁移「覆盖目标库已有日志」幂等前提用)。
|
||||
DeleteAll(ctx context.Context) (int64, error)
|
||||
Count(ctx context.Context, filter analyticsmodel.AccessLogFilter) (uint64, error)
|
||||
List(ctx context.Context, filter analyticsmodel.AccessLogFilter, page, pageSize int) ([]analyticsmodel.UserAccessLog, uint64, error)
|
||||
GetDailyTrend(ctx context.Context, days int) ([]analyticsmodel.DailyTrend, error)
|
||||
GetBrowserDistribution(ctx context.Context, startTime time.Time) ([]analyticsmodel.BrowserShare, error)
|
||||
GetTopActiveUsers(ctx context.Context, startTime time.Time, limit int) ([]analyticsmodel.TopUser, error)
|
||||
// ListForMigration 按 id 升序分页读取(迁移复制用)。
|
||||
ListForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.UserAccessLog, error)
|
||||
// MigrationRange 返回源表 created_at 的最小/最大值(空表返回零值),迁移预建分区用。
|
||||
MigrationRange(ctx context.Context) (from, to time.Time, err error)
|
||||
}
|
||||
|
||||
// StatusStore 日志库状态(供管理端状态端点)。
|
||||
type StatusStore interface {
|
||||
ActiveDatabase(ctx context.Context) (string, error)
|
||||
ClickHouseOperationalStats(ctx context.Context) (*analyticsmodel.ClickHouseOperationalStats, error) // 仅 CH 激活时非 nil
|
||||
}
|
||||
|
||||
// Store 聚合当前生效日志库的全部域存储。
|
||||
type Store struct {
|
||||
AccessLogs AccessLogStore
|
||||
Observability ObservabilityStore
|
||||
UserAccessLogs UserAccessLogStore
|
||||
Status StatusStore
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package logstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// listPartitionNames 列出 table 在当前 schema 下的全部直接分区表名(pg_inherits)。
|
||||
func listPartitionNames(ctx context.Context, gdb *gorm.DB, table string) ([]string, error) {
|
||||
var names []string
|
||||
if err := gdb.WithContext(ctx).Raw(`
|
||||
SELECT c.relname
|
||||
FROM pg_inherits i
|
||||
JOIN pg_class c ON c.oid = i.inhrelid
|
||||
JOIN pg_class p ON p.oid = i.inhparent
|
||||
JOIN pg_namespace n ON n.oid = p.relnamespace AND n.nspname = current_schema()
|
||||
WHERE p.relname = ?`, table).Scan(&names).Error; err != nil {
|
||||
return nil, fmt.Errorf("list partitions of %s: %w", table, err)
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
|
||||
// DropExpiredPartitions 直接删除完全过期的 PG 整月分区(避免 retention 清理逐行 DELETE):
|
||||
// 候选 = 月份早于 cutoff 月(按 cutoff 的 UTC 时刻取月,避免本地时区偏移超前误删)的分区,
|
||||
// 且删除前校验分区内不存在 logged_at >= cutoff 的行(分区边界随会话时区偏移,
|
||||
// 名称月份只能粗筛,必须以数据为准);仅处理 of_node_access_logs
|
||||
// (w_user_access_logs 无 retention 清理,刻意不删其分区);迁移冻结期间(ensureWritable)
|
||||
// 直接返回 ErrMigrating,避免对冻结源库整月 DROP 丢数据;CH/SQLite 为 no-op。
|
||||
func (s *gormLogStore) DropExpiredPartitions(ctx context.Context, cutoff time.Time) error {
|
||||
if !isPostgresDialect(s.db) {
|
||||
return nil
|
||||
}
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
names, err := listPartitionNames(ctx, s.db, "of_node_access_logs")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cu := cutoff.UTC()
|
||||
cutoffMonth := time.Date(cu.Year(), cu.Month(), 1, 0, 0, 0, 0, time.UTC)
|
||||
for _, name := range names {
|
||||
month, ok := partitionNameMonth("of_node_access_logs", name)
|
||||
if !ok || !month.Before(cutoffMonth) {
|
||||
continue // 非法命名或当月/未来月分区,必须保留
|
||||
}
|
||||
// 数据校验:分区内仍有 logged_at >= cutoff 的行则保留(时区偏移下名称月份可能超前于真实边界)。
|
||||
var hasRetained int
|
||||
if err := s.db.WithContext(ctx).Raw("SELECT 1 FROM "+name+" WHERE logged_at >= ? LIMIT 1", cu).Scan(&hasRetained).Error; err != nil {
|
||||
return fmt.Errorf("check partition %s retained rows: %w", name, err)
|
||||
}
|
||||
if hasRetained == 1 {
|
||||
continue
|
||||
}
|
||||
if err := s.db.WithContext(ctx).Exec("DROP TABLE IF EXISTS " + name).Error; err != nil {
|
||||
return fmt.Errorf("drop expired partition %s: %w", name, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
+700
@@ -0,0 +1,700 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package logstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gorm.io/driver/postgres"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/logger"
|
||||
|
||||
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
|
||||
)
|
||||
|
||||
// TestEnsurePartitionsPostgresInsertAcrossMonths 需要 TEST_POSTGRES_DSN(未设置时跳过):
|
||||
// 验证 EnsurePartitions 预建任意月份范围分区后,跨月历史数据可写入 PG 分区表
|
||||
// (对应迁移任务从 CH/SQLite 复制历史日志到 PG 时先预建分区的场景)。
|
||||
func TestEnsurePartitionsPostgresInsertAcrossMonths(t *testing.T) {
|
||||
dsn := strings.TrimSpace(os.Getenv("TEST_POSTGRES_DSN"))
|
||||
if dsn == "" {
|
||||
t.Skip("TEST_POSTGRES_DSN is not set")
|
||||
}
|
||||
|
||||
gdb, err := gorm.Open(postgres.Open(dsn), &gorm.Config{
|
||||
DisableForeignKeyConstraintWhenMigrating: true,
|
||||
Logger: logger.Default.LogMode(logger.Silent),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("open postgres: %v", err)
|
||||
}
|
||||
sqlDB, err := gdb.DB()
|
||||
if err != nil {
|
||||
t.Fatalf("sql db: %v", err)
|
||||
}
|
||||
sqlDB.SetMaxOpenConns(1)
|
||||
|
||||
schema := fmt.Sprintf("logstore_partition_%d", time.Now().UnixNano())
|
||||
if !regexp.MustCompile(`^[a-z0-9_]+$`).MatchString(schema) {
|
||||
t.Fatalf("invalid schema: %s", schema)
|
||||
}
|
||||
if err := gdb.Exec(`CREATE SCHEMA "` + schema + `"`).Error; err != nil {
|
||||
t.Fatalf("create schema: %v", err)
|
||||
}
|
||||
if err := gdb.Exec(`SET search_path TO "` + schema + `"`).Error; err != nil {
|
||||
t.Fatalf("set search_path: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_ = gdb.Exec("SET search_path TO public").Error
|
||||
_ = gdb.Exec(`DROP SCHEMA IF EXISTS "` + schema + `" CASCADE`).Error
|
||||
_ = sqlDB.Close()
|
||||
})
|
||||
|
||||
// 与 goose/postgres/202608080001_create_log_tables.sql 保持一致的分区父表 DDL。
|
||||
for _, ddl := range []string{postgresNodeAccessLogsDDL, postgresUserAccessLogsDDL} {
|
||||
if err := gdb.Exec(ddl).Error; err != nil {
|
||||
t.Fatalf("create partitioned table: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
ResetForTest()
|
||||
SetConfigReader(func(_ context.Context, _ string) (string, error) { return "", nil })
|
||||
defer ResetForTest()
|
||||
|
||||
ctx := context.Background()
|
||||
store := newGormStore(gdb)
|
||||
ua := newUserAccessLogGormStore(gdb)
|
||||
|
||||
// 源范围跨 3 个月:2026-01-10 ~ 2026-03-20;to+1 月兜底生成 202601..202604 分区。
|
||||
from := time.Date(2026, 1, 10, 8, 0, 0, 0, time.UTC)
|
||||
max := time.Date(2026, 3, 20, 9, 30, 0, 0, time.UTC)
|
||||
if err := store.EnsurePartitions(ctx, from, max.AddDate(0, 1, 0)); err != nil {
|
||||
t.Fatalf("EnsurePartitions: %v", err)
|
||||
}
|
||||
|
||||
// 幂等:重复调用不报错(CREATE TABLE IF NOT EXISTS ... PARTITION OF)。
|
||||
if err := store.EnsurePartitions(ctx, from, max.AddDate(0, 1, 0)); err != nil {
|
||||
t.Fatalf("EnsurePartitions idempotent: %v", err)
|
||||
}
|
||||
|
||||
var partitionCount int64
|
||||
if err := gdb.Raw(
|
||||
"SELECT count(*) FROM pg_inherits WHERE inhparent = to_regclass('of_node_access_logs')",
|
||||
).Scan(&partitionCount).Error; err != nil {
|
||||
t.Fatalf("count partitions: %v", err)
|
||||
}
|
||||
if partitionCount != 4 {
|
||||
t.Fatalf("of_node_access_logs partitions = %d, want 4", partitionCount)
|
||||
}
|
||||
|
||||
// 跨月插入:1/2/3 月各 2 条节点访问日志 + 2 条用户访问日志,均应命中已有分区。
|
||||
nodeRows := []analyticsmodel.NodeAccessLog{
|
||||
{ID: 1, NodeID: "n1", LoggedAt: time.Date(2026, 1, 15, 0, 0, 0, 0, time.UTC), RemoteAddr: "1.1.1.1"},
|
||||
{ID: 2, NodeID: "n1", LoggedAt: time.Date(2026, 1, 20, 0, 0, 0, 0, time.UTC), RemoteAddr: "1.1.1.2"},
|
||||
{ID: 3, NodeID: "n2", LoggedAt: time.Date(2026, 2, 10, 0, 0, 0, 0, time.UTC), RemoteAddr: "2.2.2.2"},
|
||||
{ID: 4, NodeID: "n2", LoggedAt: time.Date(2026, 2, 12, 0, 0, 0, 0, time.UTC), RemoteAddr: "2.2.2.3"},
|
||||
{ID: 5, NodeID: "n1", LoggedAt: time.Date(2026, 3, 5, 0, 0, 0, 0, time.UTC), RemoteAddr: "3.3.3.3"},
|
||||
{ID: 6, NodeID: "n1", LoggedAt: time.Date(2026, 3, 18, 0, 0, 0, 0, time.UTC), RemoteAddr: "3.3.3.4"},
|
||||
}
|
||||
if err := store.BatchInsertNodeAccessLogs(ctx, nodeRows); err != nil {
|
||||
t.Fatalf("insert node access logs across months: %v", err)
|
||||
}
|
||||
|
||||
userRows := []analyticsmodel.UserAccessLog{
|
||||
{ID: 1, UserID: 101, Path: "/a", CreatedAt: time.Date(2026, 1, 16, 0, 0, 0, 0, time.UTC)},
|
||||
{ID: 2, UserID: 102, Path: "/b", CreatedAt: time.Date(2026, 3, 17, 0, 0, 0, 0, time.UTC)},
|
||||
}
|
||||
if err := ua.BatchInsert(ctx, userRows); err != nil {
|
||||
t.Fatalf("insert user access logs across months: %v", err)
|
||||
}
|
||||
|
||||
var nodeCount, userCount int64
|
||||
if err := gdb.Model(&analyticsmodel.NodeAccessLog{}).Count(&nodeCount).Error; err != nil {
|
||||
t.Fatalf("count node access logs: %v", err)
|
||||
}
|
||||
if err := gdb.Model(&analyticsmodel.UserAccessLog{}).Count(&userCount).Error; err != nil {
|
||||
t.Fatalf("count user access logs: %v", err)
|
||||
}
|
||||
if nodeCount != 6 {
|
||||
t.Fatalf("node access log count = %d, want 6", nodeCount)
|
||||
}
|
||||
if userCount != 2 {
|
||||
t.Fatalf("user access log count = %d, want 2", userCount)
|
||||
}
|
||||
|
||||
// MigrationRange 返回跨月范围(覆盖两表)。
|
||||
gotFrom, gotTo, err := store.MigrationRange(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("node MigrationRange: %v", err)
|
||||
}
|
||||
if !gotFrom.Equal(time.Date(2026, 1, 15, 0, 0, 0, 0, time.UTC)) || !gotTo.Equal(time.Date(2026, 3, 18, 0, 0, 0, 0, time.UTC)) {
|
||||
t.Fatalf("node MigrationRange = %s ~ %s, want 2026-01-15 ~ 2026-03-18", gotFrom, gotTo)
|
||||
}
|
||||
uaFrom, uaTo, err := ua.MigrationRange(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("user MigrationRange: %v", err)
|
||||
}
|
||||
if !uaFrom.Equal(time.Date(2026, 1, 16, 0, 0, 0, 0, time.UTC)) || !uaTo.Equal(time.Date(2026, 3, 17, 0, 0, 0, 0, time.UTC)) {
|
||||
t.Fatalf("user MigrationRange = %s ~ %s", uaFrom, uaTo)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDropEmptyPartitionsPostgres 需要 TEST_POSTGRES_DSN(未设置时跳过):
|
||||
// 验证空分区清理只删除 before 月份之前且无数据的分区:空旧月删除、有数据旧月保留、
|
||||
// 当月/未来月保留;用户访问日志分区同步清理。
|
||||
func TestDropEmptyPartitionsPostgres(t *testing.T) {
|
||||
dsn := strings.TrimSpace(os.Getenv("TEST_POSTGRES_DSN"))
|
||||
if dsn == "" {
|
||||
t.Skip("TEST_POSTGRES_DSN is not set")
|
||||
}
|
||||
|
||||
gdb, err := gorm.Open(postgres.Open(dsn), &gorm.Config{
|
||||
DisableForeignKeyConstraintWhenMigrating: true,
|
||||
Logger: logger.Default.LogMode(logger.Silent),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("open postgres: %v", err)
|
||||
}
|
||||
sqlDB, err := gdb.DB()
|
||||
if err != nil {
|
||||
t.Fatalf("sql db: %v", err)
|
||||
}
|
||||
sqlDB.SetMaxOpenConns(1)
|
||||
|
||||
schema := fmt.Sprintf("logstore_drop_partition_%d", time.Now().UnixNano())
|
||||
if !regexp.MustCompile(`^[a-z0-9_]+$`).MatchString(schema) {
|
||||
t.Fatalf("invalid schema: %s", schema)
|
||||
}
|
||||
if err := gdb.Exec(`CREATE SCHEMA "` + schema + `"`).Error; err != nil {
|
||||
t.Fatalf("create schema: %v", err)
|
||||
}
|
||||
if err := gdb.Exec(`SET search_path TO "` + schema + `"`).Error; err != nil {
|
||||
t.Fatalf("set search_path: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_ = gdb.Exec("SET search_path TO public").Error
|
||||
_ = gdb.Exec(`DROP SCHEMA IF EXISTS "` + schema + `" CASCADE`).Error
|
||||
_ = sqlDB.Close()
|
||||
})
|
||||
|
||||
for _, ddl := range []string{postgresNodeAccessLogsDDL, postgresUserAccessLogsDDL} {
|
||||
if err := gdb.Exec(ddl).Error; err != nil {
|
||||
t.Fatalf("create partitioned table: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
store := newGormStore(gdb)
|
||||
ua := newUserAccessLogGormStore(gdb)
|
||||
|
||||
// 预建 202601..202603 分区,仅 202602 有数据(节点+用户各 1 条),202601/202603 为空。
|
||||
if err := store.EnsurePartitions(ctx,
|
||||
time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC),
|
||||
time.Date(2026, 3, 20, 0, 0, 0, 0, time.UTC)); err != nil {
|
||||
t.Fatalf("EnsurePartitions: %v", err)
|
||||
}
|
||||
if err := store.BatchInsertNodeAccessLogs(ctx, []analyticsmodel.NodeAccessLog{
|
||||
{ID: 1, NodeID: "n1", LoggedAt: time.Date(2026, 2, 10, 0, 0, 0, 0, time.UTC), RemoteAddr: "1.1.1.1"},
|
||||
}); err != nil {
|
||||
t.Fatalf("insert node access log: %v", err)
|
||||
}
|
||||
if err := ua.BatchInsert(ctx, []analyticsmodel.UserAccessLog{
|
||||
{ID: 1, UserID: 101, Path: "/a", CreatedAt: time.Date(2026, 2, 11, 0, 0, 0, 0, time.UTC)},
|
||||
}); err != nil {
|
||||
t.Fatalf("insert user access log: %v", err)
|
||||
}
|
||||
|
||||
// before=2026-03:202601(空)应删,202602(有数据)与 202603(当月)保留。
|
||||
if err := store.DropEmptyPartitions(ctx, time.Date(2026, 3, 15, 0, 0, 0, 0, time.UTC)); err != nil {
|
||||
t.Fatalf("DropEmptyPartitions: %v", err)
|
||||
}
|
||||
|
||||
assertPartitions := func(parent string, want int64) {
|
||||
t.Helper()
|
||||
var n int64
|
||||
if err := gdb.Raw(
|
||||
"SELECT count(*) FROM pg_inherits WHERE inhparent = to_regclass(?)",
|
||||
parent,
|
||||
).Scan(&n).Error; err != nil {
|
||||
t.Fatalf("count partitions of %s: %v", parent, err)
|
||||
}
|
||||
if n != want {
|
||||
t.Fatalf("%s partitions = %d, want %d", parent, n, want)
|
||||
}
|
||||
}
|
||||
assertPartitions("of_node_access_logs", 2)
|
||||
assertPartitions("w_user_access_logs", 2)
|
||||
|
||||
// 数据未受影响。
|
||||
var nodeCount, userCount int64
|
||||
if err := gdb.Model(&analyticsmodel.NodeAccessLog{}).Count(&nodeCount).Error; err != nil {
|
||||
t.Fatalf("count node access logs: %v", err)
|
||||
}
|
||||
if err := gdb.Model(&analyticsmodel.UserAccessLog{}).Count(&userCount).Error; err != nil {
|
||||
t.Fatalf("count user access logs: %v", err)
|
||||
}
|
||||
if nodeCount != 1 || userCount != 1 {
|
||||
t.Fatalf("data counts = (%d, %d), want (1, 1)", nodeCount, userCount)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDropExpiredPartitionsPostgres 需要 TEST_POSTGRES_DSN(未设置时跳过):
|
||||
// 验证直接删除完全早于 cutoff 月份的整月分区:早于 cutoff 月的分区(含其中全部数据)被整表 DROP、
|
||||
// 边界月分区保留且数据仍在;重复调用幂等;w_user_access_logs 分区不受影响(无 retention 清理)。
|
||||
func TestDropExpiredPartitionsPostgres(t *testing.T) {
|
||||
dsn := strings.TrimSpace(os.Getenv("TEST_POSTGRES_DSN"))
|
||||
if dsn == "" {
|
||||
t.Skip("TEST_POSTGRES_DSN is not set")
|
||||
}
|
||||
|
||||
gdb, err := gorm.Open(postgres.Open(dsn), &gorm.Config{
|
||||
DisableForeignKeyConstraintWhenMigrating: true,
|
||||
Logger: logger.Default.LogMode(logger.Silent),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("open postgres: %v", err)
|
||||
}
|
||||
sqlDB, err := gdb.DB()
|
||||
if err != nil {
|
||||
t.Fatalf("sql db: %v", err)
|
||||
}
|
||||
sqlDB.SetMaxOpenConns(1)
|
||||
|
||||
schema := fmt.Sprintf("logstore_drop_expired_%d", time.Now().UnixNano())
|
||||
if !regexp.MustCompile(`^[a-z0-9_]+$`).MatchString(schema) {
|
||||
t.Fatalf("invalid schema: %s", schema)
|
||||
}
|
||||
if err := gdb.Exec(`CREATE SCHEMA "` + schema + `"`).Error; err != nil {
|
||||
t.Fatalf("create schema: %v", err)
|
||||
}
|
||||
if err := gdb.Exec(`SET search_path TO "` + schema + `"`).Error; err != nil {
|
||||
t.Fatalf("set search_path: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_ = gdb.Exec("SET search_path TO public").Error
|
||||
_ = gdb.Exec(`DROP SCHEMA IF EXISTS "` + schema + `" CASCADE`).Error
|
||||
_ = sqlDB.Close()
|
||||
})
|
||||
|
||||
for _, ddl := range []string{postgresNodeAccessLogsDDL, postgresUserAccessLogsDDL} {
|
||||
if err := gdb.Exec(ddl).Error; err != nil {
|
||||
t.Fatalf("create partitioned table: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
store := newGormStore(gdb)
|
||||
ua := newUserAccessLogGormStore(gdb)
|
||||
|
||||
// 预建 202601..202604 分区;1/3 月有数据、2/4 月为空。
|
||||
if err := store.EnsurePartitions(ctx,
|
||||
time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC),
|
||||
time.Date(2026, 4, 20, 0, 0, 0, 0, time.UTC)); err != nil {
|
||||
t.Fatalf("EnsurePartitions: %v", err)
|
||||
}
|
||||
if err := store.BatchInsertNodeAccessLogs(ctx, []analyticsmodel.NodeAccessLog{
|
||||
{ID: 1, NodeID: "n1", LoggedAt: time.Date(2026, 1, 15, 0, 0, 0, 0, time.UTC), RemoteAddr: "1.1.1.1"},
|
||||
{ID: 2, NodeID: "n1", LoggedAt: time.Date(2026, 1, 20, 0, 0, 0, 0, time.UTC), RemoteAddr: "1.1.1.2"},
|
||||
{ID: 3, NodeID: "n2", LoggedAt: time.Date(2026, 3, 5, 0, 0, 0, 0, time.UTC), RemoteAddr: "3.3.3.3"},
|
||||
{ID: 4, NodeID: "n2", LoggedAt: time.Date(2026, 3, 18, 0, 0, 0, 0, time.UTC), RemoteAddr: "3.3.3.4"},
|
||||
}); err != nil {
|
||||
t.Fatalf("insert node access logs: %v", err)
|
||||
}
|
||||
if err := ua.BatchInsert(ctx, []analyticsmodel.UserAccessLog{
|
||||
{ID: 1, UserID: 101, Path: "/a", CreatedAt: time.Date(2026, 1, 16, 0, 0, 0, 0, time.UTC)},
|
||||
}); err != nil {
|
||||
t.Fatalf("insert user access log: %v", err)
|
||||
}
|
||||
|
||||
// cutoff=2026-03-10:分区月份早于 2026-03 的(202601、202602)整表 DROP;
|
||||
// 202603(边界月,可能含未过期数据)与 202604(未来月)保留。
|
||||
if err := store.DropExpiredPartitions(ctx, time.Date(2026, 3, 10, 0, 0, 0, 0, time.UTC)); err != nil {
|
||||
t.Fatalf("DropExpiredPartitions: %v", err)
|
||||
}
|
||||
// 幂等:重复调用不报错、不额外删除。
|
||||
if err := store.DropExpiredPartitions(ctx, time.Date(2026, 3, 10, 0, 0, 0, 0, time.UTC)); err != nil {
|
||||
t.Fatalf("DropExpiredPartitions idempotent: %v", err)
|
||||
}
|
||||
|
||||
assertPartitions := func(parent string, want int64) {
|
||||
t.Helper()
|
||||
var n int64
|
||||
if err := gdb.Raw(
|
||||
"SELECT count(*) FROM pg_inherits WHERE inhparent = to_regclass(?)",
|
||||
parent,
|
||||
).Scan(&n).Error; err != nil {
|
||||
t.Fatalf("count partitions of %s: %v", parent, err)
|
||||
}
|
||||
if n != want {
|
||||
t.Fatalf("%s partitions = %d, want %d", parent, n, want)
|
||||
}
|
||||
}
|
||||
// of_node_access_logs 只剩边界月+未来月 2 个分区;w_user_access_logs 不受影响(仍 4 个)。
|
||||
assertPartitions("of_node_access_logs", 2)
|
||||
assertPartitions("w_user_access_logs", 4)
|
||||
|
||||
// 202601/202602 分区被整表 DROP:1 月数据随之消失,3 月数据保留。
|
||||
var nodeCount int64
|
||||
if err := gdb.Model(&analyticsmodel.NodeAccessLog{}).Count(&nodeCount).Error; err != nil {
|
||||
t.Fatalf("count node access logs: %v", err)
|
||||
}
|
||||
if nodeCount != 2 {
|
||||
t.Fatalf("node access log count = %d, want 2(仅剩 3 月数据)", nodeCount)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDropExpiredPartitionsTimezoneSafety 需要 TEST_POSTGRES_DSN(未设置时跳过):
|
||||
// 覆盖本地时区偏移下 DropExpiredPartitions 的时区安全性:cutoff 为 UTC+8 本地时刻
|
||||
// (其实刻 = 2026-02-28T21:00Z),名称月份早于 cutoff 月但分区内仍含保留期行的
|
||||
// 202602 不得被误删(旧实现按本地月份取 cutoffMonth=2026-03 会整表 DROP 丢数据);
|
||||
// 完全过期的 202601 正常整表 DROP;保留期行仍可查询到。
|
||||
func TestDropExpiredPartitionsTimezoneSafety(t *testing.T) {
|
||||
dsn := strings.TrimSpace(os.Getenv("TEST_POSTGRES_DSN"))
|
||||
if dsn == "" {
|
||||
t.Skip("TEST_POSTGRES_DSN is not set")
|
||||
}
|
||||
|
||||
gdb, err := gorm.Open(postgres.Open(dsn), &gorm.Config{
|
||||
DisableForeignKeyConstraintWhenMigrating: true,
|
||||
Logger: logger.Default.LogMode(logger.Silent),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("open postgres: %v", err)
|
||||
}
|
||||
sqlDB, err := gdb.DB()
|
||||
if err != nil {
|
||||
t.Fatalf("sql db: %v", err)
|
||||
}
|
||||
sqlDB.SetMaxOpenConns(1)
|
||||
|
||||
schema := fmt.Sprintf("logstore_drop_expired_tz_%d", time.Now().UnixNano())
|
||||
if !regexp.MustCompile(`^[a-z0-9_]+$`).MatchString(schema) {
|
||||
t.Fatalf("invalid schema: %s", schema)
|
||||
}
|
||||
if err := gdb.Exec(`CREATE SCHEMA "` + schema + `"`).Error; err != nil {
|
||||
t.Fatalf("create schema: %v", err)
|
||||
}
|
||||
if err := gdb.Exec(`SET search_path TO "` + schema + `"`).Error; err != nil {
|
||||
t.Fatalf("set search_path: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_ = gdb.Exec("SET search_path TO public").Error
|
||||
_ = gdb.Exec(`DROP SCHEMA IF EXISTS "` + schema + `" CASCADE`).Error
|
||||
_ = sqlDB.Close()
|
||||
})
|
||||
|
||||
for _, ddl := range []string{postgresNodeAccessLogsDDL, postgresUserAccessLogsDDL} {
|
||||
if err := gdb.Exec(ddl).Error; err != nil {
|
||||
t.Fatalf("create partitioned table: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
store := newGormStore(gdb)
|
||||
|
||||
// 预建 202601..202602 分区。
|
||||
if err := store.EnsurePartitions(ctx,
|
||||
time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC),
|
||||
time.Date(2026, 2, 20, 0, 0, 0, 0, time.UTC)); err != nil {
|
||||
t.Fatalf("EnsurePartitions: %v", err)
|
||||
}
|
||||
// 202601 仅含完全过期行;202602 含一条过期行(2026-02-10)与一条保留期行
|
||||
// (2026-02-28T21:00Z,恰等于 cutoff 其实刻,>= 语义下必须保留)。
|
||||
if err := store.BatchInsertNodeAccessLogs(ctx, []analyticsmodel.NodeAccessLog{
|
||||
{ID: 1, NodeID: "n1", LoggedAt: time.Date(2026, 1, 15, 0, 0, 0, 0, time.UTC), RemoteAddr: "1.1.1.1"},
|
||||
{ID: 2, NodeID: "n1", LoggedAt: time.Date(2026, 2, 10, 0, 0, 0, 0, time.UTC), RemoteAddr: "2.2.2.2"},
|
||||
{ID: 3, NodeID: "n1", LoggedAt: time.Date(2026, 2, 28, 21, 0, 0, 0, time.UTC), RemoteAddr: "3.3.3.3"},
|
||||
}); err != nil {
|
||||
t.Fatalf("insert node access logs: %v", err)
|
||||
}
|
||||
|
||||
// cutoff 为 UTC+8 本地时刻 2026-03-01 05:00,其实刻 = 2026-02-28T21:00Z:
|
||||
// 旧实现按本地月份取 cutoffMonth=2026-03 会把 202602 误判为完全过期整表 DROP。
|
||||
cutoff := time.Date(2026, 3, 1, 5, 0, 0, 0, time.FixedZone("UTC+8", 8*3600))
|
||||
if err := store.DropExpiredPartitions(ctx, cutoff); err != nil {
|
||||
t.Fatalf("DropExpiredPartitions: %v", err)
|
||||
}
|
||||
|
||||
assertPartitions := func(parent string, want int64) {
|
||||
t.Helper()
|
||||
var n int64
|
||||
if err := gdb.Raw(
|
||||
"SELECT count(*) FROM pg_inherits WHERE inhparent = to_regclass(?)",
|
||||
parent,
|
||||
).Scan(&n).Error; err != nil {
|
||||
t.Fatalf("count partitions of %s: %v", parent, err)
|
||||
}
|
||||
if n != want {
|
||||
t.Fatalf("%s partitions = %d, want %d", parent, n, want)
|
||||
}
|
||||
}
|
||||
// 202601 已整表 DROP,202602 保留;w_user_access_logs 不受影响(仍 2 个)。
|
||||
assertPartitions("of_node_access_logs", 1)
|
||||
assertPartitions("w_user_access_logs", 2)
|
||||
|
||||
// 202601 数据随之消失,202602 内保留期行(2026-02-28T21:00Z)仍可查询到。
|
||||
var nodeCount int64
|
||||
if err := gdb.Model(&analyticsmodel.NodeAccessLog{}).Count(&nodeCount).Error; err != nil {
|
||||
t.Fatalf("count node access logs: %v", err)
|
||||
}
|
||||
if nodeCount != 2 {
|
||||
t.Fatalf("node access log count = %d, want 2(仅剩 202602 两行)", nodeCount)
|
||||
}
|
||||
var retained int64
|
||||
if err := gdb.Raw(
|
||||
"SELECT count(*) FROM of_node_access_logs WHERE logged_at >= ?",
|
||||
cutoff.UTC(),
|
||||
).Scan(&retained).Error; err != nil {
|
||||
t.Fatalf("count retained rows: %v", err)
|
||||
}
|
||||
if retained != 1 {
|
||||
t.Fatalf("retained rows (logged_at >= cutoff) = %d, want 1", retained)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBatchInsertGeneratesIDsPostgres 回归:PG 日志表 id BIGINT NOT NULL 且无默认值;
|
||||
// GORM 把零值 uint64 主键视为自增并省略 id 列,直接插入会报 23502 not-null 违例。
|
||||
// 验证 6 张日志表 BatchInsert* 为零 ID 行生成雪花 ID 后正常落库(修复前本测试失败)。
|
||||
func TestBatchInsertGeneratesIDsPostgres(t *testing.T) {
|
||||
dsn := strings.TrimSpace(os.Getenv("TEST_POSTGRES_DSN"))
|
||||
if dsn == "" {
|
||||
t.Skip("TEST_POSTGRES_DSN is not set")
|
||||
}
|
||||
|
||||
gdb, err := gorm.Open(postgres.Open(dsn), &gorm.Config{
|
||||
DisableForeignKeyConstraintWhenMigrating: true,
|
||||
Logger: logger.Default.LogMode(logger.Silent),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("open postgres: %v", err)
|
||||
}
|
||||
sqlDB, err := gdb.DB()
|
||||
if err != nil {
|
||||
t.Fatalf("sql db: %v", err)
|
||||
}
|
||||
sqlDB.SetMaxOpenConns(1)
|
||||
|
||||
schema := fmt.Sprintf("logstore_ids_%d", time.Now().UnixNano())
|
||||
if !regexp.MustCompile(`^[a-z0-9_]+$`).MatchString(schema) {
|
||||
t.Fatalf("invalid schema: %s", schema)
|
||||
}
|
||||
if err := gdb.Exec(`CREATE SCHEMA "` + schema + `"`).Error; err != nil {
|
||||
t.Fatalf("create schema: %v", err)
|
||||
}
|
||||
if err := gdb.Exec(`SET search_path TO "` + schema + `"`).Error; err != nil {
|
||||
t.Fatalf("set search_path: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_ = gdb.Exec("SET search_path TO public").Error
|
||||
_ = gdb.Exec(`DROP SCHEMA IF EXISTS "` + schema + `" CASCADE`).Error
|
||||
_ = sqlDB.Close()
|
||||
})
|
||||
|
||||
for _, ddl := range []string{
|
||||
postgresNodeAccessLogsDDL,
|
||||
postgresUserAccessLogsDDL,
|
||||
postgresMetricSnapshotsDDL,
|
||||
postgresEdgeHealthDDL,
|
||||
postgresObsFrpsDDL,
|
||||
postgresObsFrpcDDL,
|
||||
} {
|
||||
if err := gdb.Exec(ddl).Error; err != nil {
|
||||
t.Fatalf("create table: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
ResetForTest()
|
||||
SetConfigReader(func(_ context.Context, _ string) (string, error) { return "", nil })
|
||||
defer ResetForTest()
|
||||
|
||||
ctx := context.Background()
|
||||
store := newGormStore(gdb)
|
||||
ua := newUserAccessLogGormStore(gdb)
|
||||
|
||||
now := time.Now().UTC()
|
||||
if err := store.EnsurePartitions(ctx, now, now.AddDate(0, 1, 0)); err != nil {
|
||||
t.Fatalf("EnsurePartitions: %v", err)
|
||||
}
|
||||
|
||||
nodeRows := []analyticsmodel.NodeAccessLog{
|
||||
{NodeID: "n1", LoggedAt: now, RemoteAddr: "1.1.1.1", StatusCode: 200},
|
||||
{NodeID: "n1", LoggedAt: now.Add(time.Second), RemoteAddr: "2.2.2.2", StatusCode: 500},
|
||||
}
|
||||
if err := store.BatchInsertNodeAccessLogs(ctx, nodeRows); err != nil {
|
||||
t.Fatalf("insert node access logs with zero ids: %v", err)
|
||||
}
|
||||
if nodeRows[0].ID == 0 || nodeRows[1].ID == 0 || nodeRows[0].ID == nodeRows[1].ID {
|
||||
t.Fatalf("node access log ids not generated: %+v", nodeRows)
|
||||
}
|
||||
|
||||
metricRows := []analyticsmodel.NodeMetricSnapshot{
|
||||
{NodeID: "n1", CapturedAt: now},
|
||||
{NodeID: "n2", CapturedAt: now},
|
||||
}
|
||||
if err := store.BatchInsertNodeMetricSnapshots(ctx, metricRows); err != nil {
|
||||
t.Fatalf("insert metric snapshots with zero ids: %v", err)
|
||||
}
|
||||
if metricRows[0].ID == 0 || metricRows[1].ID == 0 || metricRows[0].ID == metricRows[1].ID {
|
||||
t.Fatalf("metric snapshot ids not generated: %+v", metricRows)
|
||||
}
|
||||
|
||||
edgeRows := []analyticsmodel.NodeEdgeHealth{
|
||||
{NodeID: "n1", CapturedAt: now, Status: "ok"},
|
||||
{NodeID: "n2", CapturedAt: now, Status: "ok"},
|
||||
}
|
||||
if err := store.BatchInsertNodeEdgeHealth(ctx, edgeRows); err != nil {
|
||||
t.Fatalf("insert edge health with zero ids: %v", err)
|
||||
}
|
||||
if edgeRows[0].ID == 0 || edgeRows[1].ID == 0 || edgeRows[0].ID == edgeRows[1].ID {
|
||||
t.Fatalf("edge health ids not generated: %+v", edgeRows)
|
||||
}
|
||||
|
||||
frpsRows := []analyticsmodel.NodeObsFrps{
|
||||
{NodeID: "n1", CapturedAt: now, FrpsConnections: 1},
|
||||
{NodeID: "n2", CapturedAt: now, FrpsConnections: 2},
|
||||
}
|
||||
if err := store.BatchInsertNodeObsFrps(ctx, frpsRows); err != nil {
|
||||
t.Fatalf("insert obs frps with zero ids: %v", err)
|
||||
}
|
||||
if frpsRows[0].ID == 0 || frpsRows[1].ID == 0 || frpsRows[0].ID == frpsRows[1].ID {
|
||||
t.Fatalf("obs frps ids not generated: %+v", frpsRows)
|
||||
}
|
||||
|
||||
frpcRows := []analyticsmodel.NodeObsFrpc{
|
||||
{NodeID: "n1", CapturedAt: now, TunnelStatus: "online"},
|
||||
{NodeID: "n2", CapturedAt: now, TunnelStatus: "online"},
|
||||
}
|
||||
if err := store.BatchInsertNodeObsFrpc(ctx, frpcRows); err != nil {
|
||||
t.Fatalf("insert obs frpc with zero ids: %v", err)
|
||||
}
|
||||
if frpcRows[0].ID == 0 || frpcRows[1].ID == 0 || frpcRows[0].ID == frpcRows[1].ID {
|
||||
t.Fatalf("obs frpc ids not generated: %+v", frpcRows)
|
||||
}
|
||||
|
||||
userRows := []analyticsmodel.UserAccessLog{
|
||||
{UserID: 101, Path: "/a", CreatedAt: now},
|
||||
{UserID: 102, Path: "/b", CreatedAt: now},
|
||||
}
|
||||
if err := ua.BatchInsert(ctx, userRows); err != nil {
|
||||
t.Fatalf("insert user access logs with zero ids: %v", err)
|
||||
}
|
||||
if userRows[0].ID == 0 || userRows[1].ID == 0 || userRows[0].ID == userRows[1].ID {
|
||||
t.Fatalf("user access log ids not generated: %+v", userRows)
|
||||
}
|
||||
|
||||
expect := []struct {
|
||||
name string
|
||||
model any
|
||||
want int64
|
||||
}{
|
||||
{"of_node_access_logs", &analyticsmodel.NodeAccessLog{}, 2},
|
||||
{"of_node_metric_snapshots", &analyticsmodel.NodeMetricSnapshot{}, 2},
|
||||
{"of_node_edge_health", &analyticsmodel.NodeEdgeHealth{}, 2},
|
||||
{"of_node_obs_frps", &analyticsmodel.NodeObsFrps{}, 2},
|
||||
{"of_node_obs_frpc", &analyticsmodel.NodeObsFrpc{}, 2},
|
||||
{"w_user_access_logs", &analyticsmodel.UserAccessLog{}, 2},
|
||||
}
|
||||
for _, e := range expect {
|
||||
var got int64
|
||||
if err := gdb.Model(e.model).Count(&got).Error; err != nil {
|
||||
t.Fatalf("count %s: %v", e.name, err)
|
||||
}
|
||||
if got != e.want {
|
||||
t.Fatalf("%s count = %d, want %d", e.name, got, e.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// postgresNodeAccessLogsDDL 与 goose/postgres/202608080001_create_log_tables.sql 对齐。
|
||||
const postgresNodeAccessLogsDDL = `
|
||||
CREATE TABLE IF NOT EXISTS of_node_access_logs (
|
||||
id BIGINT NOT NULL,
|
||||
node_id VARCHAR(64) NOT NULL DEFAULT '',
|
||||
logged_at TIMESTAMPTZ NOT NULL,
|
||||
remote_addr VARCHAR(128) NOT NULL DEFAULT '',
|
||||
region VARCHAR(128) NOT NULL DEFAULT '',
|
||||
host VARCHAR(255) NOT NULL DEFAULT '',
|
||||
path VARCHAR(2048) NOT NULL DEFAULT '',
|
||||
user_agent TEXT NOT NULL DEFAULT '',
|
||||
cache_status VARCHAR(64) NOT NULL DEFAULT '',
|
||||
status_code INTEGER NOT NULL DEFAULT 0,
|
||||
bytes_sent BIGINT NOT NULL DEFAULT 0,
|
||||
request_length BIGINT NOT NULL DEFAULT 0,
|
||||
request_time_ms INTEGER NOT NULL DEFAULT 0,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (id, logged_at)
|
||||
) PARTITION BY RANGE (logged_at)`
|
||||
|
||||
// postgresUserAccessLogsDDL 与 goose/postgres/202608080001_create_log_tables.sql 对齐。
|
||||
const postgresUserAccessLogsDDL = `
|
||||
CREATE TABLE IF NOT EXISTS w_user_access_logs (
|
||||
id BIGINT NOT NULL,
|
||||
user_id BIGINT NOT NULL DEFAULT 0,
|
||||
path VARCHAR(2048) NOT NULL DEFAULT '',
|
||||
method VARCHAR(16) NOT NULL DEFAULT '',
|
||||
ip VARCHAR(128) NOT NULL DEFAULT '',
|
||||
user_agent TEXT NOT NULL DEFAULT '',
|
||||
headers TEXT NOT NULL DEFAULT '',
|
||||
status INTEGER NOT NULL DEFAULT 0,
|
||||
latency BIGINT NOT NULL DEFAULT 0,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (id, created_at)
|
||||
) PARTITION BY RANGE (created_at)`
|
||||
|
||||
// postgresMetricSnapshotsDDL / postgresEdgeHealthDDL / postgresObsFrpsDDL / postgresObsFrpcDDL
|
||||
// 与 goose/postgres/202608080001_create_log_tables.sql 对齐(普通表,无分区)。
|
||||
const postgresMetricSnapshotsDDL = `
|
||||
CREATE TABLE IF NOT EXISTS of_node_metric_snapshots (
|
||||
id BIGINT NOT NULL PRIMARY KEY,
|
||||
node_id VARCHAR(64) NOT NULL DEFAULT '',
|
||||
captured_at TIMESTAMPTZ NOT NULL,
|
||||
cpu_usage_percent DOUBLE PRECISION NOT NULL DEFAULT 0,
|
||||
memory_used_bytes BIGINT NOT NULL DEFAULT 0,
|
||||
memory_total_bytes BIGINT NOT NULL DEFAULT 0,
|
||||
storage_used_bytes BIGINT NOT NULL DEFAULT 0,
|
||||
storage_total_bytes BIGINT NOT NULL DEFAULT 0,
|
||||
disk_read_bytes BIGINT NOT NULL DEFAULT 0,
|
||||
disk_write_bytes BIGINT NOT NULL DEFAULT 0,
|
||||
network_rx_bytes BIGINT NOT NULL DEFAULT 0,
|
||||
network_tx_bytes BIGINT NOT NULL DEFAULT 0,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
)`
|
||||
|
||||
const postgresEdgeHealthDDL = `
|
||||
CREATE TABLE IF NOT EXISTS of_node_edge_health (
|
||||
id BIGINT NOT NULL PRIMARY KEY,
|
||||
node_id VARCHAR(64) NOT NULL DEFAULT '',
|
||||
captured_at TIMESTAMPTZ NOT NULL,
|
||||
status VARCHAR(64) NOT NULL DEFAULT '',
|
||||
connections BIGINT NOT NULL DEFAULT 0,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
)`
|
||||
|
||||
const postgresObsFrpsDDL = `
|
||||
CREATE TABLE IF NOT EXISTS of_node_obs_frps (
|
||||
id BIGINT NOT NULL PRIMARY KEY,
|
||||
node_id VARCHAR(64) NOT NULL DEFAULT '',
|
||||
captured_at TIMESTAMPTZ NOT NULL,
|
||||
frps_connections INTEGER NOT NULL DEFAULT 0,
|
||||
frps_proxy_count INTEGER NOT NULL DEFAULT 0,
|
||||
frps_client_count INTEGER NOT NULL DEFAULT 0,
|
||||
frps_proxies TEXT NOT NULL DEFAULT '',
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
)`
|
||||
|
||||
const postgresObsFrpcDDL = `
|
||||
CREATE TABLE IF NOT EXISTS of_node_obs_frpc (
|
||||
id BIGINT NOT NULL PRIMARY KEY,
|
||||
node_id VARCHAR(64) NOT NULL DEFAULT '',
|
||||
captured_at TIMESTAMPTZ NOT NULL,
|
||||
tunnel_status VARCHAR(16) NOT NULL DEFAULT '',
|
||||
connected_relays_count INTEGER NOT NULL DEFAULT 0,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
)`
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,217 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package logstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/model"
|
||||
"Wavelet/openflare/plugins/server/kernel/runtimeconfig"
|
||||
"Wavelet/pkg/logger"
|
||||
db "Wavelet/plugins/infra/database"
|
||||
)
|
||||
|
||||
// logDatabaseKey / logMigrationKey 对应 model.ConfigKeyLogDatabase / ConfigKeyLogDBMigration。
|
||||
const (
|
||||
logDatabaseKey = model.ConfigKeyLogDatabase
|
||||
logMigrationKey = model.ConfigKeyLogDBMigration
|
||||
)
|
||||
|
||||
// 日志库名常量(与 model 配置值一致,集中避免散落字符串字面量)。
|
||||
const (
|
||||
dbNamePostgres = "postgres"
|
||||
dbNameSQLite = "sqlite"
|
||||
dbNameClickHouse = "clickhouse"
|
||||
)
|
||||
|
||||
// errConfigReaderNotWired 表示 config reader 尚未注入(首启/测试场景按 seed 规则兜底)。
|
||||
var errConfigReaderNotWired = errors.New("logstore: config reader not wired")
|
||||
|
||||
// ConfigReader 读取系统配置字符串值,由 bootstrap 注入(避免 logstore ↔ repository 循环依赖)。
|
||||
type ConfigReader func(ctx context.Context, key string) (string, error)
|
||||
|
||||
const resolveCacheTTL = 1 * time.Second
|
||||
|
||||
var (
|
||||
configReader ConfigReader
|
||||
|
||||
storeMu sync.RWMutex
|
||||
active *Store
|
||||
activeDB string
|
||||
lastResolveDB string
|
||||
lastResolveTime time.Time
|
||||
)
|
||||
|
||||
// SetConfigReader 注入系统配置读取函数(bootstrap 调用,测试可注入内存实现)。
|
||||
func SetConfigReader(fn ConfigReader) { configReader = fn }
|
||||
|
||||
func getConfig(ctx context.Context, key string) (string, error) {
|
||||
if configReader == nil {
|
||||
return "", errConfigReaderNotWired
|
||||
}
|
||||
return configReader(ctx, key)
|
||||
}
|
||||
|
||||
// Active 返回当前生效的日志库 Store。按 log_database 系统配置惰性解析并缓存,
|
||||
// 配置更新(含迁移任务翻转)后自动重建。
|
||||
func Active(ctx context.Context) (*Store, error) {
|
||||
current, err := resolveDatabase(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
storeMu.RLock()
|
||||
if active != nil && activeDB == current {
|
||||
s := active
|
||||
storeMu.RUnlock()
|
||||
return s, nil
|
||||
}
|
||||
storeMu.RUnlock()
|
||||
|
||||
storeMu.Lock()
|
||||
defer storeMu.Unlock()
|
||||
if active != nil && activeDB == current {
|
||||
return active, nil
|
||||
}
|
||||
s, err := buildStore(ctx, current, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
active = s
|
||||
activeDB = current
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// Build 直接按目标构造 store(不经 Active 缓存)。
|
||||
func Build(ctx context.Context, database string) (*Store, error) {
|
||||
return buildStore(ctx, database, false)
|
||||
}
|
||||
|
||||
// BuildForMigration 构造迁移目标 store:与 Build 相同但不做冻结检查
|
||||
// (迁移期间 log_db_migration=migrating 已冻结源库写入,目标库的清空/复制写入必须放行)。
|
||||
func BuildForMigration(ctx context.Context, database string) (*Store, error) {
|
||||
return buildStore(ctx, database, true)
|
||||
}
|
||||
|
||||
// buildStore 按目标构造实现。skipFreeze 为 true 时该 store 跳过冻结检查
|
||||
// (仅迁移任务的目标 store 使用)。gorm 分支 UserAccessLogs 用独立包装类型
|
||||
// (gormLogStore 已占用 List/Count 方法名,无法再实现 UserAccessLogStore)。
|
||||
func buildStore(ctx context.Context, database string, skipFreeze bool) (*Store, error) {
|
||||
switch database {
|
||||
case dbNameClickHouse:
|
||||
ch := newClickHouseStore()
|
||||
ch.skipFreeze = skipFreeze
|
||||
ual := newClickHouseUserAccessLogStore()
|
||||
ual.skipFreeze = skipFreeze
|
||||
return &Store{
|
||||
AccessLogs: ch,
|
||||
Observability: ch,
|
||||
UserAccessLogs: ual,
|
||||
Status: ch,
|
||||
}, nil
|
||||
case dbNamePostgres, dbNameSQLite:
|
||||
gdb := db.DB(ctx)
|
||||
g := newGormStore(gdb)
|
||||
g.skipFreeze = skipFreeze
|
||||
ual := newUserAccessLogGormStore(gdb)
|
||||
ual.skipFreeze = skipFreeze
|
||||
return &Store{
|
||||
AccessLogs: g,
|
||||
Observability: g,
|
||||
UserAccessLogs: ual,
|
||||
Status: g,
|
||||
}, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported log database: %s", database)
|
||||
}
|
||||
}
|
||||
|
||||
// Migrating 返回日志库是否处于迁移冻结状态。
|
||||
func Migrating(ctx context.Context) bool {
|
||||
v, err := getConfig(ctx, logMigrationKey)
|
||||
if err != nil {
|
||||
if !errors.Is(err, errConfigReaderNotWired) {
|
||||
logger.ErrorF(ctx, "read log migration config failed: %v", err)
|
||||
}
|
||||
return false
|
||||
}
|
||||
return v == "migrating"
|
||||
}
|
||||
|
||||
// Init 在 bootstrap 阶段预热一次激活 store(幂等,失败不致命——首次使用时再解析),
|
||||
// 并兜底预建「当前月 + 未来 2 个月」分区:进程停机跨月边界、重启后每日 cleanup 之前
|
||||
// 首次写入不会报 "no partition of relation found"(CH/SQLite 分支 EnsurePartitions 为 no-op)。
|
||||
func Init(ctx context.Context) {
|
||||
s, err := Active(ctx)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
if err := s.AccessLogs.EnsurePartitions(ctx, now, now.AddDate(0, partitionLeadMonths, 0)); err != nil {
|
||||
logger.WarnF(ctx, "logstore: ensure startup partitions failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// InvalidateCache 清空日志库解析缓存(在修改 log_database 配置后显式调用)。
|
||||
func InvalidateCache() {
|
||||
storeMu.Lock()
|
||||
defer storeMu.Unlock()
|
||||
lastResolveTime = time.Time{}
|
||||
lastResolveDB = ""
|
||||
}
|
||||
|
||||
// ResetForTest 清空缓存的激活 store 与 config reader,便于测试注入。
|
||||
func ResetForTest() {
|
||||
storeMu.Lock()
|
||||
active = nil
|
||||
activeDB = ""
|
||||
lastResolveDB = ""
|
||||
lastResolveTime = time.Time{}
|
||||
storeMu.Unlock()
|
||||
configReader = nil
|
||||
}
|
||||
|
||||
// ActiveDatabase 返回当前日志主库名(postgres|sqlite|clickhouse)。
|
||||
func ActiveDatabase(ctx context.Context) (string, error) {
|
||||
return resolveDatabase(ctx)
|
||||
}
|
||||
|
||||
// resolveDatabase 读取 log_database:值缺失或 reader 未装配(首启)时按启动规则 seed;
|
||||
// 已装配 reader 的真实读取错误直接透出,避免把读失败当首次启动。
|
||||
func resolveDatabase(ctx context.Context) (string, error) {
|
||||
storeMu.RLock()
|
||||
if active != nil && time.Since(lastResolveTime) < resolveCacheTTL {
|
||||
db := lastResolveDB
|
||||
storeMu.RUnlock()
|
||||
return db, nil
|
||||
}
|
||||
storeMu.RUnlock()
|
||||
|
||||
v, err := getConfig(ctx, logDatabaseKey)
|
||||
if err != nil && !errors.Is(err, errConfigReaderNotWired) {
|
||||
return "", err
|
||||
}
|
||||
|
||||
resolved := v
|
||||
if resolved == "" {
|
||||
// 首次启动 seed:CH 启用 → clickhouse;否则随主库。
|
||||
resolved = dbNameSQLite
|
||||
if runtimeconfig.DatabaseEnabled() {
|
||||
resolved = dbNamePostgres
|
||||
}
|
||||
if runtimeconfig.ClickHouseEnabled() {
|
||||
resolved = dbNameClickHouse
|
||||
}
|
||||
}
|
||||
|
||||
storeMu.Lock()
|
||||
lastResolveDB = resolved
|
||||
lastResolveTime = time.Now()
|
||||
storeMu.Unlock()
|
||||
|
||||
return resolved, nil
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package logstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
|
||||
)
|
||||
|
||||
func TestMigratingReadsConfig(t *testing.T) {
|
||||
ResetForTest()
|
||||
SetConfigReader(func(_ context.Context, key string) (string, error) {
|
||||
if key == logMigrationKey {
|
||||
return "migrating", nil
|
||||
}
|
||||
return "", nil
|
||||
})
|
||||
if !Migrating(context.Background()) {
|
||||
t.Fatal("Migrating() = false, want true when key=migrating")
|
||||
}
|
||||
SetConfigReader(func(_ context.Context, key string) (string, error) {
|
||||
return "", nil
|
||||
})
|
||||
if Migrating(context.Background()) {
|
||||
t.Fatal("Migrating() = true, want false when key empty")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveDatabaseDefaults(t *testing.T) {
|
||||
ResetForTest()
|
||||
// 配置缺失(reader 返回空值)时按主库规则 seed(config.Config 默认值由既有测试基建决定)。
|
||||
SetConfigReader(func(_ context.Context, key string) (string, error) {
|
||||
return "", nil
|
||||
})
|
||||
got, err := resolveDatabase(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("resolveDatabase: %v", err)
|
||||
}
|
||||
if got != "postgres" && got != "sqlite" && got != "clickhouse" {
|
||||
t.Fatalf("unexpected default log database: %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveDatabaseSurfacesReadError(t *testing.T) {
|
||||
ResetForTest()
|
||||
wantErr := errors.New("boom")
|
||||
SetConfigReader(func(_ context.Context, key string) (string, error) {
|
||||
return "", wantErr
|
||||
})
|
||||
if _, err := resolveDatabase(context.Background()); !errors.Is(err, wantErr) {
|
||||
t.Fatalf("resolveDatabase error = %v, want %v", err, wantErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestActiveBuildsStore(t *testing.T) {
|
||||
ResetForTest()
|
||||
SetConfigReader(func(_ context.Context, key string) (string, error) {
|
||||
if key == logDatabaseKey {
|
||||
return "sqlite", nil
|
||||
}
|
||||
return "", nil
|
||||
})
|
||||
store, err := Active(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Active: %v", err)
|
||||
}
|
||||
if store == nil {
|
||||
t.Fatal("Active() returned nil store")
|
||||
}
|
||||
if store.AccessLogs == nil || store.Observability == nil || store.UserAccessLogs == nil || store.Status == nil {
|
||||
t.Fatalf("Active() store fields not fully wired: %+v", store)
|
||||
}
|
||||
// 再次调用应命中缓存。
|
||||
again, err := Active(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Active (cached): %v", err)
|
||||
}
|
||||
if again != store {
|
||||
t.Fatal("Active() did not return cached store")
|
||||
}
|
||||
}
|
||||
|
||||
// TestClickHouseUserAccessLogBatchInsertFreeze 覆盖 CH 用户访问日志 flush 的冻结检查:
|
||||
// 冻结期非空批次返回 ErrMigrating(在触碰 CH 连接之前),空批次直接成功。
|
||||
func TestClickHouseUserAccessLogBatchInsertFreeze(t *testing.T) {
|
||||
ResetForTest()
|
||||
SetConfigReader(func(_ context.Context, key string) (string, error) {
|
||||
if key == logMigrationKey {
|
||||
return "migrating", nil
|
||||
}
|
||||
return "", nil
|
||||
})
|
||||
defer ResetForTest()
|
||||
s := newClickHouseUserAccessLogStore()
|
||||
ctx := context.Background()
|
||||
now := time.Now()
|
||||
|
||||
if err := s.BatchInsert(ctx, []analyticsmodel.UserAccessLog{{UserID: 1, CreatedAt: now}}); !errors.Is(err, ErrMigrating) {
|
||||
t.Fatalf("BatchInsert during migration: want ErrMigrating, got %v", err)
|
||||
}
|
||||
if err := s.BatchInsert(ctx, nil); err != nil {
|
||||
t.Fatalf("BatchInsert empty batch: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,442 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/model"
|
||||
"Wavelet/openflare/plugins/server/kernel/repository/logstore"
|
||||
)
|
||||
|
||||
const (
|
||||
sortOrderAsc = "asc"
|
||||
secondsPerMinute = 60
|
||||
)
|
||||
|
||||
// ListOpenFlareAccessLogWAFIPAggregates returns per-IP aggregates for WAF automatic rules.
|
||||
func ListOpenFlareAccessLogWAFIPAggregates(ctx context.Context, query model.OpenFlareAccessLogQuery) ([]*model.OpenFlareAccessLogWAFIPAggregate, error) {
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows, err := s.AccessLogs.WAFIPAggregates(ctx, query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := make([]*model.OpenFlareAccessLogWAFIPAggregate, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
remoteAddr := strings.TrimSpace(row.RemoteAddr)
|
||||
if remoteAddr == "" {
|
||||
continue
|
||||
}
|
||||
statusCounts := make(map[int]int, len(row.StatusCounts))
|
||||
for code, count := range row.StatusCounts {
|
||||
statusCounts[code] = int(count)
|
||||
}
|
||||
result = append(result, &model.OpenFlareAccessLogWAFIPAggregate{
|
||||
RemoteAddr: remoteAddr,
|
||||
RequestCount: int(row.RequestCount),
|
||||
Status404Count: int(row.Status404Count),
|
||||
ClientErrorCount: int(row.ClientErrorCount),
|
||||
ServerErrorCount: int(row.ServerErrorCount),
|
||||
IPHostCount: int(row.IPHostCount),
|
||||
LastSeenEpoch: row.LastSeenEpoch,
|
||||
StatusCounts: statusCounts,
|
||||
})
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// InsertOpenFlareAccessLogsBatch inserts access log rows into the active log store.
|
||||
func InsertOpenFlareAccessLogsBatch(ctx context.Context, records []*model.OpenFlareAccessLog) error {
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.AccessLogs.InsertBatch(ctx, records)
|
||||
}
|
||||
|
||||
// ListOpenFlareAccessLogs lists access logs matching the query.
|
||||
func ListOpenFlareAccessLogs(ctx context.Context, query model.OpenFlareAccessLogQuery) ([]*model.OpenFlareAccessLog, error) {
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.AccessLogs.List(ctx, query)
|
||||
}
|
||||
|
||||
// CountOpenFlareAccessLogs counts access logs, distinct IPs, and total bytes sent matching the query.
|
||||
func CountOpenFlareAccessLogs(ctx context.Context, query model.OpenFlareAccessLogQuery) (int64, int64, int64, error) {
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return 0, 0, 0, err
|
||||
}
|
||||
return s.AccessLogs.Count(ctx, query)
|
||||
}
|
||||
|
||||
// TrafficSummaryOpenFlareAccessLogs returns window-level request/error/UV/bytes summary.
|
||||
func TrafficSummaryOpenFlareAccessLogs(ctx context.Context, query model.OpenFlareAccessLogQuery) (model.OpenFlareAccessLogTrafficSummary, error) {
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return model.OpenFlareAccessLogTrafficSummary{}, err
|
||||
}
|
||||
return s.AccessLogs.TrafficSummary(ctx, query)
|
||||
}
|
||||
|
||||
// ValueCountsOpenFlareAccessLogs groups logs by status_code, host, path, remote_addr, or user_agent.
|
||||
func ValueCountsOpenFlareAccessLogs(ctx context.Context, query model.OpenFlareAccessLogQuery, column string, limit int) ([]model.OpenFlareAccessLogValueCount, error) {
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.AccessLogs.ValueCounts(ctx, query, column, limit)
|
||||
}
|
||||
|
||||
// NodeAggregatesOpenFlareAccessLogs returns per-node request/error/UV for the window.
|
||||
func NodeAggregatesOpenFlareAccessLogs(ctx context.Context, query model.OpenFlareAccessLogQuery) ([]model.OpenFlareAccessLogNodeAggregate, error) {
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.AccessLogs.NodeAggregates(ctx, query)
|
||||
}
|
||||
|
||||
// ListOpenFlareAccessLogRegionCounts returns region counts for access logs.
|
||||
func ListOpenFlareAccessLogRegionCounts(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareAccessLogRegionCount, error) {
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.AccessLogs.RegionCounts(ctx, nodeID, since, limit)
|
||||
}
|
||||
|
||||
// ListOpenFlareAccessLogBuckets lists folded access log buckets.
|
||||
func ListOpenFlareAccessLogBuckets(ctx context.Context, query model.OpenFlareAccessLogBucketQuery) ([]*model.OpenFlareAccessLogBucketRow, error) {
|
||||
return buildOpenFlareAccessLogBucketRows(ctx, query)
|
||||
}
|
||||
|
||||
// CountOpenFlareAccessLogBuckets counts folded access log buckets.
|
||||
func CountOpenFlareAccessLogBuckets(ctx context.Context, query model.OpenFlareAccessLogBucketQuery) (int64, error) {
|
||||
filter := openFlareAccessLogQueryFromBucket(query)
|
||||
bucketSeconds := int64(query.FoldMinutes * secondsPerMinute)
|
||||
if bucketSeconds <= 0 {
|
||||
bucketSeconds = 180
|
||||
}
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return s.AccessLogs.CountBuckets(ctx, filter, bucketSeconds)
|
||||
}
|
||||
|
||||
// ListOpenFlareAccessLogBucketIPs lists folded IP rows for a bucket window.
|
||||
func ListOpenFlareAccessLogBucketIPs(ctx context.Context, query model.OpenFlareAccessLogBucketIPQuery) ([]*model.OpenFlareAccessLogBucketIPRow, error) {
|
||||
rows, err := buildOpenFlareAccessLogBucketIPRows(ctx, query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
start, end := openFlareAccessLogPaginateBounds(len(rows), query.Page, query.PageSize)
|
||||
if start >= len(rows) {
|
||||
return []*model.OpenFlareAccessLogBucketIPRow{}, nil
|
||||
}
|
||||
return rows[start:end], nil
|
||||
}
|
||||
|
||||
// CountOpenFlareAccessLogBucketIPs counts folded IP rows for a bucket window.
|
||||
func CountOpenFlareAccessLogBucketIPs(ctx context.Context, query model.OpenFlareAccessLogBucketIPQuery) (int64, error) {
|
||||
rows, err := buildOpenFlareAccessLogBucketIPRows(ctx, query)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return int64(len(rows)), nil
|
||||
}
|
||||
|
||||
// ListOpenFlareAccessLogIPSummaries lists IP summaries.
|
||||
func ListOpenFlareAccessLogIPSummaries(ctx context.Context, query model.OpenFlareAccessLogIPSummaryQuery, recentSince time.Time) ([]*analyticsmodel.NodeAccessLogIPSummary, error) {
|
||||
return buildOpenFlareAccessLogIPSummaryRows(ctx, query, recentSince)
|
||||
}
|
||||
|
||||
// CountOpenFlareAccessLogIPSummaries counts IP summaries.
|
||||
func CountOpenFlareAccessLogIPSummaries(ctx context.Context, query model.OpenFlareAccessLogIPSummaryQuery) (int64, error) {
|
||||
filter := openFlareAccessLogQueryFromIPSummary(query)
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return s.AccessLogs.CountIPSummaries(ctx, filter)
|
||||
}
|
||||
|
||||
// ListOpenFlareAccessLogIPTrend lists IP trend points.
|
||||
func ListOpenFlareAccessLogIPTrend(ctx context.Context, query model.OpenFlareAccessLogIPTrendQuery) ([]*analyticsmodel.NodeAccessLogIPTrend, error) {
|
||||
remoteAddr := strings.TrimSpace(query.RemoteAddr)
|
||||
if remoteAddr == "" {
|
||||
return []*analyticsmodel.NodeAccessLogIPTrend{}, nil
|
||||
}
|
||||
filter := model.OpenFlareAccessLogQuery{
|
||||
NodeID: query.NodeID,
|
||||
RemoteAddr: remoteAddr,
|
||||
Host: query.Host,
|
||||
Since: query.Since,
|
||||
}
|
||||
bucketSeconds := int64(query.BucketMinutes * secondsPerMinute)
|
||||
if bucketSeconds <= 0 {
|
||||
bucketSeconds = 1800
|
||||
}
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows, err := s.AccessLogs.IPTrend(ctx, filter, bucketSeconds)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := make([]*analyticsmodel.NodeAccessLogIPTrend, len(rows))
|
||||
for index, row := range rows {
|
||||
result[index] = &analyticsmodel.NodeAccessLogIPTrend{
|
||||
BucketEpoch: row.BucketEpoch,
|
||||
RequestCount: row.RequestCount,
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// DeleteAllOpenFlareAccessLogs deletes all access logs.
|
||||
func DeleteAllOpenFlareAccessLogs(ctx context.Context) (int64, error) {
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return s.AccessLogs.DeleteAll(ctx)
|
||||
}
|
||||
|
||||
// DeleteOpenFlareAccessLogsBefore deletes access logs older than cutoff.
|
||||
func DeleteOpenFlareAccessLogsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return s.AccessLogs.DeleteBefore(ctx, cutoff)
|
||||
}
|
||||
|
||||
// DeleteOpenFlareAccessLogsByNodeBefore deletes access logs for a node older than cutoff.
|
||||
func DeleteOpenFlareAccessLogsByNodeBefore(ctx context.Context, nodeID string, cutoff time.Time) (int64, error) {
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return s.AccessLogs.DeleteByNodeBefore(ctx, nodeID, cutoff)
|
||||
}
|
||||
|
||||
func buildOpenFlareAccessLogBucketRows(ctx context.Context, query model.OpenFlareAccessLogBucketQuery) ([]*model.OpenFlareAccessLogBucketRow, error) {
|
||||
filter := openFlareAccessLogQueryFromBucket(query)
|
||||
bucketSeconds := int64(query.FoldMinutes * secondsPerMinute)
|
||||
if bucketSeconds <= 0 {
|
||||
bucketSeconds = 180
|
||||
}
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
partials, err := s.AccessLogs.BucketAggregates(ctx, filter, bucketSeconds)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows := make([]*model.OpenFlareAccessLogBucketRow, 0, len(partials))
|
||||
for _, partial := range partials {
|
||||
rows = append(rows, &model.OpenFlareAccessLogBucketRow{
|
||||
BucketEpoch: partial.BucketEpoch,
|
||||
RequestCount: partial.RequestCount,
|
||||
UniqueIPCount: partial.UniqueIPCount,
|
||||
UniqueHostCount: partial.UniqueHostCount,
|
||||
SuccessCount: partial.SuccessCount,
|
||||
ClientErrorCount: partial.ClientErrorCount,
|
||||
ServerErrorCount: partial.ServerErrorCount,
|
||||
Status2xxCount: partial.Status2xxCount,
|
||||
Status4xxCount: partial.Status4xxCount,
|
||||
Status5xxCount: partial.Status5xxCount,
|
||||
BytesSent: partial.BytesSent,
|
||||
RequestLength: partial.RequestLength,
|
||||
})
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
func buildOpenFlareAccessLogBucketIPRows(ctx context.Context, query model.OpenFlareAccessLogBucketIPQuery) ([]*model.OpenFlareAccessLogBucketIPRow, error) {
|
||||
if query.BucketStartedAt.IsZero() {
|
||||
return []*model.OpenFlareAccessLogBucketIPRow{}, nil
|
||||
}
|
||||
foldMinutes := query.FoldMinutes
|
||||
if foldMinutes <= 0 {
|
||||
foldMinutes = 3
|
||||
}
|
||||
bucketStartedAt := query.BucketStartedAt.UTC()
|
||||
filter := model.OpenFlareAccessLogQuery{
|
||||
NodeID: query.NodeID,
|
||||
RemoteAddr: query.RemoteAddr,
|
||||
Host: query.Host,
|
||||
Path: query.Path,
|
||||
Since: bucketStartedAt,
|
||||
Until: bucketStartedAt.Add(time.Duration(foldMinutes) * time.Minute),
|
||||
}
|
||||
rows, err := queryOpenFlareAccessLogIPAggregateRows(ctx, filter, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sortOpenFlareAccessLogBucketIPRows(rows, query.SortBy, query.SortOrder)
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
func buildOpenFlareAccessLogIPSummaryRows(ctx context.Context, query model.OpenFlareAccessLogIPSummaryQuery, recentSince time.Time) ([]*analyticsmodel.NodeAccessLogIPSummary, error) {
|
||||
filter := openFlareAccessLogQueryFromIPSummary(query)
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
partials, err := s.AccessLogs.IPSummaries(ctx, filter, recentSince)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows := make([]*analyticsmodel.NodeAccessLogIPSummary, 0, len(partials))
|
||||
for _, partial := range partials {
|
||||
remoteAddr := strings.TrimSpace(partial.RemoteAddr)
|
||||
if remoteAddr == "" {
|
||||
continue
|
||||
}
|
||||
rows = append(rows, &analyticsmodel.NodeAccessLogIPSummary{
|
||||
RemoteAddr: remoteAddr,
|
||||
Region: strings.TrimSpace(partial.Region),
|
||||
TotalRequests: partial.TotalRequests,
|
||||
Success2xxCount: partial.Success2xxCount,
|
||||
SuccessRatio: partial.SuccessRatio,
|
||||
BytesReceived: partial.BytesReceived,
|
||||
BytesSent: partial.BytesSent,
|
||||
RecentRequests: 0,
|
||||
LastSeenEpoch: partial.LastSeenEpoch,
|
||||
})
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
func queryOpenFlareAccessLogIPAggregateRows(ctx context.Context, filter model.OpenFlareAccessLogQuery, exactRemoteAddr bool) ([]*model.OpenFlareAccessLogBucketIPRow, error) {
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
partials, err := s.AccessLogs.IPAggregates(ctx, filter, exactRemoteAddr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows := make([]*model.OpenFlareAccessLogBucketIPRow, 0, len(partials))
|
||||
for _, partial := range partials {
|
||||
remoteAddr := strings.TrimSpace(partial.RemoteAddr)
|
||||
if remoteAddr == "" {
|
||||
continue
|
||||
}
|
||||
rows = append(rows, &model.OpenFlareAccessLogBucketIPRow{
|
||||
RemoteAddr: remoteAddr,
|
||||
RequestCount: partial.RequestCount,
|
||||
SuccessCount: partial.SuccessCount,
|
||||
ClientErrorCount: partial.ClientErrorCount,
|
||||
ServerErrorCount: partial.ServerErrorCount,
|
||||
LastSeenEpoch: partial.LastSeenEpoch,
|
||||
})
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
func openFlareAccessLogQueryFromBucket(query model.OpenFlareAccessLogBucketQuery) model.OpenFlareAccessLogQuery {
|
||||
return model.OpenFlareAccessLogQuery{
|
||||
NodeID: query.NodeID,
|
||||
RemoteAddr: query.RemoteAddr,
|
||||
Host: query.Host,
|
||||
Hosts: query.Hosts,
|
||||
Path: query.Path,
|
||||
Since: query.Since,
|
||||
Until: query.Until,
|
||||
Page: query.Page,
|
||||
PageSize: query.PageSize,
|
||||
SortBy: query.SortBy,
|
||||
SortOrder: query.SortOrder,
|
||||
}
|
||||
}
|
||||
|
||||
func openFlareAccessLogQueryFromIPSummary(query model.OpenFlareAccessLogIPSummaryQuery) model.OpenFlareAccessLogQuery {
|
||||
return model.OpenFlareAccessLogQuery{
|
||||
NodeID: query.NodeID,
|
||||
RemoteAddr: query.RemoteAddr,
|
||||
Host: query.Host,
|
||||
Since: query.Since,
|
||||
Until: query.Until,
|
||||
Page: query.Page,
|
||||
PageSize: query.PageSize,
|
||||
SortBy: query.SortBy,
|
||||
SortOrder: query.SortOrder,
|
||||
}
|
||||
}
|
||||
|
||||
func sortOpenFlareAccessLogBucketIPRows(items []*model.OpenFlareAccessLogBucketIPRow, sortBy string, sortOrder string) {
|
||||
desc := openFlareAccessLogNormalizeSortOrder(sortOrder) != sortOrderAsc
|
||||
sort.Slice(items, func(i, j int) bool {
|
||||
left := items[i]
|
||||
right := items[j]
|
||||
if left == nil || right == nil {
|
||||
return left != nil
|
||||
}
|
||||
var compare int
|
||||
switch strings.TrimSpace(sortBy) {
|
||||
case "last_seen_at":
|
||||
compare = openFlareAccessLogCompareInt64(left.LastSeenEpoch, right.LastSeenEpoch)
|
||||
case "remote_addr":
|
||||
compare = strings.Compare(left.RemoteAddr, right.RemoteAddr)
|
||||
default:
|
||||
compare = openFlareAccessLogCompareInt64(left.RequestCount, right.RequestCount)
|
||||
}
|
||||
if compare == 0 {
|
||||
compare = openFlareAccessLogCompareInt64(left.LastSeenEpoch, right.LastSeenEpoch)
|
||||
}
|
||||
if compare == 0 {
|
||||
compare = strings.Compare(left.RemoteAddr, right.RemoteAddr)
|
||||
}
|
||||
if desc {
|
||||
return compare > 0
|
||||
}
|
||||
return compare < 0
|
||||
})
|
||||
}
|
||||
|
||||
func openFlareAccessLogPaginateBounds(total int, page int, pageSize int) (int, int) {
|
||||
if page < 0 {
|
||||
page = 0
|
||||
}
|
||||
if pageSize <= 0 {
|
||||
return 0, total
|
||||
}
|
||||
start := min(page*pageSize, total)
|
||||
end := min(start+pageSize, total)
|
||||
return start, end
|
||||
}
|
||||
|
||||
func openFlareAccessLogNormalizeSortOrder(sortOrder string) string {
|
||||
if strings.EqualFold(strings.TrimSpace(sortOrder), sortOrderAsc) {
|
||||
return sortOrderAsc
|
||||
}
|
||||
return "desc"
|
||||
}
|
||||
|
||||
func openFlareAccessLogCompareInt64(left int64, right int64) int {
|
||||
switch {
|
||||
case left > right:
|
||||
return 1
|
||||
case left < right:
|
||||
return -1
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/glebarez/sqlite"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/logger"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/model"
|
||||
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
|
||||
"Wavelet/openflare/plugins/server/kernel/repository/logstore"
|
||||
"Wavelet/pkg/idgen"
|
||||
db "Wavelet/plugins/infra/database"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// accessLogTestDBSeq 保证每个测试获得独立的 sqlite 内存库(cache=shared 下同名 DSN 复用同一库)。
|
||||
var accessLogTestDBSeq int64
|
||||
|
||||
func setupOpenFlareAccessLogTestEnvironment(t *testing.T) (context.Context, func()) {
|
||||
t.Helper()
|
||||
dsn := fmt.Sprintf("file:repo-access-log-test-%d?mode=memory&cache=shared", atomic.AddInt64(&accessLogTestDBSeq, 1))
|
||||
gdb, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{
|
||||
DisableForeignKeyConstraintWhenMigrating: true,
|
||||
Logger: logger.Default.LogMode(logger.Silent),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, gdb.AutoMigrate(&analyticsmodel.NodeAccessLog{}))
|
||||
db.SetDB(gdb)
|
||||
require.NoError(t, idgen.Init(1))
|
||||
|
||||
logstore.ResetForTest()
|
||||
logstore.SetConfigReader(func(_ context.Context, key string) (string, error) {
|
||||
if key == model.ConfigKeyLogDatabase {
|
||||
return "sqlite", nil
|
||||
}
|
||||
return "", nil
|
||||
})
|
||||
logstore.SetAccessLogHooks(logstore.AccessLogHooks{})
|
||||
logstore.SetObservabilityHooks(logstore.ObservabilityHooks{})
|
||||
|
||||
ctx := context.Background()
|
||||
store, err := logstore.Active(ctx)
|
||||
require.NoError(t, err)
|
||||
// 写入入口只入队;测试环境立即 flush,保证后续查询可见。
|
||||
logstore.SetAccessLogHooks(logstore.AccessLogHooks{
|
||||
QueueNodeAccessLogs: func(logs []analyticsmodel.NodeAccessLog) {
|
||||
require.NoError(t, store.AccessLogs.BatchInsertNodeAccessLogs(context.Background(), logs))
|
||||
},
|
||||
})
|
||||
return ctx, func() {
|
||||
logstore.SetAccessLogHooks(logstore.AccessLogHooks{})
|
||||
logstore.ResetForTest()
|
||||
db.SetDB(nil)
|
||||
}
|
||||
}
|
||||
|
||||
func seedOpenFlareAccessLogs(t *testing.T, ctx context.Context, now time.Time) {
|
||||
t.Helper()
|
||||
records := []*model.OpenFlareAccessLog{
|
||||
{NodeID: "node-a", LoggedAt: now.Add(-5 * time.Minute), RemoteAddr: "1.1.1.1", Region: "US", Host: "a.example.com", Path: "/alpha", StatusCode: 200},
|
||||
{NodeID: "node-a", LoggedAt: now.Add(-4 * time.Minute), RemoteAddr: "2.2.2.2", Region: "US", Host: "a.example.com", Path: "/beta", StatusCode: 404},
|
||||
{NodeID: "node-b", LoggedAt: now.Add(-3 * time.Minute), RemoteAddr: "1.1.1.1", Region: "EU", Host: "b.example.com", Path: "/gamma", StatusCode: 502},
|
||||
{NodeID: "node-b", LoggedAt: now.Add(-2 * time.Minute), RemoteAddr: "3.3.3.3", Region: "EU", Host: "b.example.com", Path: "/delta", StatusCode: 200},
|
||||
{NodeID: "node-b", LoggedAt: now.Add(-1 * time.Minute), RemoteAddr: "", Region: "", Host: "b.example.com", Path: "/empty-ip", StatusCode: 200},
|
||||
}
|
||||
require.NoError(t, InsertOpenFlareAccessLogsBatch(ctx, records))
|
||||
}
|
||||
|
||||
func TestListOpenFlareAccessLogsPaginated(t *testing.T) {
|
||||
ctx, cleanup := setupOpenFlareAccessLogTestEnvironment(t)
|
||||
defer cleanup()
|
||||
|
||||
now := time.Now().UTC()
|
||||
for index := range 15 {
|
||||
record := &model.OpenFlareAccessLog{
|
||||
NodeID: "node-page",
|
||||
LoggedAt: now.Add(-time.Duration(index) * time.Minute),
|
||||
RemoteAddr: fmt.Sprintf("203.0.113.%d", (index%5)+1),
|
||||
Host: "example.com",
|
||||
Path: fmt.Sprintf("/path-%02d", index),
|
||||
StatusCode: 200,
|
||||
}
|
||||
require.NoError(t, InsertOpenFlareAccessLogsBatch(ctx, []*model.OpenFlareAccessLog{record}))
|
||||
}
|
||||
|
||||
// 0-based 分页与 CH ListNodeAccessLogs 一致:page=1 size=5 → OFFSET 5 → /path-05..09。
|
||||
query := model.OpenFlareAccessLogQuery{
|
||||
NodeID: "node-page",
|
||||
Since: now.Add(-24 * time.Hour),
|
||||
Page: 1,
|
||||
PageSize: 5,
|
||||
SortBy: "logged_at",
|
||||
SortOrder: "desc",
|
||||
}
|
||||
page, err := ListOpenFlareAccessLogs(ctx, query)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, page, 5)
|
||||
assert.Equal(t, "/path-05", page[0].Path)
|
||||
assert.Equal(t, "/path-09", page[4].Path)
|
||||
}
|
||||
|
||||
func TestCountOpenFlareAccessLogs(t *testing.T) {
|
||||
ctx, cleanup := setupOpenFlareAccessLogTestEnvironment(t)
|
||||
defer cleanup()
|
||||
|
||||
now := time.Now().UTC()
|
||||
seedOpenFlareAccessLogs(t, ctx, now)
|
||||
|
||||
query := model.OpenFlareAccessLogQuery{
|
||||
Since: now.Add(-10 * time.Minute),
|
||||
}
|
||||
totalRecords, totalIPs, _, err := CountOpenFlareAccessLogs(ctx, query)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(5), totalRecords)
|
||||
// GORM 与 CH 一致:distinct IP 排除空 remote_addr(CH uniqExactIf(remote_addr, remote_addr != ''))。
|
||||
assert.Equal(t, int64(3), totalIPs)
|
||||
}
|
||||
|
||||
func TestListOpenFlareAccessLogsFiltersAndSort(t *testing.T) {
|
||||
ctx, cleanup := setupOpenFlareAccessLogTestEnvironment(t)
|
||||
defer cleanup()
|
||||
|
||||
now := time.Now().UTC()
|
||||
seedOpenFlareAccessLogs(t, ctx, now)
|
||||
|
||||
query := model.OpenFlareAccessLogQuery{
|
||||
NodeID: "node-a",
|
||||
Since: now.Add(-10 * time.Minute),
|
||||
SortBy: "status_code",
|
||||
SortOrder: "desc",
|
||||
}
|
||||
rows, err := ListOpenFlareAccessLogs(ctx, query)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, rows, 2)
|
||||
assert.Equal(t, 404, rows[0].StatusCode)
|
||||
assert.Equal(t, 200, rows[1].StatusCode)
|
||||
}
|
||||
|
||||
func TestDeleteOpenFlareAccessLogsBefore(t *testing.T) {
|
||||
ctx, cleanup := setupOpenFlareAccessLogTestEnvironment(t)
|
||||
defer cleanup()
|
||||
|
||||
now := time.Now().UTC()
|
||||
seedOpenFlareAccessLogs(t, ctx, now)
|
||||
|
||||
deleted, err := DeleteOpenFlareAccessLogsBefore(ctx, now.Add(-2*time.Minute))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(3), deleted)
|
||||
|
||||
totalRecords, _, _, err := CountOpenFlareAccessLogs(ctx, model.OpenFlareAccessLogQuery{Since: now.Add(-10 * time.Minute)})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(2), totalRecords)
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"gorm.io/gorm"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/model"
|
||||
db "Wavelet/plugins/infra/database"
|
||||
)
|
||||
|
||||
// GetAcmeAccountByID 按 ID 查询 ACME 账号。
|
||||
func GetAcmeAccountByID(ctx context.Context, id uint) (*model.AcmeAccount, error) {
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return nil, errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
var account model.AcmeAccount
|
||||
if err := conn.First(&account, id).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &account, nil
|
||||
}
|
||||
|
||||
// CreateAcmeAccountRecord 创建 ACME 账号。
|
||||
func CreateAcmeAccountRecord(ctx context.Context, account *model.AcmeAccount) error {
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
return conn.Create(account).Error
|
||||
}
|
||||
|
||||
// SaveAcmeAccount 保存 ACME 账号。
|
||||
func SaveAcmeAccount(ctx context.Context, account *model.AcmeAccount) error {
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
return conn.Save(account).Error
|
||||
}
|
||||
|
||||
// GetDefaultAcmeAccount 获取默认 ACME 账号,不存在时创建占位记录。
|
||||
func GetDefaultAcmeAccount(ctx context.Context) (*model.AcmeAccount, error) {
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return nil, errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
var account model.AcmeAccount
|
||||
err := conn.Order("id asc").First(&account).Error
|
||||
if err == nil {
|
||||
return &account, nil
|
||||
}
|
||||
if !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, err
|
||||
}
|
||||
account = model.AcmeAccount{
|
||||
Email: "admin@openflare.dev",
|
||||
}
|
||||
if err = conn.Create(&account).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &account, nil
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/model"
|
||||
db "Wavelet/plugins/infra/database"
|
||||
)
|
||||
|
||||
// ListOpenFlareApplyLogs returns apply logs ordered by id desc with optional pagination.
|
||||
func ListOpenFlareApplyLogs(ctx context.Context, query model.OpenFlareApplyLogQuery) ([]*model.OpenFlareApplyLog, error) {
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return nil, errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
|
||||
dbQuery := conn.Model(&model.OpenFlareApplyLog{}).Order("id desc")
|
||||
if query.NodeID != "" {
|
||||
dbQuery = dbQuery.Where("node_id = ?", query.NodeID)
|
||||
}
|
||||
if query.PageSize > 0 {
|
||||
offset := 0
|
||||
if query.PageNo > 1 {
|
||||
offset = (query.PageNo - 1) * query.PageSize
|
||||
}
|
||||
dbQuery = dbQuery.Limit(query.PageSize).Offset(offset)
|
||||
}
|
||||
|
||||
var logs []*model.OpenFlareApplyLog
|
||||
if err := dbQuery.Find(&logs).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return logs, nil
|
||||
}
|
||||
|
||||
// CountOpenFlareApplyLogs returns total apply logs, optionally filtered by node_id.
|
||||
func CountOpenFlareApplyLogs(ctx context.Context, nodeID string) (int64, error) {
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return 0, errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
|
||||
query := conn.Model(&model.OpenFlareApplyLog{})
|
||||
if nodeID != "" {
|
||||
query = query.Where("node_id = ?", nodeID)
|
||||
}
|
||||
|
||||
var total int64
|
||||
if err := query.Count(&total).Error; err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
// GetLatestOpenFlareApplyLogByNodeID returns the most recent apply log for a node.
|
||||
func GetLatestOpenFlareApplyLogByNodeID(ctx context.Context, nodeID string) (*model.OpenFlareApplyLog, error) {
|
||||
nodeID = strings.TrimSpace(nodeID)
|
||||
if nodeID == "" {
|
||||
return nil, errors.New("node_id is required")
|
||||
}
|
||||
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return nil, errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
|
||||
var log model.OpenFlareApplyLog
|
||||
err := conn.Where("node_id = ?", nodeID).Order("id desc").First(&log).Error
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &log, nil
|
||||
}
|
||||
|
||||
// GetLatestOpenFlareApplyLogsByNodeIDs returns the latest apply log per node id.
|
||||
func GetLatestOpenFlareApplyLogsByNodeIDs(ctx context.Context, nodeIDs []string) (map[string]*model.OpenFlareApplyLog, error) {
|
||||
result := make(map[string]*model.OpenFlareApplyLog)
|
||||
if len(nodeIDs) == 0 {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return nil, errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
|
||||
var logs []*model.OpenFlareApplyLog
|
||||
subQuery := conn.Model(&model.OpenFlareApplyLog{}).
|
||||
Select("MAX(id) AS id").
|
||||
Where("node_id IN ?", nodeIDs).
|
||||
Group("node_id")
|
||||
if err := conn.Where("id IN (?)", subQuery).Find(&logs).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, log := range logs {
|
||||
result[log.NodeID] = log
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// CreateOpenFlareApplyLog inserts an apply log row.
|
||||
func CreateOpenFlareApplyLog(ctx context.Context, log *model.OpenFlareApplyLog) error {
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
return conn.Create(log).Error
|
||||
}
|
||||
|
||||
// CreateOpenFlareApplyLogAndUpdateNode creates an apply log and updates the node from the apply result in one transaction.
|
||||
func CreateOpenFlareApplyLogAndUpdateNode(ctx context.Context, log *model.OpenFlareApplyLog, applyResult, version, message string) error {
|
||||
if log == nil {
|
||||
return errors.New("apply log is required")
|
||||
}
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
now := log.CreatedAt
|
||||
if now.IsZero() {
|
||||
now = time.Now()
|
||||
}
|
||||
return conn.Transaction(func(tx *gorm.DB) error {
|
||||
if err := tx.Create(log).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
return updateOpenFlareNodeFromApplyResultTx(tx, log.NodeID, applyResult, version, message, now)
|
||||
})
|
||||
}
|
||||
|
||||
// DeleteAllOpenFlareApplyLogs removes every apply log record.
|
||||
func DeleteAllOpenFlareApplyLogs(ctx context.Context) (int64, error) {
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return 0, errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
|
||||
result := conn.Session(&gorm.Session{AllowGlobalUpdate: true}).Delete(&model.OpenFlareApplyLog{})
|
||||
return result.RowsAffected, result.Error
|
||||
}
|
||||
|
||||
// DeleteOpenFlareApplyLogsBefore removes apply logs created before the cutoff time.
|
||||
func DeleteOpenFlareApplyLogsBefore(ctx context.Context, before time.Time) (int64, error) {
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return 0, errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
|
||||
result := conn.Where("created_at < ?", before).Delete(&model.OpenFlareApplyLog{})
|
||||
return result.RowsAffected, result.Error
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/model"
|
||||
|
||||
db "Wavelet/plugins/infra/database"
|
||||
|
||||
"github.com/glebarez/sqlite"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func setupApplyLogModelTestDB(t *testing.T) func() {
|
||||
t.Helper()
|
||||
|
||||
sqliteDB, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{
|
||||
DisableForeignKeyConstraintWhenMigrating: true,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, sqliteDB.AutoMigrate(&model.OpenFlareApplyLog{}))
|
||||
|
||||
db.SetDB(sqliteDB)
|
||||
return func() {
|
||||
db.SetDB(nil)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsRepeatSuccessApplyLog(t *testing.T) {
|
||||
latest := &model.OpenFlareApplyLog{
|
||||
Version: "20260615-001",
|
||||
Checksum: "checksum-a",
|
||||
Result: "success",
|
||||
}
|
||||
|
||||
assert.True(t, model.IsRepeatSuccessApplyLog(latest, "20260615-001", "checksum-a", "success"))
|
||||
assert.False(t, model.IsRepeatSuccessApplyLog(latest, "20260615-002", "checksum-a", "success"))
|
||||
assert.False(t, model.IsRepeatSuccessApplyLog(latest, "20260615-001", "checksum-b", "success"))
|
||||
assert.False(t, model.IsRepeatSuccessApplyLog(latest, "20260615-001", "checksum-a", "failed"))
|
||||
assert.False(t, model.IsRepeatSuccessApplyLog(nil, "20260615-001", "checksum-a", "success"))
|
||||
}
|
||||
|
||||
func TestGetLatestOpenFlareApplyLogByNodeID(t *testing.T) {
|
||||
cleanup := setupApplyLogModelTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
require.NoError(t, db.DB(ctx).Create(&model.OpenFlareApplyLog{
|
||||
NodeID: "node-1",
|
||||
Version: "v1",
|
||||
Result: "success",
|
||||
Checksum: "checksum-1",
|
||||
CreatedAt: now.Add(-time.Hour),
|
||||
}).Error)
|
||||
require.NoError(t, db.DB(ctx).Create(&model.OpenFlareApplyLog{
|
||||
NodeID: "node-1",
|
||||
Version: "v2",
|
||||
Result: "success",
|
||||
Checksum: "checksum-2",
|
||||
CreatedAt: now,
|
||||
}).Error)
|
||||
|
||||
latest, err := GetLatestOpenFlareApplyLogByNodeID(ctx, "node-1")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, latest)
|
||||
assert.Equal(t, "v2", latest.Version)
|
||||
|
||||
missing, err := GetLatestOpenFlareApplyLogByNodeID(ctx, "node-missing")
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, missing)
|
||||
}
|
||||
@@ -0,0 +1,226 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/model"
|
||||
db "Wavelet/plugins/infra/database"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
const singletonCFConnectionID uint = 1
|
||||
|
||||
// CFPointingMemberContext contains all local state needed to reconcile one member.
|
||||
type CFPointingMemberContext struct {
|
||||
Member model.CFPointingMember
|
||||
Group model.CFPointingGroup
|
||||
Domain model.ZoneDomain
|
||||
Zone model.Zone
|
||||
Node model.OpenFlareNode
|
||||
}
|
||||
|
||||
// GetCFConnection returns the global Cloudflare connection.
|
||||
func GetCFConnection(ctx context.Context) (*model.CFConnection, error) {
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return nil, errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
var item model.CFConnection
|
||||
if err := conn.First(&item, singletonCFConnectionID).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &item, nil
|
||||
}
|
||||
|
||||
// UpsertCFConnection creates or replaces the global Cloudflare connection.
|
||||
func UpsertCFConnection(ctx context.Context, item *model.CFConnection) error {
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
item.ID = singletonCFConnectionID
|
||||
return conn.Save(item).Error
|
||||
}
|
||||
|
||||
// DeleteCFConnection clears the global Cloudflare connection.
|
||||
func DeleteCFConnection(ctx context.Context) error {
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
return conn.Delete(&model.CFConnection{}, singletonCFConnectionID).Error
|
||||
}
|
||||
|
||||
// ListCFPointingGroups lists Cloudflare pointing groups newest first.
|
||||
func ListCFPointingGroups(ctx context.Context) ([]model.CFPointingGroup, error) {
|
||||
var items []model.CFPointingGroup
|
||||
if err := db.DB(ctx).Order("id desc").Find(&items).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// GetCFPointingGroup returns a group by ID.
|
||||
func GetCFPointingGroup(ctx context.Context, id uint) (*model.CFPointingGroup, error) {
|
||||
var item model.CFPointingGroup
|
||||
if err := db.DB(ctx).First(&item, id).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &item, nil
|
||||
}
|
||||
|
||||
// CreateCFPointingGroup creates a group.
|
||||
func CreateCFPointingGroup(ctx context.Context, item *model.CFPointingGroup) error {
|
||||
return db.DB(ctx).Create(item).Error
|
||||
}
|
||||
|
||||
// SaveCFPointingGroup persists a group.
|
||||
func SaveCFPointingGroup(ctx context.Context, item *model.CFPointingGroup) error {
|
||||
return db.DB(ctx).Save(item).Error
|
||||
}
|
||||
|
||||
// DeleteCFPointingGroup deletes an empty group.
|
||||
func DeleteCFPointingGroup(ctx context.Context, id uint) error {
|
||||
return db.DB(ctx).Delete(&model.CFPointingGroup{}, id).Error
|
||||
}
|
||||
|
||||
// CountCFPointingMembersByGroupID counts members in a group.
|
||||
func CountCFPointingMembersByGroupID(ctx context.Context, groupID uint) (int64, error) {
|
||||
var count int64
|
||||
err := db.DB(ctx).Table("of_cf_pointing_members AS members").
|
||||
Joins("JOIN of_zone_domains AS domains ON domains.id = members.zone_domain_id").
|
||||
Where("members.group_id = ?", groupID).Count(&count).Error
|
||||
return count, err
|
||||
}
|
||||
|
||||
// ListCFPointingMembersByGroupID lists members by group.
|
||||
func ListCFPointingMembersByGroupID(ctx context.Context, groupID uint) ([]model.CFPointingMember, error) {
|
||||
var items []model.CFPointingMember
|
||||
if err := db.DB(ctx).Where("group_id = ?", groupID).Order("id asc").Find(&items).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// ListCFPointingMembersByActiveNodeID lists members whose group currently targets a node.
|
||||
func ListCFPointingMembersByActiveNodeID(ctx context.Context, nodeID uint) ([]model.CFPointingMember, error) {
|
||||
var items []model.CFPointingMember
|
||||
err := db.DB(ctx).Table("of_cf_pointing_members AS members").
|
||||
Select("members.*").
|
||||
Joins("JOIN of_cf_pointing_groups AS groups ON groups.id = members.group_id").
|
||||
Where("groups.active_node_id = ? AND groups.enabled = ?", nodeID, true).
|
||||
Order("members.id asc").Scan(&items).Error
|
||||
return items, err
|
||||
}
|
||||
|
||||
// GetCFPointingMember returns a member scoped to its group.
|
||||
func GetCFPointingMember(ctx context.Context, groupID, memberID uint) (*model.CFPointingMember, error) {
|
||||
var item model.CFPointingMember
|
||||
if err := db.DB(ctx).Where("id = ? AND group_id = ?", memberID, groupID).First(&item).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &item, nil
|
||||
}
|
||||
|
||||
// GetCFPointingMemberByID returns a member by ID.
|
||||
func GetCFPointingMemberByID(ctx context.Context, id uint) (*model.CFPointingMember, error) {
|
||||
var item model.CFPointingMember
|
||||
if err := db.DB(ctx).First(&item, id).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &item, nil
|
||||
}
|
||||
|
||||
// GetCFPointingMemberByZoneDomainID returns the member managing a ZoneDomain.
|
||||
func GetCFPointingMemberByZoneDomainID(ctx context.Context, zoneDomainID uint) (*model.CFPointingMember, error) {
|
||||
var item model.CFPointingMember
|
||||
if err := db.DB(ctx).Where("zone_domain_id = ?", zoneDomainID).First(&item).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &item, nil
|
||||
}
|
||||
|
||||
// CreateCFPointingMember creates a member.
|
||||
func CreateCFPointingMember(ctx context.Context, item *model.CFPointingMember) error {
|
||||
return db.DB(ctx).Create(item).Error
|
||||
}
|
||||
|
||||
// SaveCFPointingMember persists a member.
|
||||
func SaveCFPointingMember(ctx context.Context, item *model.CFPointingMember) error {
|
||||
return db.DB(ctx).Save(item).Error
|
||||
}
|
||||
|
||||
// UpdateCFPointingMemberColumns updates selected member fields.
|
||||
func UpdateCFPointingMemberColumns(ctx context.Context, id uint, changes map[string]any) error {
|
||||
return db.DB(ctx).Model(&model.CFPointingMember{}).Where("id = ?", id).Updates(changes).Error
|
||||
}
|
||||
|
||||
// DeleteCFPointingMember deletes a member.
|
||||
func DeleteCFPointingMember(ctx context.Context, item *model.CFPointingMember) error {
|
||||
return db.DB(ctx).Delete(item).Error
|
||||
}
|
||||
|
||||
// ListAvailableCFZoneDomains returns ZoneDomains not already managed by Cloudflare pointing.
|
||||
func ListAvailableCFZoneDomains(ctx context.Context) ([]model.ZoneDomain, error) {
|
||||
var items []model.ZoneDomain
|
||||
err := db.DB(ctx).Where(`NOT EXISTS (
|
||||
SELECT 1 FROM of_cf_pointing_members AS members
|
||||
WHERE members.zone_domain_id = of_zone_domains.id
|
||||
)`).Order("domain asc").Find(&items).Error
|
||||
return items, err
|
||||
}
|
||||
|
||||
// GetCFPointingMemberContext loads one member and all referenced local objects.
|
||||
func GetCFPointingMemberContext(ctx context.Context, memberID uint) (*CFPointingMemberContext, error) {
|
||||
member, err := GetCFPointingMemberByID(ctx, memberID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
group, err := GetCFPointingGroup(ctx, member.GroupID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
domain, err := GetZoneDomainByID(ctx, member.ZoneDomainID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
zone, err := GetZoneByID(ctx, domain.ZoneID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
node, err := GetOpenFlareNodeByID(ctx, group.ActiveNodeID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &CFPointingMemberContext{Member: *member, Group: *group, Domain: *domain, Zone: *zone, Node: *node}, nil
|
||||
}
|
||||
|
||||
// GetZoneDomainByID returns a ZoneDomain by primary key.
|
||||
func GetZoneDomainByID(ctx context.Context, id uint) (*model.ZoneDomain, error) {
|
||||
var item model.ZoneDomain
|
||||
if err := db.DB(ctx).First(&item, id).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &item, nil
|
||||
}
|
||||
|
||||
// MarkCFPointingGroupMembersPending resets every member after target changes.
|
||||
func MarkCFPointingGroupMembersPending(ctx context.Context, groupID uint) error {
|
||||
return db.DB(ctx).Model(&model.CFPointingMember{}).Where("group_id = ?", groupID).
|
||||
Updates(map[string]any{"sync_status": model.CFMemberSyncPending, "last_error": ""}).Error
|
||||
}
|
||||
|
||||
// DeleteCFPointingGroupAndMembers removes a group after its remote records are deleted.
|
||||
func DeleteCFPointingGroupAndMembers(ctx context.Context, groupID uint) error {
|
||||
return db.DB(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
if err := tx.Where("group_id = ?", groupID).Delete(&model.CFPointingMember{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Delete(&model.CFPointingGroup{}, groupID).Error
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/model"
|
||||
db "Wavelet/plugins/infra/database"
|
||||
|
||||
"github.com/glebarez/sqlite"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func setupCloudflareRepositoryDB(t *testing.T) *gorm.DB {
|
||||
t.Helper()
|
||||
conn, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{DisableForeignKeyConstraintWhenMigrating: true})
|
||||
if err != nil {
|
||||
t.Fatalf("gorm.Open() error = %v", err)
|
||||
}
|
||||
if err := conn.AutoMigrate(
|
||||
&model.CFConnection{}, &model.CFPointingGroup{}, &model.CFPointingMember{},
|
||||
&model.Zone{}, &model.ZoneDomain{}, &model.OpenFlareNode{}, &model.DNSAccount{},
|
||||
); err != nil {
|
||||
t.Fatalf("AutoMigrate() error = %v", err)
|
||||
}
|
||||
db.SetDB(conn)
|
||||
t.Cleanup(func() { db.SetDB(nil) })
|
||||
return conn
|
||||
}
|
||||
|
||||
func TestUpsertCFConnectionKeepsSingleRow(t *testing.T) {
|
||||
setupCloudflareRepositoryDB(t)
|
||||
ctx := context.Background()
|
||||
|
||||
first := &model.CFConnection{Source: model.CFConnectionSourceStandalone, Authorization: "one"}
|
||||
if err := UpsertCFConnection(ctx, first); err != nil {
|
||||
t.Fatalf("UpsertCFConnection(first) error = %v", err)
|
||||
}
|
||||
accountID := uint(9)
|
||||
second := &model.CFConnection{Source: model.CFConnectionSourceDNSAccount, DNSAccountID: &accountID}
|
||||
if err := UpsertCFConnection(ctx, second); err != nil {
|
||||
t.Fatalf("UpsertCFConnection(second) error = %v", err)
|
||||
}
|
||||
|
||||
got, err := GetCFConnection(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("GetCFConnection() error = %v", err)
|
||||
}
|
||||
if got.ID != first.ID || got.Source != model.CFConnectionSourceDNSAccount {
|
||||
t.Errorf("GetCFConnection() = %+v, want same row with dns_account source", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListAvailableCFZoneDomainsExcludesMembers(t *testing.T) {
|
||||
conn := setupCloudflareRepositoryDB(t)
|
||||
ctx := context.Background()
|
||||
zone := model.Zone{Domain: "example.com"}
|
||||
if err := conn.Create(&zone).Error; err != nil {
|
||||
t.Fatalf("Create(zone) error = %v", err)
|
||||
}
|
||||
domains := []model.ZoneDomain{
|
||||
{ZoneID: zone.ID, Domain: "api.example.com"},
|
||||
{ZoneID: zone.ID, Domain: "www.example.com"},
|
||||
}
|
||||
if err := conn.Create(&domains).Error; err != nil {
|
||||
t.Fatalf("Create(domains) error = %v", err)
|
||||
}
|
||||
group := model.CFPointingGroup{Name: "edge", PrimaryNodeID: 1, ActiveNodeID: 1, Enabled: true}
|
||||
if err := CreateCFPointingGroup(ctx, &group); err != nil {
|
||||
t.Fatalf("CreateCFPointingGroup() error = %v", err)
|
||||
}
|
||||
member := model.CFPointingMember{GroupID: group.ID, ZoneDomainID: domains[0].ID}
|
||||
if err := CreateCFPointingMember(ctx, &member); err != nil {
|
||||
t.Fatalf("CreateCFPointingMember() error = %v", err)
|
||||
}
|
||||
|
||||
got, err := ListAvailableCFZoneDomains(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("ListAvailableCFZoneDomains() error = %v", err)
|
||||
}
|
||||
if len(got) != 1 || got[0].ID != domains[1].ID {
|
||||
t.Errorf("ListAvailableCFZoneDomains() = %+v, want only %d", got, domains[1].ID)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"gorm.io/gorm"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/model"
|
||||
db "Wavelet/plugins/infra/database"
|
||||
)
|
||||
|
||||
// ListConfigVersionSummaries returns config version summaries ordered by created_at desc.
|
||||
func ListConfigVersionSummaries(ctx context.Context) ([]*model.ConfigVersionSummary, error) {
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return nil, errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
var versions []*model.ConfigVersionSummary
|
||||
err := conn.Model(&model.ConfigVersion{}).
|
||||
Select("version", "checksum", "is_active", "created_by", "created_at").
|
||||
Order("created_at desc, version desc").
|
||||
Find(&versions).Error
|
||||
return versions, err
|
||||
}
|
||||
|
||||
// GetConfigVersionByVersion returns a config version by version string.
|
||||
func GetConfigVersionByVersion(ctx context.Context, version string) (*model.ConfigVersion, error) {
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return nil, errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
var cv model.ConfigVersion
|
||||
if err := conn.First(&cv, "version = ?", version).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &cv, nil
|
||||
}
|
||||
|
||||
// GetActiveConfigVersion returns the currently active config version.
|
||||
func GetActiveConfigVersion(ctx context.Context) (*model.ConfigVersion, error) {
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return nil, errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
var version model.ConfigVersion
|
||||
if err := conn.Where("is_active = ?", true).Order("version desc").First(&version).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &version, nil
|
||||
}
|
||||
|
||||
// GetLatestConfigVersionByPrefix returns the latest version string matching a date prefix.
|
||||
func GetLatestConfigVersionByPrefix(ctx context.Context, prefix string) (string, error) {
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return "", errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
var version model.ConfigVersion
|
||||
err := conn.Model(&model.ConfigVersion{}).
|
||||
Select("version").
|
||||
Where("version LIKE ?", prefix+"-%").
|
||||
Order("version desc").
|
||||
First(&version).Error
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return version.Version, nil
|
||||
}
|
||||
|
||||
// CreateConfigVersion inserts a new config version record.
|
||||
func CreateConfigVersion(ctx context.Context, version *model.ConfigVersion) error {
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
return conn.Create(version).Error
|
||||
}
|
||||
|
||||
// PublishConfigVersionTx deactivates all versions and creates a new active version.
|
||||
func PublishConfigVersionTx(ctx context.Context, version *model.ConfigVersion) error {
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
return conn.Transaction(func(tx *gorm.DB) error {
|
||||
if err := tx.Model(&model.ConfigVersion{}).Where("is_active = ?", true).Update("is_active", false).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Create(version).Error
|
||||
})
|
||||
}
|
||||
|
||||
// ActivateConfigVersionTx marks the given version active and deactivates others.
|
||||
func ActivateConfigVersionTx(ctx context.Context, version string) error {
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
return conn.Transaction(func(tx *gorm.DB) error {
|
||||
if err := tx.Model(&model.ConfigVersion{}).Where("is_active = ?", true).Update("is_active", false).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Model(&model.ConfigVersion{}).Where("version = ?", version).Update("is_active", true).Error
|
||||
})
|
||||
}
|
||||
|
||||
// DeleteConfigVersionsByVersions removes config versions by versions.
|
||||
func DeleteConfigVersionsByVersions(ctx context.Context, versions []string) (int64, error) {
|
||||
if len(versions) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return 0, errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
result := conn.Where("version IN ?", versions).Delete(&model.ConfigVersion{})
|
||||
return result.RowsAffected, result.Error
|
||||
}
|
||||
|
||||
// ListEnabledProxyRoutes returns enabled proxy routes ordered by id asc.
|
||||
func ListEnabledProxyRoutes(ctx context.Context) ([]*model.ProxyRoute, error) {
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return nil, errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
var routes []*model.ProxyRoute
|
||||
if err := conn.Where("enabled = ?", true).Order("id asc").Find(&routes).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return routes, nil
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/model"
|
||||
db "Wavelet/plugins/infra/database"
|
||||
)
|
||||
|
||||
// ListDNSAccounts 列出全部 DNS 账号(授权信息不通过 JSON 暴露)。
|
||||
func ListDNSAccounts(ctx context.Context) ([]model.DNSAccount, error) {
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return nil, errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
var accounts []model.DNSAccount
|
||||
if err := conn.Order("id desc").Find(&accounts).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return accounts, nil
|
||||
}
|
||||
|
||||
// GetDNSAccountByID 按 ID 查询 DNS 账号。
|
||||
func GetDNSAccountByID(ctx context.Context, id uint) (*model.DNSAccount, error) {
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return nil, errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
var account model.DNSAccount
|
||||
if err := conn.First(&account, id).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &account, nil
|
||||
}
|
||||
|
||||
// CreateDNSAccountRecord 创建 DNS 账号。
|
||||
func CreateDNSAccountRecord(ctx context.Context, account *model.DNSAccount) error {
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
return conn.Create(account).Error
|
||||
}
|
||||
|
||||
// SaveDNSAccount 保存 DNS 账号。
|
||||
func SaveDNSAccount(ctx context.Context, account *model.DNSAccount) error {
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
return conn.Save(account).Error
|
||||
}
|
||||
|
||||
// DeleteDNSAccountRecord 删除 DNS 账号。
|
||||
func DeleteDNSAccountRecord(ctx context.Context, id uint) error {
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
return conn.Delete(&model.DNSAccount{}, id).Error
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/model"
|
||||
db "Wavelet/plugins/infra/database"
|
||||
)
|
||||
|
||||
const (
|
||||
openFlareNodeStatusOnline = "online"
|
||||
openFlareApplyResultSuccess = "success"
|
||||
)
|
||||
|
||||
// ListOpenFlareNodes returns all nodes ordered by id desc.
|
||||
func ListOpenFlareNodes(ctx context.Context) ([]model.OpenFlareNode, error) {
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return nil, errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
var nodes []model.OpenFlareNode
|
||||
if err := conn.Order("id desc").Find(&nodes).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return nodes, nil
|
||||
}
|
||||
|
||||
// ListOpenFlareNodesByNodeIDs returns nodes matching the given node ids.
|
||||
func ListOpenFlareNodesByNodeIDs(ctx context.Context, nodeIDs []string) ([]model.OpenFlareNode, error) {
|
||||
if len(nodeIDs) == 0 {
|
||||
return []model.OpenFlareNode{}, nil
|
||||
}
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return nil, errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
var nodes []model.OpenFlareNode
|
||||
if err := conn.Where("node_id IN ?", nodeIDs).Find(&nodes).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return nodes, nil
|
||||
}
|
||||
|
||||
// GetOpenFlareNodeByID returns a node by primary key.
|
||||
func GetOpenFlareNodeByID(ctx context.Context, id uint) (*model.OpenFlareNode, error) {
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return nil, errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
var node model.OpenFlareNode
|
||||
if err := conn.First(&node, id).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &node, nil
|
||||
}
|
||||
|
||||
// GetOpenFlareNodeByNodeID returns a node by node_id.
|
||||
func GetOpenFlareNodeByNodeID(ctx context.Context, nodeID string) (*model.OpenFlareNode, error) {
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return nil, errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
var node model.OpenFlareNode
|
||||
if err := conn.Where("node_id = ?", nodeID).First(&node).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &node, nil
|
||||
}
|
||||
|
||||
// GetOpenFlareNodeByAccessToken returns a node by access token.
|
||||
func GetOpenFlareNodeByAccessToken(ctx context.Context, token string) (*model.OpenFlareNode, error) {
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return nil, errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
var node model.OpenFlareNode
|
||||
if err := conn.Where("access_token = ?", token).First(&node).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &node, nil
|
||||
}
|
||||
|
||||
// CreateOpenFlareNode inserts a new node.
|
||||
func CreateOpenFlareNode(ctx context.Context, node *model.OpenFlareNode) error {
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
return conn.Create(node).Error
|
||||
}
|
||||
|
||||
// SaveOpenFlareNode persists node changes.
|
||||
func SaveOpenFlareNode(ctx context.Context, node *model.OpenFlareNode) error {
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
return conn.Save(node).Error
|
||||
}
|
||||
|
||||
// UpdateOpenFlareNodeFields updates selected columns for a node.
|
||||
func UpdateOpenFlareNodeFields(ctx context.Context, node *model.OpenFlareNode, fields ...string) error {
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
if len(fields) == 0 {
|
||||
return conn.Save(node).Error
|
||||
}
|
||||
return conn.Model(node).Select(fields).Updates(node).Error
|
||||
}
|
||||
|
||||
// UpdateOpenFlareNodeColumns updates node columns from a map of column values.
|
||||
// Empty maps are no-ops.
|
||||
func UpdateOpenFlareNodeColumns(ctx context.Context, node *model.OpenFlareNode, changes map[string]any) error {
|
||||
if node == nil || len(changes) == 0 {
|
||||
return nil
|
||||
}
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
return conn.Model(node).Updates(changes).Error
|
||||
}
|
||||
|
||||
// UpdateOpenFlareNodeFromApplyResult updates node status, last_seen, version and last_error after an apply report.
|
||||
// When applyResult is "success", current_version is set and last_error is cleared; otherwise last_error is set to message.
|
||||
func UpdateOpenFlareNodeFromApplyResult(ctx context.Context, nodeID, applyResult, version, message string, now time.Time) error {
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
return updateOpenFlareNodeFromApplyResultTx(conn, nodeID, applyResult, version, message, now)
|
||||
}
|
||||
|
||||
func updateOpenFlareNodeFromApplyResultTx(tx *gorm.DB, nodeID, applyResult, version, message string, now time.Time) error {
|
||||
record := &model.OpenFlareNode{}
|
||||
if err := tx.Where("node_id = ?", nodeID).First(record).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
record.Status = openFlareNodeStatusOnline
|
||||
lastSeen := now
|
||||
record.LastSeenAt = &lastSeen
|
||||
if applyResult == openFlareApplyResultSuccess {
|
||||
record.CurrentVersion = version
|
||||
record.LastError = ""
|
||||
} else {
|
||||
record.LastError = message
|
||||
}
|
||||
return tx.Model(record).Select("status", "last_seen_at", "current_version", "last_error").Updates(record).Error
|
||||
}
|
||||
|
||||
// DeleteOpenFlareNode removes a node by primary key.
|
||||
func DeleteOpenFlareNode(ctx context.Context, id uint) error {
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
return conn.Delete(&model.OpenFlareNode{}, id).Error
|
||||
}
|
||||
@@ -0,0 +1,644 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/model"
|
||||
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
|
||||
analyticsrepo "Wavelet/openflare/plugins/server/kernel/repository/analytics"
|
||||
"Wavelet/openflare/plugins/server/kernel/repository/logstore"
|
||||
"Wavelet/pkg/logger"
|
||||
db "Wavelet/plugins/infra/database"
|
||||
)
|
||||
|
||||
const (
|
||||
openFlareHealthEventStatusActive = "active"
|
||||
openFlareHealthEventStatusResolved = "resolved"
|
||||
openFlareHealthSeverityInfo = "info"
|
||||
openFlareHealthSeverityWarning = "warning"
|
||||
openFlareHealthSeverityCritical = "critical"
|
||||
openFlareHealthEventMessageMaxLen = 4096
|
||||
|
||||
// logStoreNameClickHouse 与 logstore 内部 dbNameClickHouse 取值一致。
|
||||
logStoreNameClickHouse = "clickhouse"
|
||||
)
|
||||
|
||||
// OpenFlareHealthEventInput describes a desired active health event for reconciliation.
|
||||
type OpenFlareHealthEventInput struct {
|
||||
EventType string
|
||||
Severity string
|
||||
Message string
|
||||
TriggeredAtUnix int64
|
||||
Metadata map[string]string
|
||||
}
|
||||
|
||||
func isMissingTableError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return false
|
||||
}
|
||||
msg := strings.ToLower(err.Error())
|
||||
return strings.Contains(msg, "no such table") ||
|
||||
strings.Contains(msg, "doesn't exist") ||
|
||||
strings.Contains(msg, "does not exist")
|
||||
}
|
||||
|
||||
// InsertOpenFlareMetricSnapshot inserts a metric snapshot into ClickHouse.
|
||||
func InsertOpenFlareMetricSnapshot(ctx context.Context, record *model.OpenFlareMetricSnapshot) error {
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.Observability.InsertMetricSnapshot(ctx, record)
|
||||
}
|
||||
|
||||
// InsertOpenFlareEdgeHealth inserts an L2 edge health snapshot into ClickHouse.
|
||||
func InsertOpenFlareEdgeHealth(ctx context.Context, record *model.OpenFlareEdgeHealth) error {
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.Observability.InsertEdgeHealth(ctx, record)
|
||||
}
|
||||
|
||||
// InsertOpenFlareNodeObservationFrps inserts an FRPS observation into ClickHouse.
|
||||
func InsertOpenFlareNodeObservationFrps(ctx context.Context, record *model.OpenFlareNodeObservationFrps) error {
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.Observability.InsertNodeObservationFrps(ctx, record)
|
||||
}
|
||||
|
||||
// InsertOpenFlareNodeObservationFrpc inserts an FRPC observation into ClickHouse.
|
||||
func InsertOpenFlareNodeObservationFrpc(ctx context.Context, record *model.OpenFlareNodeObservationFrpc) error {
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.Observability.InsertNodeObservationFrpc(ctx, record)
|
||||
}
|
||||
|
||||
// ListOpenFlareMetricSnapshotsSince returns metric snapshots since the given time.
|
||||
func ListOpenFlareMetricSnapshotsSince(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareMetricSnapshot, error) {
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.Observability.ListMetricSnapshots(ctx, nodeID, since, limit)
|
||||
}
|
||||
|
||||
// ListOpenFlareLatestMetricSnapshotsSince returns the latest metric snapshot per node.
|
||||
// The ClickHouse LIMIT 1 BY fast path is used only when ClickHouse is the ACTIVE log
|
||||
// database; otherwise the request goes straight to the active log store (PG/SQLite),
|
||||
// avoiding stale reads of the previous CH store after a migration.
|
||||
func ListOpenFlareLatestMetricSnapshotsSince(ctx context.Context, nodeID string, since time.Time) ([]*model.OpenFlareMetricSnapshot, error) {
|
||||
active, err := logstore.ActiveDatabase(ctx)
|
||||
if err != nil {
|
||||
logger.ErrorF(ctx, "failed to resolve active log database for latest metric snapshots: %v", err)
|
||||
} else if active == logStoreNameClickHouse {
|
||||
rows, chErr := analyticsrepo.ListLatestNodeMetricSnapshots(ctx, analyticsrepo.NodeObservabilityFilter{
|
||||
NodeID: nodeID,
|
||||
Since: since,
|
||||
})
|
||||
if chErr == nil {
|
||||
return fromAnalyticsNodeMetricSnapshots(rows), nil
|
||||
}
|
||||
logger.ErrorF(ctx, "clickhouse fast-path ListLatestNodeMetricSnapshots failed: %v", chErr)
|
||||
return nil, chErr
|
||||
}
|
||||
// Routes through the active log store (PG/SQLite active).
|
||||
all, listErr := ListOpenFlareMetricSnapshotsSince(ctx, nodeID, since, 0)
|
||||
if listErr != nil {
|
||||
return nil, listErr
|
||||
}
|
||||
return openFlareLatestMetricSnapshots(all), nil
|
||||
}
|
||||
|
||||
func openFlareLatestMetricSnapshots(snapshots []*model.OpenFlareMetricSnapshot) []*model.OpenFlareMetricSnapshot {
|
||||
latestByNode := make(map[string]*model.OpenFlareMetricSnapshot, len(snapshots))
|
||||
for _, snapshot := range snapshots {
|
||||
if snapshot == nil || snapshot.NodeID == "" {
|
||||
continue
|
||||
}
|
||||
if existing, ok := latestByNode[snapshot.NodeID]; ok && !snapshot.CapturedAt.After(existing.CapturedAt) {
|
||||
continue
|
||||
}
|
||||
latestByNode[snapshot.NodeID] = snapshot
|
||||
}
|
||||
result := make([]*model.OpenFlareMetricSnapshot, 0, len(latestByNode))
|
||||
for _, snapshot := range latestByNode {
|
||||
result = append(result, snapshot)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// fromAnalyticsNodeMetricSnapshots converts analytics rows back to the business model.
|
||||
func fromAnalyticsNodeMetricSnapshots(rows []analyticsmodel.NodeMetricSnapshot) []*model.OpenFlareMetricSnapshot {
|
||||
result := make([]*model.OpenFlareMetricSnapshot, len(rows))
|
||||
for index, row := range rows {
|
||||
result[index] = &model.OpenFlareMetricSnapshot{
|
||||
ID: uint(row.ID),
|
||||
NodeID: row.NodeID,
|
||||
CapturedAt: row.CapturedAt,
|
||||
CPUUsagePercent: row.CPUUsagePercent,
|
||||
MemoryUsedBytes: row.MemoryUsedBytes,
|
||||
MemoryTotalBytes: row.MemoryTotalBytes,
|
||||
StorageUsedBytes: row.StorageUsedBytes,
|
||||
StorageTotalBytes: row.StorageTotalBytes,
|
||||
DiskReadBytes: row.DiskReadBytes,
|
||||
DiskWriteBytes: row.DiskWriteBytes,
|
||||
NetworkRxBytes: row.NetworkRxBytes,
|
||||
NetworkTxBytes: row.NetworkTxBytes,
|
||||
CreatedAt: row.CreatedAt,
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// ListOpenFlareTrafficHourlySince returns hourly traffic rollup rows since the given time.
|
||||
// CH 读 of_access_log_hourly rollup;PG/SQLite 经 logstore 从 of_node_access_logs 实时聚合。
|
||||
func ListOpenFlareTrafficHourlySince(ctx context.Context, nodeID string, since time.Time) ([]*model.OpenFlareTrafficHourly, error) {
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows, err := s.Observability.ListTrafficHourly(ctx, nodeID, since)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := make([]*model.OpenFlareTrafficHourly, len(rows))
|
||||
for index, row := range rows {
|
||||
result[index] = &model.OpenFlareTrafficHourly{
|
||||
NodeID: row.NodeID,
|
||||
Hour: row.Hour,
|
||||
RequestCount: row.RequestCount,
|
||||
ErrorCount: row.ErrorCount,
|
||||
UniqueVisitorCount: row.UniqueVisitorCount,
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// ListOpenFlareAccessLogHourlySince returns hourly access-log rollups since the given time,
|
||||
// read through logstore's active backend (ClickHouse of_access_log_hourly rollup;
|
||||
// PostgreSQL/SQLite real-time aggregation from of_node_access_logs).
|
||||
func ListOpenFlareAccessLogHourlySince(ctx context.Context, nodeID string, since time.Time) ([]*model.OpenFlareAccessLogHourly, error) {
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows, err := s.Observability.ListAccessLogHourly(ctx, nodeID, since)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := make([]*model.OpenFlareAccessLogHourly, len(rows))
|
||||
for index, row := range rows {
|
||||
result[index] = &model.OpenFlareAccessLogHourly{
|
||||
NodeID: row.NodeID,
|
||||
Hour: row.Hour,
|
||||
Host: row.Host,
|
||||
RequestCount: row.RequestCount,
|
||||
ErrorCount: row.ErrorCount,
|
||||
BytesSent: row.BytesSent,
|
||||
RequestLength: row.RequestLength,
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// ListOpenFlareMetricHourlySince returns hourly metric aggregates since the given time.
|
||||
func ListOpenFlareMetricHourlySince(ctx context.Context, nodeID string, since time.Time) ([]*model.OpenFlareMetricHourly, error) {
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows, err := s.Observability.ListMetricHourly(ctx, nodeID, since)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := make([]*model.OpenFlareMetricHourly, len(rows))
|
||||
for index, row := range rows {
|
||||
result[index] = &model.OpenFlareMetricHourly{
|
||||
Hour: row.Hour,
|
||||
AverageCPUUsagePercent: row.AverageCPUUsagePercent,
|
||||
AverageMemoryUsagePercent: row.AverageMemoryUsagePercent,
|
||||
NetworkRxBytes: row.NetworkRxBytes,
|
||||
NetworkTxBytes: row.NetworkTxBytes,
|
||||
DiskReadBytes: row.DiskReadBytes,
|
||||
DiskWriteBytes: row.DiskWriteBytes,
|
||||
ReportedNodes: row.ReportedNodes,
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// ListOpenFlareActiveHealthEvents returns active health events across all nodes.
|
||||
func ListOpenFlareActiveHealthEvents(ctx context.Context) ([]*model.OpenFlareHealthEvent, error) {
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return nil, errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
var rows []*model.OpenFlareHealthEvent
|
||||
if err := conn.Where("status = ?", "active").Order("last_triggered_at desc").Find(&rows).Error; err != nil {
|
||||
if isMissingTableError(err) {
|
||||
return []*model.OpenFlareHealthEvent{}, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
// ListOpenFlareHealthEvents returns health events for a node.
|
||||
func ListOpenFlareHealthEvents(ctx context.Context, nodeID string, activeOnly bool, limit int) ([]*model.OpenFlareHealthEvent, error) {
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return nil, errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
query := conn.Model(&model.OpenFlareHealthEvent{}).Where("node_id = ?", nodeID).Order("last_triggered_at desc")
|
||||
if activeOnly {
|
||||
query = query.Where("status = ?", "active")
|
||||
}
|
||||
if limit > 0 {
|
||||
query = query.Limit(limit)
|
||||
}
|
||||
var rows []*model.OpenFlareHealthEvent
|
||||
if err := query.Find(&rows).Error; err != nil {
|
||||
if isMissingTableError(err) {
|
||||
return []*model.OpenFlareHealthEvent{}, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
// DeleteOpenFlareMetricSnapshotsBefore deletes metric snapshots captured before cutoff.
|
||||
func DeleteOpenFlareMetricSnapshotsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return s.Observability.DeleteMetricSnapshotsBefore(ctx, cutoff)
|
||||
}
|
||||
|
||||
// DeleteAllOpenFlareMetricSnapshots deletes all metric snapshots.
|
||||
func DeleteAllOpenFlareMetricSnapshots(ctx context.Context) (int64, error) {
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return s.Observability.DeleteAllMetricSnapshots(ctx)
|
||||
}
|
||||
|
||||
// DeleteOpenFlareEdgeHealthBefore deletes edge health rows captured before cutoff.
|
||||
func DeleteOpenFlareEdgeHealthBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return s.Observability.DeleteEdgeHealthBefore(ctx, cutoff)
|
||||
}
|
||||
|
||||
// DeleteAllOpenFlareEdgeHealth deletes all edge health snapshots.
|
||||
func DeleteAllOpenFlareEdgeHealth(ctx context.Context) (int64, error) {
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return s.Observability.DeleteAllEdgeHealth(ctx)
|
||||
}
|
||||
|
||||
// DeleteOpenFlareNodeObservationFrpsBefore deletes FRPS observations captured before cutoff.
|
||||
func DeleteOpenFlareNodeObservationFrpsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return s.Observability.DeleteNodeObservationFrpsBefore(ctx, cutoff)
|
||||
}
|
||||
|
||||
// DeleteAllOpenFlareNodeObservationFrps deletes all FRPS observations.
|
||||
func DeleteAllOpenFlareNodeObservationFrps(ctx context.Context) (int64, error) {
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return s.Observability.DeleteAllNodeObservationFrps(ctx)
|
||||
}
|
||||
|
||||
// DeleteOpenFlareNodeObservationFrpcBefore deletes FRPC observations captured before cutoff.
|
||||
func DeleteOpenFlareNodeObservationFrpcBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return s.Observability.DeleteNodeObservationFrpcBefore(ctx, cutoff)
|
||||
}
|
||||
|
||||
// DeleteAllOpenFlareNodeObservationFrpc deletes all FRPC observations.
|
||||
func DeleteAllOpenFlareNodeObservationFrpc(ctx context.Context) (int64, error) {
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return s.Observability.DeleteAllNodeObservationFrpc(ctx)
|
||||
}
|
||||
|
||||
// DeleteOpenFlareHealthEventsByNodeID deletes all health events for a node.
|
||||
func DeleteOpenFlareHealthEventsByNodeID(ctx context.Context, nodeID string) (int64, error) {
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return 0, errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
result := conn.Where("node_id = ?", nodeID).Delete(&model.OpenFlareHealthEvent{})
|
||||
if result.Error != nil {
|
||||
if isMissingTableError(result.Error) {
|
||||
return 0, nil
|
||||
}
|
||||
return 0, result.Error
|
||||
}
|
||||
return result.RowsAffected, nil
|
||||
}
|
||||
|
||||
// GetOpenFlareNodeSystemProfile returns the system profile for a node.
|
||||
func GetOpenFlareNodeSystemProfile(ctx context.Context, nodeID string) (*model.OpenFlareNodeSystemProfile, error) {
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return nil, errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
var profile model.OpenFlareNodeSystemProfile
|
||||
if err := conn.Where("node_id = ?", nodeID).First(&profile).Error; err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) || isMissingTableError(err) {
|
||||
return nil, gorm.ErrRecordNotFound
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return &profile, nil
|
||||
}
|
||||
|
||||
// UpsertOpenFlareNodeSystemProfile inserts or updates the latest system profile for a node.
|
||||
func UpsertOpenFlareNodeSystemProfile(ctx context.Context, record *model.OpenFlareNodeSystemProfile) error {
|
||||
if record == nil {
|
||||
return nil
|
||||
}
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
return upsertOpenFlareNodeSystemProfileTx(conn, record)
|
||||
}
|
||||
|
||||
func upsertOpenFlareNodeSystemProfileTx(tx *gorm.DB, record *model.OpenFlareNodeSystemProfile) error {
|
||||
return tx.Clauses(clause.OnConflict{
|
||||
Columns: []clause.Column{{Name: "node_id"}},
|
||||
DoUpdates: clause.AssignmentColumns([]string{
|
||||
"hostname",
|
||||
"os_name",
|
||||
"os_version",
|
||||
"kernel_version",
|
||||
"architecture",
|
||||
"cpu_model",
|
||||
"cpu_cores",
|
||||
"total_memory_bytes",
|
||||
"total_disk_bytes",
|
||||
"uptime_seconds",
|
||||
"reported_at",
|
||||
"updated_at",
|
||||
}),
|
||||
}).Create(record).Error
|
||||
}
|
||||
|
||||
// ReconcileOpenFlareHealthEvents reconciles active health events for a node.
|
||||
// Desired active events are created or updated; previously active types not present are resolved.
|
||||
// When managedEventTypes is non-empty, only those event types are considered.
|
||||
// Runs inside a transaction so multi-row create/update/resolve stays atomic.
|
||||
func ReconcileOpenFlareHealthEvents(
|
||||
ctx context.Context,
|
||||
nodeID string,
|
||||
events []OpenFlareHealthEventInput,
|
||||
reportedAt time.Time,
|
||||
managedEventTypes map[string]struct{},
|
||||
) error {
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
return conn.Transaction(func(tx *gorm.DB) error {
|
||||
return reconcileOpenFlareHealthEventsTx(tx, nodeID, events, reportedAt, managedEventTypes)
|
||||
})
|
||||
}
|
||||
|
||||
// PersistOpenFlareNodePGObservability upserts an optional system profile and optionally reconciles
|
||||
// health events in a single transaction (Postgres-side heartbeat observability).
|
||||
// When reconcileHealth is false, health events are left untouched.
|
||||
func PersistOpenFlareNodePGObservability(
|
||||
ctx context.Context,
|
||||
profile *model.OpenFlareNodeSystemProfile,
|
||||
nodeID string,
|
||||
events []OpenFlareHealthEventInput,
|
||||
reconcileHealth bool,
|
||||
reportedAt time.Time,
|
||||
managedEventTypes map[string]struct{},
|
||||
) error {
|
||||
if profile == nil && !reconcileHealth {
|
||||
return nil
|
||||
}
|
||||
conn := db.DB(ctx)
|
||||
if conn == nil {
|
||||
return errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
return conn.Transaction(func(tx *gorm.DB) error {
|
||||
if profile != nil {
|
||||
if err := upsertOpenFlareNodeSystemProfileTx(tx, profile); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if reconcileHealth {
|
||||
if err := reconcileOpenFlareHealthEventsTx(tx, nodeID, events, reportedAt, managedEventTypes); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func reconcileOpenFlareHealthEventsTx(
|
||||
tx *gorm.DB,
|
||||
nodeID string,
|
||||
events []OpenFlareHealthEventInput,
|
||||
reportedAt time.Time,
|
||||
managedEventTypes map[string]struct{},
|
||||
) error {
|
||||
activeTypes := make(map[string]OpenFlareHealthEventInput, len(events))
|
||||
for _, event := range events {
|
||||
eventType := normalizeOpenFlareHealthEventType(event.EventType)
|
||||
if eventType == "" {
|
||||
continue
|
||||
}
|
||||
if len(managedEventTypes) > 0 {
|
||||
if _, ok := managedEventTypes[eventType]; !ok {
|
||||
continue
|
||||
}
|
||||
}
|
||||
event.EventType = eventType
|
||||
event.Severity = normalizeOpenFlareHealthSeverity(event.Severity)
|
||||
if event.TriggeredAtUnix <= 0 {
|
||||
event.TriggeredAtUnix = reportedAt.Unix()
|
||||
}
|
||||
activeTypes[eventType] = event
|
||||
}
|
||||
|
||||
var activeEvents []*model.OpenFlareHealthEvent
|
||||
query := tx.Where("node_id = ? AND status = ?", nodeID, openFlareHealthEventStatusActive)
|
||||
if len(managedEventTypes) > 0 {
|
||||
scopedTypes := make([]string, 0, len(managedEventTypes))
|
||||
for eventType := range managedEventTypes {
|
||||
eventType = normalizeOpenFlareHealthEventType(eventType)
|
||||
if eventType != "" {
|
||||
scopedTypes = append(scopedTypes, eventType)
|
||||
}
|
||||
}
|
||||
if len(scopedTypes) == 0 {
|
||||
return nil
|
||||
}
|
||||
query = query.Where("event_type IN ?", scopedTypes)
|
||||
}
|
||||
if err := query.Find(&activeEvents).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
activeByType := make(map[string]*model.OpenFlareHealthEvent, len(activeEvents))
|
||||
for _, event := range activeEvents {
|
||||
activeByType[event.EventType] = event
|
||||
}
|
||||
|
||||
for eventType, event := range activeTypes {
|
||||
triggeredAt := timeFromUnixSeconds(event.TriggeredAtUnix, reportedAt)
|
||||
if existing, ok := activeByType[eventType]; ok {
|
||||
existing.Severity = event.Severity
|
||||
existing.Message = normalizeOpenFlareHealthEventMessage(event.Message)
|
||||
existing.LastTriggeredAt = triggeredAt
|
||||
existing.ReportedAt = reportedAt
|
||||
existing.MetadataJSON = marshalOpenFlareHealthMetadata(event.Metadata)
|
||||
existing.ResolvedAt = nil
|
||||
if err := tx.Save(existing).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
record := &model.OpenFlareHealthEvent{
|
||||
NodeID: nodeID,
|
||||
EventType: eventType,
|
||||
Severity: event.Severity,
|
||||
Status: openFlareHealthEventStatusActive,
|
||||
Message: normalizeOpenFlareHealthEventMessage(event.Message),
|
||||
FirstTriggeredAt: triggeredAt,
|
||||
LastTriggeredAt: triggeredAt,
|
||||
ReportedAt: reportedAt,
|
||||
MetadataJSON: marshalOpenFlareHealthMetadata(event.Metadata),
|
||||
}
|
||||
if err := tx.Create(record).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
for _, existing := range activeEvents {
|
||||
if _, ok := activeTypes[existing.EventType]; ok {
|
||||
continue
|
||||
}
|
||||
resolvedAt := reportedAt
|
||||
existing.Status = openFlareHealthEventStatusResolved
|
||||
existing.ReportedAt = reportedAt
|
||||
existing.ResolvedAt = &resolvedAt
|
||||
if err := tx.Save(existing).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func normalizeOpenFlareHealthEventType(eventType string) string {
|
||||
eventType = strings.TrimSpace(strings.ToLower(eventType))
|
||||
eventType = strings.ReplaceAll(eventType, " ", "_")
|
||||
return eventType
|
||||
}
|
||||
|
||||
func normalizeOpenFlareHealthSeverity(severity string) string {
|
||||
switch strings.ToLower(strings.TrimSpace(severity)) {
|
||||
case openFlareHealthSeverityCritical:
|
||||
return openFlareHealthSeverityCritical
|
||||
case openFlareHealthSeverityInfo:
|
||||
return openFlareHealthSeverityInfo
|
||||
default:
|
||||
return openFlareHealthSeverityWarning
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeOpenFlareHealthEventMessage(message string) string {
|
||||
if openFlareHealthEventMessageMaxLen <= 0 {
|
||||
return ""
|
||||
}
|
||||
runes := []rune(strings.TrimSpace(message))
|
||||
if len(runes) <= openFlareHealthEventMessageMaxLen {
|
||||
return string(runes)
|
||||
}
|
||||
return string(runes[:openFlareHealthEventMessageMaxLen])
|
||||
}
|
||||
|
||||
func timeFromUnixSeconds(unixSeconds int64, fallback time.Time) time.Time {
|
||||
if unixSeconds <= 0 {
|
||||
return fallback
|
||||
}
|
||||
return time.Unix(unixSeconds, 0).UTC()
|
||||
}
|
||||
|
||||
func marshalOpenFlareHealthMetadata(value map[string]string) string {
|
||||
if value == nil {
|
||||
return ""
|
||||
}
|
||||
raw, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return string(raw)
|
||||
}
|
||||
|
||||
// ListOpenFlareEdgeHealth returns L2 edge health snapshots.
|
||||
func ListOpenFlareEdgeHealth(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareEdgeHealth, error) {
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.Observability.ListEdgeHealth(ctx, nodeID, since, limit)
|
||||
}
|
||||
|
||||
// ListOpenFlareNodeObservationFrpc returns frpc observations.
|
||||
func ListOpenFlareNodeObservationFrpc(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareNodeObservationFrpc, error) {
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.Observability.ListNodeObservationFrpc(ctx, nodeID, since, limit)
|
||||
}
|
||||
|
||||
// ListOpenFlareNodeObservationFrps returns frps observations.
|
||||
func ListOpenFlareNodeObservationFrps(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareNodeObservationFrps, error) {
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.Observability.ListNodeObservationFrps(ctx, nodeID, since, limit)
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"gorm.io/gorm"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/model"
|
||||
db "Wavelet/plugins/infra/database"
|
||||
)
|
||||
|
||||
// WithOriginTx runs fn inside a database transaction for origin multi-step work.
|
||||
func WithOriginTx(ctx context.Context, fn func(tx *gorm.DB) error) error {
|
||||
return db.DB(ctx).Transaction(fn)
|
||||
}
|
||||
|
||||
// HasProxyRoutesTable 判断代理规则表是否已迁移。
|
||||
func HasProxyRoutesTable(ctx context.Context) bool {
|
||||
return db.DB(ctx).Migrator().HasTable(&model.OriginProxyRoute{})
|
||||
}
|
||||
|
||||
// ListOrigins 列出全部源站。
|
||||
func ListOrigins(ctx context.Context) ([]model.Origin, error) {
|
||||
var origins []model.Origin
|
||||
if err := db.DB(ctx).Order("id desc").Find(&origins).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return origins, nil
|
||||
}
|
||||
|
||||
// GetOriginByID 按 ID 查询源站。
|
||||
func GetOriginByID(ctx context.Context, id uint) (*model.Origin, error) {
|
||||
var origin model.Origin
|
||||
if err := db.DB(ctx).First(&origin, id).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &origin, nil
|
||||
}
|
||||
|
||||
// GetOriginByAddress 按地址查询源站。
|
||||
func GetOriginByAddress(ctx context.Context, address string) (*model.Origin, error) {
|
||||
var origin model.Origin
|
||||
if err := db.DB(ctx).Where("address = ?", address).First(&origin).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &origin, nil
|
||||
}
|
||||
|
||||
// CreateOriginRecord 创建源站。
|
||||
func CreateOriginRecord(ctx context.Context, origin *model.Origin) error {
|
||||
return db.DB(ctx).Create(origin).Error
|
||||
}
|
||||
|
||||
// SaveOrigin 保存源站。
|
||||
func SaveOrigin(ctx context.Context, origin *model.Origin) error {
|
||||
return SaveOriginTx(db.DB(ctx), origin)
|
||||
}
|
||||
|
||||
// SaveOriginTx saves an origin within an existing transaction.
|
||||
func SaveOriginTx(tx *gorm.DB, origin *model.Origin) error {
|
||||
return tx.Save(origin).Error
|
||||
}
|
||||
|
||||
// DeleteOriginRecord 删除源站。
|
||||
func DeleteOriginRecord(ctx context.Context, id uint) error {
|
||||
return db.DB(ctx).Delete(&model.Origin{}, id).Error
|
||||
}
|
||||
|
||||
// ListOriginRouteCounts 统计各源站关联的代理规则数量。
|
||||
func ListOriginRouteCounts(ctx context.Context) ([]model.OriginRouteCount, error) {
|
||||
if !HasProxyRoutesTable(ctx) {
|
||||
return nil, nil
|
||||
}
|
||||
result := make([]model.OriginRouteCount, 0)
|
||||
err := db.DB(ctx).Model(&model.OriginProxyRoute{}).
|
||||
Select("origin_id, COUNT(*) AS route_count").
|
||||
Where("origin_id IS NOT NULL").
|
||||
Group("origin_id").
|
||||
Scan(&result).Error
|
||||
return result, err
|
||||
}
|
||||
|
||||
// ListProxyRoutesByOriginID 列出源站关联的代理规则。
|
||||
func ListProxyRoutesByOriginID(ctx context.Context, originID uint) ([]model.OriginProxyRoute, error) {
|
||||
if !HasProxyRoutesTable(ctx) {
|
||||
return nil, nil
|
||||
}
|
||||
var routes []model.OriginProxyRoute
|
||||
if err := db.DB(ctx).Where("origin_id = ?", originID).Order("id desc").Find(&routes).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return routes, nil
|
||||
}
|
||||
|
||||
// ListProxyRoutesByOriginIDAscTx lists origin-linked proxy routes ordered by id asc within a transaction.
|
||||
func ListProxyRoutesByOriginIDAscTx(tx *gorm.DB, originID uint) ([]model.OriginProxyRoute, error) {
|
||||
var routes []model.OriginProxyRoute
|
||||
if err := tx.Where("origin_id = ?", originID).Order("id asc").Find(&routes).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return routes, nil
|
||||
}
|
||||
|
||||
// UpdateProxyRouteOriginAddressTx updates a proxy route's origin_url and upstreams within a transaction.
|
||||
func UpdateProxyRouteOriginAddressTx(tx *gorm.DB, routeID uint, originURL, upstreamsJSON string) error {
|
||||
return tx.Model(&model.OriginProxyRoute{}).
|
||||
Where("id = ?", routeID).
|
||||
Updates(map[string]any{
|
||||
"origin_url": originURL,
|
||||
"upstreams": upstreamsJSON,
|
||||
}).Error
|
||||
}
|
||||
|
||||
// CountProxyRoutesByOriginID 统计源站关联的代理规则数量。
|
||||
func CountProxyRoutesByOriginID(ctx context.Context, originID uint) (int64, error) {
|
||||
if !HasProxyRoutesTable(ctx) {
|
||||
return 0, nil
|
||||
}
|
||||
var count int64
|
||||
if err := db.DB(ctx).Model(&model.OriginProxyRoute{}).Where("origin_id = ?", originID).Count(&count).Error; err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/model"
|
||||
db "Wavelet/plugins/infra/database"
|
||||
)
|
||||
|
||||
// HasPagesProjectsTable 判断 Pages 项目表是否已迁移。
|
||||
func HasPagesProjectsTable(ctx context.Context) bool {
|
||||
return db.DB(ctx).Migrator().HasTable(&model.PagesProject{})
|
||||
}
|
||||
|
||||
// ListPagesProjects 列出全部 Pages 项目。
|
||||
func ListPagesProjects(ctx context.Context) ([]model.PagesProject, error) {
|
||||
var projects []model.PagesProject
|
||||
if err := db.DB(ctx).Order("id desc").Find(&projects).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return projects, nil
|
||||
}
|
||||
|
||||
// GetPagesProjectByID 按 ID 查询 Pages 项目。
|
||||
func GetPagesProjectByID(ctx context.Context, id uint) (*model.PagesProject, error) {
|
||||
var project model.PagesProject
|
||||
if err := db.DB(ctx).First(&project, id).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &project, nil
|
||||
}
|
||||
|
||||
// GetPagesProjectBySlug 按 slug 查询 Pages 项目。
|
||||
func GetPagesProjectBySlug(ctx context.Context, slug string) (*model.PagesProject, error) {
|
||||
var project model.PagesProject
|
||||
if err := db.DB(ctx).Where("slug = ?", slug).First(&project).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &project, nil
|
||||
}
|
||||
|
||||
// CreatePagesProjectRecord 创建 Pages 项目。
|
||||
func CreatePagesProjectRecord(ctx context.Context, project *model.PagesProject) error {
|
||||
return db.DB(ctx).Create(project).Error
|
||||
}
|
||||
|
||||
// ListPagesDeployments 列出项目的全部部署。
|
||||
func ListPagesDeployments(ctx context.Context, projectID uint) ([]model.PagesDeployment, error) {
|
||||
var deployments []model.PagesDeployment
|
||||
if err := db.DB(ctx).Where("project_id = ?", projectID).Order("id desc").Find(&deployments).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return deployments, nil
|
||||
}
|
||||
|
||||
// GetPagesDeploymentByID 按 ID 查询 Pages 部署。
|
||||
func GetPagesDeploymentByID(ctx context.Context, id uint) (*model.PagesDeployment, error) {
|
||||
var deployment model.PagesDeployment
|
||||
if err := db.DB(ctx).First(&deployment, id).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &deployment, nil
|
||||
}
|
||||
|
||||
// ListPagesDeploymentFiles 列出部署文件清单。
|
||||
func ListPagesDeploymentFiles(ctx context.Context, deploymentID uint) ([]model.PagesDeploymentFile, error) {
|
||||
var files []model.PagesDeploymentFile
|
||||
if err := db.DB(ctx).Where("deployment_id = ?", deploymentID).Order("path asc").Find(&files).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return files, nil
|
||||
}
|
||||
|
||||
// CountPagesDeploymentsByProjectID 统计项目部署数量。
|
||||
func CountPagesDeploymentsByProjectID(ctx context.Context, projectID uint) (int64, error) {
|
||||
var count int64
|
||||
if err := db.DB(ctx).Model(&model.PagesDeployment{}).Where("project_id = ?", projectID).Count(&count).Error; err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
|
||||
// CountProxyRoutesByPagesProjectID 统计引用 Pages 项目的代理规则数量。
|
||||
func CountProxyRoutesByPagesProjectID(ctx context.Context, projectID uint) (int64, error) {
|
||||
if !HasProxyRoutesTable(ctx) {
|
||||
return 0, nil
|
||||
}
|
||||
var count int64
|
||||
if err := db.DB(ctx).Model(&model.ProxyRoute{}).Where("pages_project_id = ?", projectID).Count(&count).Error; err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user