mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-11-16 14:54:23 +08:00
fix(agent): make discovery enrollment compatible and idempotent (#53)
* fix(agent): make discovery enrollment compatible and idempotent * fix(agent): confirm enrollment persistence and document token aliases
This commit is contained in:
@@ -0,0 +1,223 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/apps/agent/config"
|
||||
agentheartbeat "github.com/Rain-kl/Wavelet/internal/apps/agent/heartbeat"
|
||||
"github.com/Rain-kl/Wavelet/internal/apps/agent/httpclient"
|
||||
"github.com/Rain-kl/Wavelet/internal/apps/agent/protocol"
|
||||
"github.com/Rain-kl/Wavelet/internal/apps/agent/state"
|
||||
serveragent "github.com/Rain-kl/Wavelet/internal/apps/openflare/agent"
|
||||
db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
"github.com/Rain-kl/Wavelet/internal/repository"
|
||||
"github.com/Rain-kl/Wavelet/internal/shared/response"
|
||||
"github.com/Rain-kl/Wavelet/internal/testhelper"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/glebarez/sqlite"
|
||||
"github.com/pressly/goose/v3"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func newDiscoveryTestRunner(t *testing.T, configPath string) (*Runner, string) {
|
||||
t.Helper()
|
||||
cfg, err := config.Load(configPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store := state.NewStore(cfg.StatePath)
|
||||
nodeID, err := store.EnsureNodeID()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client := httpclient.New(cfg.ServerURL, cfg.InitialAuthToken(), time.Second)
|
||||
runner := withHeartbeatCycle(&Runner{Config: cfg, StateStore: store,
|
||||
HeartbeatService: agentheartbeat.New(client), SyncService: &fakeSyncService{}}, nil)
|
||||
return runner, nodeID
|
||||
}
|
||||
|
||||
func setupDiscoveryIntegrationDB(t *testing.T) *gorm.DB {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
conn, err := gorm.Open(sqlite.Open(filepath.Join(t.TempDir(), "discovery.db")), &gorm.Config{DisableForeignKeyConstraintWhenMigrating: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sqlDB, err := conn.DB()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
previous := db.DB(ctx)
|
||||
t.Cleanup(func() {
|
||||
if invalidateErr := repository.InvalidateAllSystemConfigCaches(ctx); invalidateErr != nil {
|
||||
t.Error(invalidateErr)
|
||||
}
|
||||
db.SetDB(previous)
|
||||
if closeErr := sqlDB.Close(); closeErr != nil {
|
||||
t.Error(closeErr)
|
||||
}
|
||||
})
|
||||
provider, err := goose.NewProvider(goose.DialectSQLite3, sqlDB, os.DirFS("../../../infra/persistence/migrator/goose/sqlite"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = provider.Up(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
db.SetDB(conn)
|
||||
return conn
|
||||
}
|
||||
|
||||
func TestDiscoveryHTTPRecoveryAndRestart(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
conn := setupDiscoveryIntegrationDB(t)
|
||||
var err error
|
||||
if err = repository.SaveOrUpdateSystemConfig(ctx, model.ConfigKeyAgentDiscoveryToken, "discovery-test"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = repository.SaveOrUpdateSystemConfig(ctx, model.ConfigKeyGeoIPProvider, "disabled"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var registrations, heartbeats atomic.Int32
|
||||
var dropResponse atomic.Bool
|
||||
dropResponse.Store(true)
|
||||
router := testhelper.NewTestGinEngine()
|
||||
router.POST("/api/v1/agent/nodes/register", serveragent.RegisterAuth(), func(c *gin.Context) {
|
||||
registrations.Add(1)
|
||||
serveragent.RegisterHandler(c)
|
||||
})
|
||||
router.POST("/api/v1/agent/nodes/heartbeat", serveragent.Auth(), func(c *gin.Context) {
|
||||
var payload map[string]json.RawMessage
|
||||
if decodeErr := c.ShouldBindJSON(&payload); decodeErr != nil {
|
||||
t.Error(decodeErr)
|
||||
}
|
||||
if _, exists := payload["registration_key"]; exists {
|
||||
t.Error("heartbeat exposed registration_key")
|
||||
}
|
||||
heartbeats.Add(1)
|
||||
c.JSON(http.StatusOK, response.OKNil())
|
||||
})
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/api/v1/agent/nodes/register" && dropResponse.Swap(false) {
|
||||
recorder := httptest.NewRecorder()
|
||||
router.ServeHTTP(recorder, r)
|
||||
if recorder.Code != http.StatusOK {
|
||||
t.Errorf("First registration status = %d, want 200", recorder.Code)
|
||||
}
|
||||
w.WriteHeader(http.StatusBadGateway) // Enrollment committed, but its response was lost.
|
||||
return
|
||||
}
|
||||
router.ServeHTTP(w, r)
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
configPath := filepath.Join(t.TempDir(), "agent.json")
|
||||
data, err := json.Marshal(map[string]string{"server_url": server.URL, "discovery_token": "discovery-test", "node_ip": "10.0.0.8", "node_name": "edge"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = os.WriteFile(configPath, data, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
first, nodeID := newDiscoveryTestRunner(t, configPath)
|
||||
if err = first.tryRegister(ctx, &nodeID); err == nil {
|
||||
t.Fatal("tryRegister(lost response) succeeded, want transport failure")
|
||||
}
|
||||
key, err := first.StateStore.EnsureRegistrationKey()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if first.hasAccessToken() {
|
||||
t.Fatal("tryRegister(lost response) activated token")
|
||||
}
|
||||
|
||||
// Restart with the same private key, then fail the second durable identity write.
|
||||
second, secondID := newDiscoveryTestRunner(t, configPath)
|
||||
backup := configPath + ".original"
|
||||
if err = os.Rename(configPath, backup); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = os.Mkdir(configPath, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = second.tryRegister(ctx, &secondID); err == nil {
|
||||
t.Fatal("tryRegister(config save failure) succeeded, want write error")
|
||||
}
|
||||
if second.hasAccessToken() || second.Config.DiscoveryToken == "" {
|
||||
t.Fatal("Config write failure activated partial in-memory credentials")
|
||||
}
|
||||
if err = os.Remove(configPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = os.Rename(backup, configPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
third, thirdID := newDiscoveryTestRunner(t, configPath)
|
||||
if err = third.tryRegister(ctx, &thirdID); err != nil {
|
||||
t.Fatalf("tryRegister(after restart) error = %v", err)
|
||||
}
|
||||
if !third.hasAccessToken() || third.Config.DiscoveryToken != "" {
|
||||
t.Fatal("Successful enrollment did not activate node credentials")
|
||||
}
|
||||
persistedKey, err := third.StateStore.EnsureRegistrationKey()
|
||||
if err != nil || persistedKey != key {
|
||||
t.Errorf("Retry key changed across partial-write restarts: %v", err)
|
||||
}
|
||||
|
||||
restarted, restartedID := newDiscoveryTestRunner(t, configPath)
|
||||
restarted.runStartupAuth(ctx, &restartedID)
|
||||
if restartedID != thirdID {
|
||||
t.Errorf("Restarted node ID = %q, want %q", restartedID, thirdID)
|
||||
}
|
||||
if registrations.Load() != 3 || heartbeats.Load() != 2 {
|
||||
t.Errorf("Enrollment/restart calls = %d registrations, %d heartbeats; want 3, 2", registrations.Load(), heartbeats.Load())
|
||||
}
|
||||
var count int64
|
||||
if err = conn.Model(&model.OpenFlareNode{}).Count(&count).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if count != 1 {
|
||||
t.Errorf("Enrollment through lost response and partial write created %d nodes, want 1", count)
|
||||
}
|
||||
|
||||
// Knowing the private retry key alone is insufficient after the global discovery token rotates.
|
||||
if err = repository.SaveOrUpdateSystemConfig(ctx, model.ConfigKeyAgentDiscoveryToken, "rotated-discovery"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
request := protocol.RegisterNodeRequest{NodePayload: protocol.NodePayload{IP: "10.0.0.8", Version: "test"}, RegistrationKey: key}
|
||||
for _, bearer := range []string{"", "invalid", "discovery-test"} {
|
||||
client := httpclient.New(server.URL, bearer, time.Second)
|
||||
result, authErr := client.RegisterNode(ctx, request)
|
||||
if authErr == nil || result != nil {
|
||||
t.Errorf("RegisterNode(unauthorized bearer) = %+v, %v; want rejection", result, authErr)
|
||||
}
|
||||
}
|
||||
encoded, err := json.Marshal(request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
unauthorized := httptest.NewRecorder()
|
||||
router.ServeHTTP(unauthorized, httptest.NewRequest(http.MethodPost, "/api/v1/agent/nodes/register", strings.NewReader(string(encoded))))
|
||||
if unauthorized.Code != http.StatusUnauthorized || strings.Contains(unauthorized.Body.String(), third.Config.AccessToken) {
|
||||
t.Errorf("Unauthorized retry response status = %d; want 401 without credential", unauthorized.Code)
|
||||
}
|
||||
client := httpclient.New(server.URL, "rotated-discovery", time.Second)
|
||||
result, err := client.RegisterNode(ctx, request)
|
||||
if err != nil || result.AccessToken != third.Config.AccessToken {
|
||||
t.Errorf("RegisterNode(current discovery bearer) error = %v, want same identity", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/apps/agent/config"
|
||||
"github.com/Rain-kl/Wavelet/internal/apps/agent/protocol"
|
||||
"github.com/Rain-kl/Wavelet/internal/apps/agent/state"
|
||||
)
|
||||
|
||||
func TestDiscoveryDoesNotRegisterBeforeRetrySecretPersists(t *testing.T) {
|
||||
blockedParent := filepath.Join(t.TempDir(), "file")
|
||||
if err := os.WriteFile(blockedParent, []byte("not a directory"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
service := &fakeHeartbeatService{}
|
||||
runner := &Runner{Config: &config.Config{DiscoveryToken: "discovery"},
|
||||
StateStore: state.NewStore(filepath.Join(blockedParent, "state.json")), HeartbeatService: service}
|
||||
nodeID := "local"
|
||||
if err := runner.tryRegister(context.Background(), &nodeID); err == nil {
|
||||
t.Fatal("tryRegister(unwritable state) succeeded, want persistence error")
|
||||
}
|
||||
if service.registerCalls != 0 {
|
||||
t.Errorf("tryRegister(unwritable retry key) sent %d requests, want 0", service.registerCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscoveryStateWriteFailureDoesNotActivateCredentials(t *testing.T) {
|
||||
configPath := filepath.Join(t.TempDir(), "agent.json")
|
||||
if err := os.WriteFile(configPath, []byte(`{"server_url":"http://127.0.0.1:3000","discovery_token":"discovery","node_ip":"10.0.0.8"}`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runner, nodeID := newDiscoveryTestRunner(t, configPath)
|
||||
statePath := runner.Config.StatePath
|
||||
backup := statePath + ".original"
|
||||
service := &fakeHeartbeatService{registerResp: &protocol.RegisterNodeResponse{NodeID: "server-assigned", AccessToken: "issued"}}
|
||||
var originalKey string
|
||||
service.onRegister = func() {
|
||||
snapshot, err := runner.StateStore.Load()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
originalKey = snapshot.RegistrationKey
|
||||
if err = os.Rename(statePath, backup); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = os.Mkdir(statePath, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
runner.HeartbeatService = service
|
||||
runner.HeartbeatCycle.Heartbeat = service
|
||||
if err := runner.tryRegister(context.Background(), &nodeID); err == nil {
|
||||
t.Fatal("tryRegister(state write failure) succeeded, want error")
|
||||
}
|
||||
if runner.hasAccessToken() || service.lastToken != "" {
|
||||
t.Fatal("Failed state persistence activated credentials")
|
||||
}
|
||||
if err := os.Remove(statePath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Rename(backup, statePath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
restarted, restartedID := newDiscoveryTestRunner(t, configPath)
|
||||
service.onRegister = nil
|
||||
restarted.HeartbeatService = service
|
||||
restarted.HeartbeatCycle.Heartbeat = service
|
||||
if err := restarted.tryRegister(context.Background(), &restartedID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, err := restarted.StateStore.EnsureRegistrationKey()
|
||||
if err != nil || key != originalKey {
|
||||
t.Errorf("Key after state write failure/restart changed: %v", err)
|
||||
}
|
||||
if restartedID != "server-assigned" || service.lastToken != "issued" {
|
||||
t.Errorf("Recovered identity/token = %q/%q, want server-assigned/issued", restartedID, service.lastToken)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package agent
|
||||
|
||||
import "time"
|
||||
|
||||
const (
|
||||
discoveryRetryInitialDelay = 3 * time.Second
|
||||
discoveryRetryMaxDelay = time.Minute
|
||||
discoveryRetryMultiplier = 2
|
||||
)
|
||||
|
||||
type discoveryRetry struct {
|
||||
delay time.Duration
|
||||
next time.Time
|
||||
}
|
||||
|
||||
func (r *discoveryRetry) ready(now time.Time) bool {
|
||||
return !now.Before(r.next)
|
||||
}
|
||||
|
||||
func (r *discoveryRetry) failed(now time.Time) {
|
||||
if r.delay == 0 {
|
||||
r.delay = discoveryRetryInitialDelay
|
||||
} else {
|
||||
r.delay = min(r.delay*discoveryRetryMultiplier, discoveryRetryMaxDelay)
|
||||
}
|
||||
r.next = now.Add(r.delay)
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package agent
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestDiscoveryRetryBackoffIsIndependentOfHeartbeat(t *testing.T) {
|
||||
now := time.Unix(100, 0)
|
||||
var retry discoveryRetry
|
||||
for _, delay := range []time.Duration{3, 6, 12, 24, 48, 60, 60} {
|
||||
if !retry.ready(now) {
|
||||
t.Fatal("Discovery retry not ready at scheduled time")
|
||||
}
|
||||
retry.failed(now)
|
||||
wantNext := now.Add(delay * time.Second)
|
||||
if retry.next != wantNext {
|
||||
t.Errorf("Discovery retry next = %v, want %v", retry.next, wantNext)
|
||||
}
|
||||
for simulatedHeartbeat := now; simulatedHeartbeat.Before(wantNext); simulatedHeartbeat = simulatedHeartbeat.Add(time.Millisecond) {
|
||||
if retry.ready(simulatedHeartbeat) {
|
||||
t.Fatal("Discovery retry ready before independent enrollment backoff expires")
|
||||
}
|
||||
}
|
||||
now = wantNext
|
||||
}
|
||||
}
|
||||
@@ -23,7 +23,7 @@ import (
|
||||
|
||||
// HeartbeatService handles node registration and periodic heartbeat reporting.
|
||||
type HeartbeatService interface {
|
||||
Register(ctx context.Context, payload protocol.NodePayload) (*protocol.RegisterNodeResponse, error)
|
||||
Register(ctx context.Context, payload protocol.RegisterNodeRequest) (*protocol.RegisterNodeResponse, error)
|
||||
Heartbeat(ctx context.Context, payload protocol.NodePayload) (*protocol.HeartbeatResult, error)
|
||||
SetToken(token string)
|
||||
}
|
||||
@@ -65,6 +65,7 @@ type Runner struct {
|
||||
configSyncRetry *configSyncRetry
|
||||
restartOpenrestyNow bool
|
||||
websocketUpgradeEnabled bool
|
||||
discoveryRetry discoveryRetry
|
||||
}
|
||||
|
||||
// Run starts the agent's main loop, performing heartbeats and upgrading to WebSocket when available.
|
||||
@@ -120,6 +121,9 @@ func (r *Runner) Run(ctx context.Context) error {
|
||||
tryStartWebSocket()
|
||||
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
slog.Info("agent runner shutting down", "error", ctx.Err())
|
||||
@@ -162,6 +166,9 @@ func (r *Runner) handleWSDisconnect(ctx context.Context, nodeID string) {
|
||||
|
||||
func (r *Runner) handleHeartbeatTick(ctx context.Context, nodeID *string, heartbeatTicker *time.Ticker, tryStartWebSocket func()) {
|
||||
if !r.hasAccessToken() {
|
||||
if !r.discoveryRetry.ready(time.Now()) {
|
||||
return
|
||||
}
|
||||
if err := r.tryRegister(ctx, nodeID); err != nil {
|
||||
slog.Error("agent discovery register failed", "error", err)
|
||||
}
|
||||
@@ -465,36 +472,46 @@ func (r *Runner) tryRestartOpenresty(ctx context.Context) {
|
||||
r.recordOpenrestyHealthy()
|
||||
}
|
||||
|
||||
func (r *Runner) tryRegister(ctx context.Context, nodeID *string) error {
|
||||
func (r *Runner) tryRegister(ctx context.Context, nodeID *string) (resultErr error) {
|
||||
defer func() {
|
||||
if resultErr != nil {
|
||||
r.discoveryRetry.failed(time.Now())
|
||||
}
|
||||
}()
|
||||
if strings.TrimSpace(r.Config.DiscoveryToken) == "" {
|
||||
return errors.New("agent_token 为空且未配置 discovery_token")
|
||||
return errors.New("agent token is empty and no discovery token is configured")
|
||||
}
|
||||
registrationKey, err := r.StateStore.EnsureRegistrationKey()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
slog.Info("agent discovery registration started")
|
||||
response, err := r.HeartbeatService.Register(ctx, r.HeartbeatCycle.NodePayload(ctx, *nodeID))
|
||||
response, err := r.HeartbeatService.Register(ctx, protocol.RegisterNodeRequest{
|
||||
NodePayload: r.HeartbeatCycle.NodePayload(ctx, *nodeID),
|
||||
RegistrationKey: registrationKey,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if response == nil || strings.TrimSpace(response.AccessToken) == "" || strings.TrimSpace(response.NodeID) == "" {
|
||||
return errors.New("discovery register response 缺少 node_id 或 agent_token")
|
||||
return errors.New("discovery registration response is missing node_id or agent_token")
|
||||
}
|
||||
snapshot, err := r.StateStore.Load()
|
||||
if err != nil {
|
||||
if err = r.StateStore.SetRegisteredNodeID(response.NodeID); err != nil {
|
||||
return err
|
||||
}
|
||||
snapshot.NodeID = response.NodeID
|
||||
if err = r.StateStore.Save(snapshot); err != nil {
|
||||
return err
|
||||
}
|
||||
r.Config.AccessToken = response.AccessToken
|
||||
r.Config.DiscoveryToken = ""
|
||||
if err = r.Config.Save(); err != nil {
|
||||
updatedConfig := *r.Config
|
||||
updatedConfig.AccessToken = response.AccessToken
|
||||
updatedConfig.DiscoveryToken = ""
|
||||
if err = updatedConfig.Save(); err != nil {
|
||||
return err
|
||||
}
|
||||
*r.Config = updatedConfig
|
||||
r.HeartbeatService.SetToken(response.AccessToken)
|
||||
if r.WebSocketService != nil {
|
||||
r.WebSocketService.SetToken(response.AccessToken)
|
||||
}
|
||||
*nodeID = response.NodeID
|
||||
r.discoveryRetry = discoveryRetry{}
|
||||
slog.Info("agent discovery registration succeeded", "node_id", response.NodeID)
|
||||
r.refreshOpenrestyHealth(ctx)
|
||||
if _, err = r.HeartbeatCycle.Perform(ctx, *nodeID, true, r); err != nil {
|
||||
|
||||
@@ -42,13 +42,17 @@ type fakeHeartbeatService struct {
|
||||
heartbeatResults []*protocol.HeartbeatResult
|
||||
heartbeatPayloads []protocol.NodePayload
|
||||
onHeartbeat func(int)
|
||||
onRegister func()
|
||||
lastToken string
|
||||
}
|
||||
|
||||
func (f *fakeHeartbeatService) Register(ctx context.Context, payload protocol.NodePayload) (*protocol.RegisterNodeResponse, error) {
|
||||
func (f *fakeHeartbeatService) Register(ctx context.Context, payload protocol.RegisterNodeRequest) (*protocol.RegisterNodeResponse, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.registerCalls++
|
||||
if f.onRegister != nil {
|
||||
f.onRegister()
|
||||
}
|
||||
return f.registerResp, f.registerErr
|
||||
}
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/apps/agent/localfile"
|
||||
"github.com/Rain-kl/Wavelet/internal/apps/edge/nodeip"
|
||||
"github.com/Rain-kl/Wavelet/pkg/util"
|
||||
)
|
||||
@@ -406,7 +407,7 @@ func (cfg *Config) Save() error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(cfg.configPath, data, configFilePerm)
|
||||
return localfile.WriteAtomic(cfg.configPath, data, configFilePerm)
|
||||
}
|
||||
|
||||
func detectHostname() string {
|
||||
|
||||
@@ -11,14 +11,14 @@ import (
|
||||
|
||||
// RemoteClient is the interface that abstracts the remote API calls performed by Service.
|
||||
type RemoteClient interface {
|
||||
RegisterNode(ctx context.Context, payload protocol.NodePayload) (*protocol.RegisterNodeResponse, error)
|
||||
RegisterNode(ctx context.Context, payload protocol.RegisterNodeRequest) (*protocol.RegisterNodeResponse, error)
|
||||
Heartbeat(ctx context.Context, payload protocol.NodePayload) (*protocol.HeartbeatResult, error)
|
||||
SetToken(token string)
|
||||
}
|
||||
|
||||
// API abstracts registration and heartbeat operations used by Cycle.
|
||||
type API interface {
|
||||
Register(ctx context.Context, payload protocol.NodePayload) (*protocol.RegisterNodeResponse, error)
|
||||
Register(ctx context.Context, payload protocol.RegisterNodeRequest) (*protocol.RegisterNodeResponse, error)
|
||||
Heartbeat(ctx context.Context, payload protocol.NodePayload) (*protocol.HeartbeatResult, error)
|
||||
SetToken(token string)
|
||||
}
|
||||
@@ -34,7 +34,7 @@ func New(client RemoteClient) *Service {
|
||||
}
|
||||
|
||||
// Register sends a node registration request to the server.
|
||||
func (s *Service) Register(ctx context.Context, payload protocol.NodePayload) (*protocol.RegisterNodeResponse, error) {
|
||||
func (s *Service) Register(ctx context.Context, payload protocol.RegisterNodeRequest) (*protocol.RegisterNodeResponse, error) {
|
||||
return s.client.RegisterNode(ctx, payload)
|
||||
}
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ func New(baseURL string, token string, timeout time.Duration) *Client {
|
||||
}
|
||||
|
||||
// RegisterNode registers the agent node with the control plane server.
|
||||
func (c *Client) RegisterNode(ctx context.Context, payload protocol.NodePayload) (*protocol.RegisterNodeResponse, error) {
|
||||
func (c *Client) RegisterNode(ctx context.Context, payload protocol.RegisterNodeRequest) (*protocol.RegisterNodeResponse, error) {
|
||||
resp := protocol.APIResponse[protocol.RegisterNodeResponse]{}
|
||||
if err := c.base.PostJSON(ctx, "/api/v1/agent/nodes/register", payload, &resp); err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Package localfile persists agent identity and configuration without truncating existing files.
|
||||
package localfile
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// WriteAtomic syncs the complete contents, replaces path, then syncs its containing directory.
|
||||
// An error from the final sync means the replacement is visible but its durability is unconfirmed.
|
||||
func WriteAtomic(path string, data []byte, perm os.FileMode) error {
|
||||
return writeAtomic(path, data, perm, SyncDirectory)
|
||||
}
|
||||
|
||||
func writeAtomic(path string, data []byte, perm os.FileMode, syncDir func(string) error) error {
|
||||
file, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+".tmp-*")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
_ = file.Close() // Best effort after an earlier write/close error.
|
||||
_ = os.Remove(file.Name()) // Best effort; rename normally removes the temporary path.
|
||||
}()
|
||||
if err = file.Chmod(perm); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err = file.Write(data); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = file.Sync(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = file.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = os.Rename(file.Name(), path); err != nil {
|
||||
return err
|
||||
}
|
||||
return syncDir(filepath.Dir(path))
|
||||
}
|
||||
|
||||
// SyncDirectory confirms a directory entry update on platforms with directory flushing.
|
||||
// Windows provides best-effort replacement without a portable directory flush guarantee.
|
||||
func SyncDirectory(path string) (resultErr error) {
|
||||
directory, err := os.Open(path) //nolint:gosec // callers provide the directory of a managed local state/config file
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
if closeErr := directory.Close(); resultErr == nil {
|
||||
resultErr = closeErr
|
||||
}
|
||||
}()
|
||||
info, err := directory.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return errors.New("cannot sync a non-directory path")
|
||||
}
|
||||
return syncDirectory(directory)
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package localfile
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestWriteAtomicReplacesCompleteFileAndCleansFailure(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "identity.json")
|
||||
if err := WriteAtomic(path, []byte("old identity"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := WriteAtomic(path, []byte("new identity"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil || string(data) != "new identity" {
|
||||
t.Errorf("WriteAtomic replacement = %q, %v; want complete new identity", data, err)
|
||||
}
|
||||
blocked := filepath.Join(dir, "directory")
|
||||
if err = os.Mkdir(blocked, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = WriteAtomic(blocked, []byte("cannot replace directory"), 0o600); err == nil {
|
||||
t.Fatal("WriteAtomic(directory) succeeded, want rename failure")
|
||||
}
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(entries) != 2 {
|
||||
t.Errorf("WriteAtomic(after failure) directory entries = %d, want 2 with no temporary file", len(entries))
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteAtomicReportsDirectorySyncFailureAfterReplacement(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "identity.json")
|
||||
if err := os.WriteFile(path, []byte("old identity"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
syncFailure := errors.New("directory I/O failure")
|
||||
syncCalls := 0
|
||||
err := writeAtomic(path, []byte("new identity"), 0o600, func(gotDir string) error {
|
||||
syncCalls++
|
||||
if gotDir != dir {
|
||||
t.Errorf("writeAtomic sync path = %q, want %q", gotDir, dir)
|
||||
}
|
||||
data, readErr := os.ReadFile(path)
|
||||
if readErr != nil || string(data) != "new identity" {
|
||||
t.Errorf("Directory sync ran before replacement: contents = %q, error %v", data, readErr)
|
||||
}
|
||||
return syncFailure
|
||||
})
|
||||
if !errors.Is(err, syncFailure) || syncCalls != 1 {
|
||||
t.Errorf("writeAtomic(directory sync failure) = %v, %d sync calls; want I/O failure, 1", err, syncCalls)
|
||||
}
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(entries) != 1 {
|
||||
t.Errorf("writeAtomic(failed directory sync) left %d entries, want only destination", len(entries))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSyncDirectoryRejectsMissingAndNonDirectoryPaths(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
if err := SyncDirectory(dir); err != nil {
|
||||
t.Fatalf("SyncDirectory(existing directory) = %v, want nil", err)
|
||||
}
|
||||
if err := SyncDirectory(filepath.Join(dir, "missing")); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Errorf("SyncDirectory(missing directory) = %v, want not-exist error", err)
|
||||
}
|
||||
path := filepath.Join(dir, "file")
|
||||
if err := os.WriteFile(path, []byte("contents"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := SyncDirectory(path); err == nil {
|
||||
t.Error("SyncDirectory(regular file) = nil, want non-directory error")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
//go:build !windows
|
||||
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package localfile
|
||||
|
||||
import "os"
|
||||
|
||||
func syncDirectory(directory *os.File) error {
|
||||
return directory.Sync()
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package localfile
|
||||
|
||||
import "os"
|
||||
|
||||
// Windows does not support a portable directory fsync through os.File.Sync.
|
||||
// The file is flushed before replacement; persisting the directory entry is best effort.
|
||||
func syncDirectory(_ *os.File) error {
|
||||
return nil
|
||||
}
|
||||
@@ -51,6 +51,9 @@ type NodeHealthEvent = pkgprotocol.NodeHealthEvent
|
||||
// RegisterNodeResponse is an alias for pkgprotocol.RegisterNodeResponse.
|
||||
type RegisterNodeResponse = pkgprotocol.RegisterNodeResponse
|
||||
|
||||
// RegisterNodeRequest is an enrollment-only request, including its retry credential.
|
||||
type RegisterNodeRequest = pkgprotocol.RegisterNodeRequest
|
||||
|
||||
// ApplyLogPayload is an alias for pkgprotocol.ApplyLogPayload.
|
||||
type ApplyLogPayload = pkgprotocol.ApplyLogPayload
|
||||
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package state
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/apps/agent/localfile"
|
||||
)
|
||||
|
||||
func TestRegistrationKeySurvivesStaleSnapshotAndRestart(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "state.json")
|
||||
store := NewStore(path)
|
||||
stale, err := store.Load()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, err := store.EnsureRegistrationKey()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = store.SetRegisteredNodeID("node-server-assigned"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
decoded, err := hex.DecodeString(key)
|
||||
if err != nil || len(decoded) != 32 {
|
||||
t.Fatalf("EnsureRegistrationKey() length = %d, error %v; want 32, nil", len(decoded), err)
|
||||
}
|
||||
stale.LastError = "background sync failed"
|
||||
if err = store.Save(stale); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
restartedKey, err := NewStore(path).EnsureRegistrationKey()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if restartedKey != key {
|
||||
t.Error("EnsureRegistrationKey(after stale save and restart) changed enrollment identity")
|
||||
}
|
||||
snapshot, err := NewStore(path).Load()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if snapshot.NodeID != "node-server-assigned" {
|
||||
t.Errorf("Load(after stale save).NodeID = %q, want server-assigned ID", snapshot.NodeID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExistingRegistrationKeyRequiresDirectorySyncBeforeUse(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "state.json")
|
||||
store := NewStore(path)
|
||||
key, err := store.EnsureRegistrationKey()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Model a prior successful rename whose final directory flush was not confirmed.
|
||||
syncFailure := errors.New("directory I/O failure")
|
||||
syncCalls := 0
|
||||
store.syncDirectory = func(dir string) error {
|
||||
syncCalls++
|
||||
if dir != filepath.Dir(path) {
|
||||
t.Errorf("EnsureRegistrationKey sync directory = %q, want state parent", dir)
|
||||
}
|
||||
return syncFailure
|
||||
}
|
||||
for range 2 {
|
||||
got, retryErr := store.EnsureRegistrationKey()
|
||||
if got != "" || !errors.Is(retryErr, syncFailure) {
|
||||
t.Errorf("EnsureRegistrationKey(unconfirmed directory) returned usable key or lost error: %v", retryErr)
|
||||
}
|
||||
}
|
||||
if syncCalls != 2 {
|
||||
t.Errorf("EnsureRegistrationKey(repeated failures) sync calls = %d, want 2", syncCalls)
|
||||
}
|
||||
store.syncDirectory = localfile.SyncDirectory
|
||||
confirmedKey, err := store.EnsureRegistrationKey()
|
||||
if err != nil || confirmedKey != key {
|
||||
t.Errorf("EnsureRegistrationKey(after successful sync) changed identity or failed: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -10,12 +10,15 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/apps/agent/localfile"
|
||||
)
|
||||
|
||||
const (
|
||||
stateDirPerm = 0o750
|
||||
stateFilePerm = 0o600
|
||||
nodeIDRandomBytes = 8
|
||||
stateDirPerm = 0o750
|
||||
stateFilePerm = 0o600
|
||||
nodeIDRandomBytes = 8
|
||||
registrationKeyBytes = 32
|
||||
)
|
||||
|
||||
// PagesDeployment records a Pages project tracked by the agent and the last
|
||||
@@ -31,6 +34,7 @@ type PagesDeployment struct {
|
||||
// Snapshot represents the state of the agent at a given point in time.
|
||||
type Snapshot struct {
|
||||
NodeID string `json:"node_id"`
|
||||
RegistrationKey string `json:"registration_key,omitempty"`
|
||||
CurrentVersion string `json:"current_version"`
|
||||
CurrentChecksum string `json:"current_checksum"`
|
||||
PagesDeployments []PagesDeployment `json:"pages_deployments"`
|
||||
@@ -47,15 +51,42 @@ type Snapshot struct {
|
||||
AccessLogOffset int64 `json:"access_log_offset"`
|
||||
}
|
||||
|
||||
// EnsureRegistrationKey persists the private retry credential before enrollment can reach the server.
|
||||
func (s *Store) EnsureRegistrationKey() (string, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
snapshot, err := s.loadUnlocked()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if snapshot.RegistrationKey != "" {
|
||||
// A previous rename may have succeeded even though its directory sync failed.
|
||||
if err = s.syncDirectory(filepath.Dir(s.path)); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return snapshot.RegistrationKey, nil
|
||||
}
|
||||
key := make([]byte, registrationKeyBytes)
|
||||
if _, err = rand.Read(key); err != nil {
|
||||
return "", err
|
||||
}
|
||||
snapshot.RegistrationKey = hex.EncodeToString(key)
|
||||
if err = s.saveUnlocked(snapshot); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return snapshot.RegistrationKey, nil
|
||||
}
|
||||
|
||||
// Store manages the storage and retrieval of the agent state snapshot.
|
||||
type Store struct {
|
||||
path string
|
||||
mu sync.Mutex
|
||||
path string
|
||||
mu sync.Mutex
|
||||
syncDirectory func(string) error
|
||||
}
|
||||
|
||||
// NewStore creates a new Store instance at the given path.
|
||||
func NewStore(path string) *Store {
|
||||
return &Store{path: filepath.Clean(path)}
|
||||
return &Store{path: filepath.Clean(path), syncDirectory: localfile.SyncDirectory}
|
||||
}
|
||||
|
||||
// Load loads the snapshot from the store.
|
||||
@@ -91,6 +122,29 @@ func (s *Store) EnsureNodeID() (string, error) {
|
||||
func (s *Store) Save(snapshot *Snapshot) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
current, err := s.loadUnlocked()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
updated := *snapshot
|
||||
|
||||
// Other workers can hold a snapshot loaded before enrollment first persisted its key.
|
||||
if current.RegistrationKey != "" {
|
||||
updated.RegistrationKey = current.RegistrationKey
|
||||
updated.NodeID = current.NodeID
|
||||
}
|
||||
return s.saveUnlocked(&updated)
|
||||
}
|
||||
|
||||
// SetRegisteredNodeID commits the server-assigned identity without overwriting concurrent runtime state.
|
||||
func (s *Store) SetRegisteredNodeID(nodeID string) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
snapshot, err := s.loadUnlocked()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
snapshot.NodeID = nodeID
|
||||
return s.saveUnlocked(snapshot)
|
||||
}
|
||||
|
||||
@@ -120,7 +174,7 @@ func (s *Store) saveUnlocked(snapshot *Snapshot) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(s.path, data, stateFilePerm)
|
||||
return localfile.WriteAtomic(s.path, data, stateFilePerm)
|
||||
}
|
||||
|
||||
func newNodeID() (string, error) {
|
||||
|
||||
Reference in New Issue
Block a user