feat(updater): throttle auto-update checks with exponential backoff

Check GitHub releases at most every 6 hours and retry transient failures
(release query, checksum or binary download) with exponential backoff
from 1 minute up to 3 hours. Forced updates bypass the throttle.
This commit is contained in:
ryan
2026-11-07 21:49:46 +08:00
parent 8867c822bb
commit b60546a94d
6 changed files with 627 additions and 31 deletions
+114 -22
View File
@@ -17,6 +17,7 @@ import (
"os"
"runtime"
"strings"
"sync"
"time"
"github.com/Rain-kl/Wavelet/pkg/util"
@@ -27,34 +28,107 @@ const (
goosWindows = "windows"
updateTmpFilePerm = 0o600
updateBinaryFilePerm = 0o755
// DefaultCheckInterval is the default interval between regular update checks.
DefaultCheckInterval = 6 * time.Hour
// DefaultInitialBackoff is the default delay before retrying after the first failed check.
DefaultInitialBackoff = 1 * time.Minute
// DefaultMaxBackoff caps the exponential retry delay after consecutive failures.
DefaultMaxBackoff = 3 * time.Hour
)
var replaceAndRestartFunc = replaceAndRestart
// Config defines the configuration for the update service.
type Config struct {
LocalVersion string
AssetPrefix string
LogLabel string
LocalVersion string
AssetPrefix string
LogLabel string
CheckInterval time.Duration
InitialBackoff time.Duration
MaxBackoff time.Duration
NowFunc func() time.Time
HTTPClient *http.Client
}
// Service handles checking and applying application binary updates.
type Service struct {
httpClient *http.Client
lastCheckKey string
localVersion string
assetPrefix string
logLabel string
httpClient *http.Client
lastCheckKey string
localVersion string
assetPrefix string
logLabel string
checkInterval time.Duration
initialBackoff time.Duration
maxBackoff time.Duration
nowFunc func() time.Time
mu sync.Mutex
nextCheckTime time.Time
consecutiveFailures int
}
// New creates a new updater Service with the provided configuration.
func New(cfg Config) *Service {
return &Service{
httpClient: &http.Client{Timeout: 30 * time.Second},
localVersion: cfg.LocalVersion,
assetPrefix: cfg.AssetPrefix,
logLabel: cfg.LogLabel,
checkInterval := cfg.CheckInterval
if checkInterval <= 0 {
checkInterval = DefaultCheckInterval
}
initialBackoff := cfg.InitialBackoff
if initialBackoff <= 0 {
initialBackoff = DefaultInitialBackoff
}
maxBackoff := cfg.MaxBackoff
if maxBackoff <= 0 {
maxBackoff = DefaultMaxBackoff
}
nowFunc := cfg.NowFunc
if nowFunc == nil {
nowFunc = time.Now
}
httpClient := cfg.HTTPClient
if httpClient == nil {
httpClient = &http.Client{Timeout: 30 * time.Second}
}
return &Service{
httpClient: httpClient,
localVersion: cfg.LocalVersion,
assetPrefix: cfg.AssetPrefix,
logLabel: cfg.LogLabel,
checkInterval: checkInterval,
initialBackoff: initialBackoff,
maxBackoff: maxBackoff,
nowFunc: nowFunc,
}
}
// CanCheck returns whether an update check is allowed to run under the check interval and failure backoff.
func (s *Service) CanCheck(options UpdateOptions) bool {
if options.Force {
return true
}
s.mu.Lock()
defer s.mu.Unlock()
return !s.nowFunc().Before(s.nextCheckTime)
}
// recordFailure schedules the next check using exponential backoff capped at maxBackoff.
func (s *Service) recordFailure() {
s.mu.Lock()
defer s.mu.Unlock()
backoff := s.initialBackoff
for i := 0; i < s.consecutiveFailures && backoff < s.maxBackoff; i++ {
backoff *= 2
}
s.consecutiveFailures++
s.nextCheckTime = s.nowFunc().Add(min(backoff, s.maxBackoff))
}
// recordSuccess resets the failure backoff and schedules the next regular check.
func (s *Service) recordSuccess() {
s.mu.Lock()
defer s.mu.Unlock()
s.consecutiveFailures = 0
s.nextCheckTime = s.nowFunc().Add(s.checkInterval)
}
// UpdateOptions specifies parameters for checking and applying updates.
@@ -78,13 +152,29 @@ type githubAsset struct {
}
// CheckAndUpdate checks for a newer release on GitHub and performs an update if available.
// Transient failures are retried with exponential backoff; otherwise the next check waits for the check interval.
// Callers are expected to invoke it serially.
func (s *Service) CheckAndUpdate(ctx context.Context, repo string, options UpdateOptions) error {
if !s.CanCheck(options) {
return nil
}
retry, err := s.checkAndUpdate(ctx, repo, options)
if retry {
s.recordFailure()
} else {
s.recordSuccess()
}
return err
}
// checkAndUpdate performs one update check and reports whether a returned error is transient and worth retrying.
func (s *Service) checkAndUpdate(ctx context.Context, repo string, options UpdateOptions) (retry bool, err error) {
release, err := s.getRelease(ctx, repo, options)
if err != nil {
return fmt.Errorf("check latest release: %w", err)
return true, fmt.Errorf("check latest release: %w", err)
}
if release == nil || release.TagName == "" {
return nil
return false, nil
}
remoteVersion := normalizeVersion(release.TagName)
@@ -92,14 +182,14 @@ func (s *Service) CheckAndUpdate(ctx context.Context, repo string, options Updat
checkKey := buildReleaseCheckKey(options, remoteVersion)
if remoteVersion == localVersion {
return nil
return false, nil
}
if !options.Force && checkKey != "" && checkKey == s.lastCheckKey {
return nil
return false, nil
}
if !isNewer(localVersion, remoteVersion) {
s.lastCheckKey = checkKey
return nil
return false, nil
}
slog.Info(s.logLabel+" update available", "from", localVersion, "to", remoteVersion)
@@ -108,20 +198,22 @@ func (s *Service) CheckAndUpdate(ctx context.Context, repo string, options Updat
downloadURL, expectedChecksum, err := s.resolveReleaseAsset(ctx, release, assetName)
if err != nil {
if downloadURL == "" {
// The release has no asset for this platform; wait for the next regular check.
s.lastCheckKey = checkKey
return false, err
}
return err
return true, err
}
execPath, err := os.Executable()
if err != nil {
return fmt.Errorf("get executable path: %w", err)
return false, fmt.Errorf("get executable path: %w", err)
}
if err = s.downloadAndRestart(ctx, downloadURL, expectedChecksum, execPath); err != nil {
return fmt.Errorf("download and restart: %w", err)
return true, fmt.Errorf("download and restart: %w", err)
}
s.lastCheckKey = checkKey
return nil
return false, nil
}
func (s *Service) getRelease(ctx context.Context, repo string, options UpdateOptions) (*githubRelease, error) {
+242 -6
View File
@@ -14,6 +14,7 @@ import (
"runtime"
"strings"
"testing"
"time"
)
type roundTripFunc func(req *http.Request) (*http.Response, error)
@@ -23,12 +24,12 @@ func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
}
func testService(httpClient *http.Client) *Service {
return &Service{
httpClient: httpClient,
localVersion: "v1.0.0",
assetPrefix: "openflare-agent",
logLabel: "agent",
}
return New(Config{
LocalVersion: "v1.0.0",
AssetPrefix: "openflare-agent",
LogLabel: "agent",
HTTPClient: httpClient,
})
}
func TestGetLatestPreviewRelease(t *testing.T) {
@@ -106,6 +107,56 @@ func TestCheckAndUpdateRequiresChecksumSource(t *testing.T) {
}
}
func TestCheckAndUpdateSchedulesNextCheckByFailureKind(t *testing.T) {
assetName := testService(nil).assetNameForGOOSGOARCH(runtime.GOOS, runtime.GOARCH)
digest := "sha256:" + strings.Repeat("a", sha256.Size*2)
testCases := []struct {
name string
releaseAssetName string
wantRetry bool
}{
{name: "asset download failure retries with backoff", releaseAssetName: assetName, wantRetry: true},
{name: "missing platform asset waits for regular interval", releaseAssetName: "openflare-agent-other", wantRetry: false},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
currentTime := time.Date(2026, 10, 17, 12, 0, 0, 0, time.UTC)
service := New(Config{
LocalVersion: "v1.0.0",
AssetPrefix: "openflare-agent",
LogLabel: "agent",
NowFunc: func() time.Time { return currentTime },
HTTPClient: &http.Client{
Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
if req.URL.Host != "api.github.com" {
return &http.Response{
StatusCode: http.StatusInternalServerError,
Status: "500 Internal Server Error",
Header: make(http.Header),
Body: io.NopCloser(strings.NewReader("error")),
}, nil
}
return &http.Response{
StatusCode: http.StatusOK,
Header: make(http.Header),
Body: io.NopCloser(strings.NewReader(`{"tag_name":"v1.0.1","assets":[` +
`{"name":"` + tc.releaseAssetName + `","browser_download_url":"https://example.test/agent","digest":"` + digest + `"}]}`)),
}, nil
}),
},
})
if err := service.CheckAndUpdate(context.Background(), "Rain-kl/OpenFlare", UpdateOptions{}); err == nil {
t.Fatalf("expected update error")
}
currentTime = currentTime.Add(DefaultInitialBackoff)
if got := service.CanCheck(UpdateOptions{}); got != tc.wantRetry {
t.Fatalf("CanCheck after initial backoff = %v, want %v", got, tc.wantRetry)
}
})
}
}
func TestNormalizeGitHubDigest(t *testing.T) {
checksum := strings.Repeat("a", sha256.Size*2)
testCases := []struct {
@@ -325,3 +376,188 @@ func TestIsNewerSupportsPrerelease(t *testing.T) {
})
}
}
func TestCheckAndUpdateIntervalAndBackoff(t *testing.T) {
currentTime := time.Date(2026, 10, 17, 12, 0, 0, 0, time.UTC)
nowFunc := func() time.Time { return currentTime }
var (
statusCode = http.StatusOK
responseBody = `{"tag_name":"v1.0.0"}`
requestCount = 0
)
httpClient := &http.Client{
Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
requestCount++
if statusCode != http.StatusOK {
return &http.Response{
StatusCode: statusCode,
Status: http.StatusText(statusCode),
Header: make(http.Header),
Body: io.NopCloser(strings.NewReader("error")),
}, nil
}
return &http.Response{
StatusCode: http.StatusOK,
Status: "200 OK",
Header: make(http.Header),
Body: io.NopCloser(strings.NewReader(responseBody)),
}, nil
}),
}
service := New(Config{
LocalVersion: "v1.0.0",
AssetPrefix: "openflare-agent",
LogLabel: "agent",
CheckInterval: 6 * time.Hour,
InitialBackoff: 1 * time.Minute,
MaxBackoff: 3 * time.Hour,
NowFunc: nowFunc,
HTTPClient: httpClient,
})
// 1. Initial state: CanCheck should be true
if !service.CanCheck(UpdateOptions{}) {
t.Fatalf("expected CanCheck to be true initially")
}
// 2. First check succeeds (version v1.0.0 is same as local, no update needed)
err := service.CheckAndUpdate(context.Background(), "Rain-kl/OpenFlare", UpdateOptions{})
if err != nil {
t.Fatalf("unexpected error on first check: %v", err)
}
if requestCount != 1 {
t.Fatalf("expected 1 request, got %d", requestCount)
}
// Immediately after success: CanCheck should be false, and CheckAndUpdate should be throttled (no HTTP call)
if service.CanCheck(UpdateOptions{}) {
t.Fatalf("expected CanCheck to be false immediately after success")
}
err = service.CheckAndUpdate(context.Background(), "Rain-kl/OpenFlare", UpdateOptions{})
if err != nil {
t.Fatalf("unexpected error on throttled check: %v", err)
}
if requestCount != 1 {
t.Fatalf("expected request count to remain 1, got %d", requestCount)
}
// 3. Fast-forward 5 hours: still throttled (< 6h)
currentTime = currentTime.Add(5 * time.Hour)
if service.CanCheck(UpdateOptions{}) {
t.Fatalf("expected CanCheck to be false after 5 hours")
}
// Force check should bypass throttle even within 6 hours
if !service.CanCheck(UpdateOptions{Force: true}) {
t.Fatalf("expected CanCheck(Force: true) to be true within 6 hours")
}
err = service.CheckAndUpdate(context.Background(), "Rain-kl/OpenFlare", UpdateOptions{Force: true})
if err != nil {
t.Fatalf("unexpected error on force check: %v", err)
}
if requestCount != 2 {
t.Fatalf("expected request count 2 after force check, got %d", requestCount)
}
// 4. Fast-forward to 6 hours after the force check: should be allowed again
currentTime = currentTime.Add(6 * time.Hour)
if !service.CanCheck(UpdateOptions{}) {
t.Fatalf("expected CanCheck to be true after 6 hours")
}
// 5. Simulate Failure: GitHub returns 403 Forbidden
statusCode = http.StatusForbidden
err = service.CheckAndUpdate(context.Background(), "Rain-kl/OpenFlare", UpdateOptions{})
if err == nil {
t.Fatalf("expected error on 403 Forbidden check")
}
if requestCount != 3 {
t.Fatalf("expected request count 3, got %d", requestCount)
}
// Immediately after 1st failure: CanCheck should be false (1 min initial backoff)
if service.CanCheck(UpdateOptions{}) {
t.Fatalf("expected CanCheck to be false after 1st failure")
}
// At +30s: still false
currentTime = currentTime.Add(30 * time.Second)
if service.CanCheck(UpdateOptions{}) {
t.Fatalf("expected CanCheck to be false at +30s after 1st failure")
}
// At +1m (total +60s): should be true
currentTime = currentTime.Add(30 * time.Second)
if !service.CanCheck(UpdateOptions{}) {
t.Fatalf("expected CanCheck to be true at +1m after 1st failure")
}
// 6. Second failure: backoff doubles to 2 minutes
err = service.CheckAndUpdate(context.Background(), "Rain-kl/OpenFlare", UpdateOptions{})
if err == nil {
t.Fatalf("expected error on 2nd failure")
}
if requestCount != 4 {
t.Fatalf("expected request count 4, got %d", requestCount)
}
// At +1m: should still be false
currentTime = currentTime.Add(1 * time.Minute)
if service.CanCheck(UpdateOptions{}) {
t.Fatalf("expected CanCheck to be false at +1m after 2nd failure (backoff is 2m)")
}
// At +2m: should be true
currentTime = currentTime.Add(1 * time.Minute)
if !service.CanCheck(UpdateOptions{}) {
t.Fatalf("expected CanCheck to be true at +2m after 2nd failure")
}
// 7. Simulate multiple consecutive failures to verify exponential doubling and capping at 3 hours
expectedBackoffs := []time.Duration{
4 * time.Minute,
8 * time.Minute,
16 * time.Minute,
32 * time.Minute,
64 * time.Minute,
128 * time.Minute,
3 * time.Hour, // 256m capped at 180m (3h)
3 * time.Hour, // capped at 3h
}
for i, expectedBackoff := range expectedBackoffs {
err = service.CheckAndUpdate(context.Background(), "Rain-kl/OpenFlare", UpdateOptions{})
if err == nil {
t.Fatalf("expected error on consecutive failure iteration %d", i)
}
// Halfway through backoff: should be false
currentTime = currentTime.Add(expectedBackoff / 2)
if service.CanCheck(UpdateOptions{}) {
t.Fatalf("expected CanCheck to be false halfway through backoff %v (iter %d)", expectedBackoff, i)
}
// At full backoff: should be true
currentTime = currentTime.Add(expectedBackoff / 2)
if !service.CanCheck(UpdateOptions{}) {
t.Fatalf("expected CanCheck to be true at full backoff %v (iter %d)", expectedBackoff, i)
}
}
// 8. Recovery: next check succeeds, backoff should be reset and next check scheduled in 6 hours
statusCode = http.StatusOK
err = service.CheckAndUpdate(context.Background(), "Rain-kl/OpenFlare", UpdateOptions{})
if err != nil {
t.Fatalf("expected success on recovery, got: %v", err)
}
// Immediately: CanCheck is false
if service.CanCheck(UpdateOptions{}) {
t.Fatalf("expected CanCheck to be false immediately after recovery")
}
// At +3 hours: still false (because next check is 6 hours, not 3 hours)
currentTime = currentTime.Add(3 * time.Hour)
if service.CanCheck(UpdateOptions{}) {
t.Fatalf("expected CanCheck to be false at +3 hours after recovery")
}
// At +6 hours: should be true
currentTime = currentTime.Add(3 * time.Hour)
if !service.CanCheck(UpdateOptions{}) {
t.Fatalf("expected CanCheck to be true at +6 hours after recovery")
}
}