fix: reduce sqlite write pressure during scans

This commit is contained in:
ShukeBta
2026-06-11 21:45:53 +08:00
parent 0e28d07122
commit 7d0afd0fcb
6 changed files with 233 additions and 41 deletions
+8 -2
View File
@@ -946,10 +946,16 @@ func (d *DownloadService) syncDownloadTaskProgress(ctx context.Context, torrent
if strings.TrimSpace(status) == "" {
status = matched.Status
}
updates := map[string]any{"progress": torrent.Progress}
if status != "" {
updates := map[string]any{}
if math.Abs(float64(matched.Progress-torrent.Progress)) > 0.0001 {
updates["progress"] = torrent.Progress
}
if status != "" && status != matched.Status {
updates["status"] = status
}
if len(updates) == 0 {
return
}
_ = d.repo.DB.WithContext(ctx).Model(&model.DownloadTask{}).Where("id = ?", matched.ID).Updates(updates).Error
}
+41
View File
@@ -10,6 +10,7 @@ import (
"strings"
"sync/atomic"
"testing"
"time"
"github.com/glebarez/sqlite"
"go.uber.org/zap"
@@ -47,6 +48,46 @@ func TestDownloadViewsDoNotExposePrivateURL(t *testing.T) {
}
}
func TestSyncDownloadTaskProgressSkipsUnchangedCompletedTask(t *testing.T) {
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
if err != nil {
t.Fatal(err)
}
if err := db.AutoMigrate(&model.DownloadTask{}); err != nil {
t.Fatal(err)
}
repos := repository.New(db)
task := &model.DownloadTask{
Source: "qbittorrent",
URL: "magnet:?xt=urn:btih:test",
Title: "Already.Done.S01E01",
SavePath: "/downloads",
Status: "completed",
Progress: 1,
}
if err := repos.Download.Create(t.Context(), task); err != nil {
t.Fatal(err)
}
var before model.DownloadTask
if err := db.First(&before, "id = ?", task.ID).Error; err != nil {
t.Fatal(err)
}
time.Sleep(10 * time.Millisecond)
svc := NewDownloadService(zap.NewNop(), repos, NewHub(zap.NewNop()), nil)
svc.syncDownloadTaskProgress(t.Context(), QBitTorrent{
Name: task.Title,
Progress: 1,
State: "completed",
}, tasksByIdentity([]model.DownloadTask{before}))
var after model.DownloadTask
if err := db.First(&after, "id = ?", task.ID).Error; err != nil {
t.Fatal(err)
}
if !after.UpdatedAt.Equal(before.UpdatedAt) {
t.Fatalf("unchanged completed torrent touched updated_at: before=%s after=%s", before.UpdatedAt, after.UpdatedAt)
}
}
func TestDownloadCompleteAutoOrganizesContentPath(t *testing.T) {
root := t.TempDir()
src := filepath.Join(root, "downloads", "国产剧", "狂飙.S01E01.2023.1080p.mkv")
+45 -12
View File
@@ -6,6 +6,7 @@ import (
"crypto/rand"
"encoding/hex"
"errors"
"sync"
"time"
"github.com/golang-jwt/jwt/v5"
@@ -37,14 +38,16 @@ type Claims struct {
// TokenService 处理双令牌认证(Access Token + Refresh Token)。
type TokenService struct {
cfg *config.Config
log *zap.Logger
repo *repository.Container
cfg *config.Config
log *zap.Logger
repo *repository.Container
delayedStoreMu sync.Mutex
delayedStores map[string]struct{}
}
// NewTokenService 创建令牌服务实例。
func NewTokenService(cfg *config.Config, log *zap.Logger, repo *repository.Container) *TokenService {
return &TokenService{cfg: cfg, log: log, repo: repo}
return &TokenService{cfg: cfg, log: log, repo: repo, delayedStores: make(map[string]struct{})}
}
// TokenPair 包含访问令牌和刷新令牌。
@@ -110,7 +113,9 @@ func (s *TokenService) issuePair(ctx context.Context, userID, role, tier string,
zap.String("user_id", userID),
zap.Error(err))
}
go s.storeRefreshTokenEventually(userID, tokenHash, rt.ExpiresAt)
if s.trackDelayedStore(userID, tokenHash) {
go s.storeRefreshTokenEventually(userID, tokenHash, rt.ExpiresAt)
}
}
return &TokenPair{
@@ -132,9 +137,12 @@ func (s *TokenService) storeRefreshToken(ctx context.Context, rt *model.RefreshT
}
func (s *TokenService) storeRefreshTokenEventually(userID, tokenHash string, expiresAt time.Time) {
delay := 500 * time.Millisecond
for attempt := 1; attempt <= 30; attempt++ {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer s.untrackDelayedStore(userID, tokenHash)
delay := 5 * time.Second
for attempt := 1; attempt <= 8; attempt++ {
timer := time.NewTimer(delay)
<-timer.C
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
err := s.storeRefreshToken(ctx, &model.RefreshToken{
UserID: userID,
TokenHash: tokenHash,
@@ -150,15 +158,13 @@ func (s *TokenService) storeRefreshTokenEventually(userID, tokenHash string, exp
}
return
}
if s.log != nil && (attempt == 1 || attempt%10 == 0) {
if s.log != nil && (attempt == 1 || attempt == 4 || attempt == 8) {
s.log.Warn("refresh token delayed store still waiting",
zap.String("user_id", userID),
zap.Int("attempt", attempt),
zap.Error(err))
}
timer := time.NewTimer(delay)
<-timer.C
if delay < 10*time.Second {
if delay < 60*time.Second {
delay *= 2
}
}
@@ -167,6 +173,33 @@ func (s *TokenService) storeRefreshTokenEventually(userID, tokenHash string, exp
}
}
func (s *TokenService) trackDelayedStore(userID, tokenHash string) bool {
if s == nil {
return false
}
key := userID + "\x00" + tokenHash
s.delayedStoreMu.Lock()
defer s.delayedStoreMu.Unlock()
if s.delayedStores == nil {
s.delayedStores = make(map[string]struct{})
}
if _, ok := s.delayedStores[key]; ok {
return false
}
s.delayedStores[key] = struct{}{}
return true
}
func (s *TokenService) untrackDelayedStore(userID, tokenHash string) {
if s == nil {
return
}
key := userID + "\x00" + tokenHash
s.delayedStoreMu.Lock()
delete(s.delayedStores, key)
s.delayedStoreMu.Unlock()
}
func (s *TokenService) maxActiveRefreshTokens(ctx context.Context) int {
cfg := loadBotConfig(ctx, s.repo)
if cfg.MaxLoggedClients < 1 {