mirror of
https://github.com/truewhile/MeBox.git
synced 2026-11-16 12:54:22 +08:00
feat: 完善媒体维护、备份恢复与播放诊断
This commit is contained in:
+81
-37
@@ -5,7 +5,7 @@
|
||||
//
|
||||
// 两种方言的备份形态不同:
|
||||
// - SQLite:用 "VACUUM INTO"(3.27.0 起可用)产出一个一致的库文件,
|
||||
// 扩展名 .db;恢复即「替换库文件 + 重启」。
|
||||
// 扩展名 .db;恢复先暂存,在下次启动打开数据库前替换文件。
|
||||
// - PostgreSQL:没有单文件语义,用逻辑备份(见 backup_postgres.go),
|
||||
// 扩展名 .json.gz;恢复是在单个事务内重建全部表。
|
||||
package service
|
||||
@@ -24,6 +24,7 @@ import (
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/config"
|
||||
"github.com/truewhile/MeBox/internal/database"
|
||||
)
|
||||
|
||||
// BackupService manages database backups.
|
||||
@@ -44,6 +45,7 @@ type BackupInfo struct {
|
||||
Path string `json:"path"`
|
||||
Size int64 `json:"size"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
Automatic bool `json:"automatic"`
|
||||
}
|
||||
|
||||
func (b *BackupService) backupDir() string {
|
||||
@@ -69,6 +71,23 @@ func (b *BackupService) backupFilePath(filename string) (string, error) {
|
||||
return path, nil
|
||||
}
|
||||
|
||||
// ResolvePath returns the absolute path of a listed backup after applying the
|
||||
// same filename and directory checks used by restore and delete operations.
|
||||
func (b *BackupService) ResolvePath(filename string) (string, error) {
|
||||
path, err := b.backupFilePath(filename)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
info, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
return "", errors.New("backup is not a regular file")
|
||||
}
|
||||
return path, nil
|
||||
}
|
||||
|
||||
// isPostgres reports whether the service is backed by PostgreSQL.
|
||||
//
|
||||
// PostgreSQL 不支持 SQLite 的 "VACUUM INTO",按文件替换恢复也会写错目标,
|
||||
@@ -80,11 +99,23 @@ func (b *BackupService) isPostgres() bool {
|
||||
// Create produces a new hot backup. SQLite uses "VACUUM INTO"; PostgreSQL uses
|
||||
// a logical dump.
|
||||
func (b *BackupService) Create(ctx context.Context) (*BackupInfo, error) {
|
||||
return b.create(ctx, false)
|
||||
}
|
||||
|
||||
func (b *BackupService) CreateAutomatic(ctx context.Context) (*BackupInfo, error) {
|
||||
return b.create(ctx, true)
|
||||
}
|
||||
|
||||
func (b *BackupService) create(ctx context.Context, automatic bool) (*BackupInfo, error) {
|
||||
dir := b.backupDir()
|
||||
if err := os.MkdirAll(dir, 0o750); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ts := time.Now().Format("20060102_150405")
|
||||
now := time.Now()
|
||||
ts := fmt.Sprintf("%s_%09d", now.Format("20060102_150405"), now.Nanosecond())
|
||||
if automatic {
|
||||
ts = "auto_" + ts
|
||||
}
|
||||
|
||||
var (
|
||||
dst string
|
||||
@@ -108,15 +139,24 @@ func (b *BackupService) Create(ctx context.Context) (*BackupInfo, error) {
|
||||
Path: dst,
|
||||
Size: stat.Size(),
|
||||
CreatedAt: stat.ModTime(),
|
||||
Automatic: automatic,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (b *BackupService) createSQLiteBackup(ctx context.Context, stamp string) (string, error) {
|
||||
dst := filepath.Join(b.backupDir(), fmt.Sprintf("mebox_%s.db", stamp))
|
||||
// VACUUM INTO creates a clean, non-WAL copy atomically.
|
||||
if err := b.db.WithContext(ctx).Exec("VACUUM INTO ?", dst).Error; err != nil {
|
||||
tmp := dst + ".tmp"
|
||||
defer os.Remove(tmp)
|
||||
// Publish only completed backups, with credentials protected by file mode.
|
||||
if err := b.db.WithContext(ctx).Exec("VACUUM INTO ?", tmp).Error; err != nil {
|
||||
return "", fmt.Errorf("VACUUM INTO failed: %w", err)
|
||||
}
|
||||
if err := os.Chmod(tmp, 0o600); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := os.Rename(tmp, dst); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return dst, nil
|
||||
}
|
||||
|
||||
@@ -132,7 +172,7 @@ func (b *BackupService) List() ([]BackupInfo, error) {
|
||||
}
|
||||
out := make([]BackupInfo, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
if e.IsDir() || !isBackupFilename(e.Name()) {
|
||||
if !e.Type().IsRegular() || !isBackupFilename(e.Name()) {
|
||||
continue
|
||||
}
|
||||
info, err := e.Info()
|
||||
@@ -144,6 +184,7 @@ func (b *BackupService) List() ([]BackupInfo, error) {
|
||||
Path: filepath.Join(dir, e.Name()),
|
||||
Size: info.Size(),
|
||||
CreatedAt: info.ModTime(),
|
||||
Automatic: strings.HasPrefix(e.Name(), "mebox_auto_"),
|
||||
})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
@@ -152,6 +193,36 @@ func (b *BackupService) List() ([]BackupInfo, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Prune keeps maxCount automatic backups for the current database dialect.
|
||||
// Manual backups and backups from another dialect are never pruned.
|
||||
// A value below one disables pruning so an invalid setting cannot erase all
|
||||
// recoverable copies.
|
||||
func (b *BackupService) Prune(maxCount int) error {
|
||||
if maxCount < 1 {
|
||||
return nil
|
||||
}
|
||||
items, err := b.List()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
eligible := items[:0]
|
||||
for _, item := range items {
|
||||
if item.Automatic && strings.HasSuffix(item.Filename, postgresBackupSuffix) == b.isPostgres() {
|
||||
eligible = append(eligible, item)
|
||||
}
|
||||
}
|
||||
items = eligible
|
||||
if maxCount >= len(items) {
|
||||
return nil
|
||||
}
|
||||
for _, item := range items[maxCount:] {
|
||||
if err := b.Delete(item.Filename); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete removes a single backup file.
|
||||
func (b *BackupService) Delete(filename string) error {
|
||||
path, err := b.backupFilePath(filename)
|
||||
@@ -161,12 +232,10 @@ func (b *BackupService) Delete(filename string) error {
|
||||
return os.Remove(path)
|
||||
}
|
||||
|
||||
// Restore replaces the live database with the given backup.
|
||||
// WARNING: this is destructive. SQLite restores by swapping the database file
|
||||
// (the caller should shut down the server afterwards); PostgreSQL restores by
|
||||
// rebuilding every table inside a single transaction.
|
||||
// Restore stages SQLite backups for the next startup. PostgreSQL restores
|
||||
// rebuild every table inside a single transaction.
|
||||
func (b *BackupService) Restore(ctx context.Context, filename string) error {
|
||||
src, err := b.backupFilePath(filename)
|
||||
src, err := b.ResolvePath(filename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -182,35 +251,10 @@ func (b *BackupService) Restore(ctx context.Context, filename string) error {
|
||||
if !strings.HasSuffix(filename, ".db") {
|
||||
return errors.New("该备份是 PostgreSQL 逻辑备份,当前运行在 SQLite 上,无法恢复")
|
||||
}
|
||||
return b.restoreSQLiteBackup(filename, src)
|
||||
}
|
||||
|
||||
func (b *BackupService) restoreSQLiteBackup(filename, src string) error {
|
||||
dbPath, err := filepath.Abs(b.cfg.Database.DBPath)
|
||||
if err != nil || dbPath == "" {
|
||||
return errors.New("invalid database path")
|
||||
}
|
||||
// Strategy: rename live → live.old, copy backup → live, delete old.
|
||||
old := dbPath + ".before_restore"
|
||||
if err := os.Rename(dbPath, old); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
if err := database.StageSQLiteRestore(ctx, b.cfg, src); err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := os.ReadFile(src) // #nosec G304 -- src is constrained by backupFilePath to the configured backups directory.
|
||||
if err != nil {
|
||||
// Revert
|
||||
_ = os.Rename(old, dbPath)
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(dbPath, data, 0o600); err != nil { // #nosec G703 -- database path is trusted process config and normalized before restore.
|
||||
_ = os.Rename(old, dbPath)
|
||||
return err
|
||||
}
|
||||
_ = os.Remove(old)
|
||||
// Also remove WAL/SHM so SQLite opens the fresh file cleanly.
|
||||
_ = os.Remove(dbPath + "-wal")
|
||||
_ = os.Remove(dbPath + "-shm")
|
||||
b.log.Warn("database restored from backup — restart the server",
|
||||
zap.String("backup", filename))
|
||||
b.log.Warn("SQLite backup staged; restart to apply", zap.String("backup", filename))
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ package service
|
||||
import (
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -87,7 +88,8 @@ func postgresBackupModelSchemas(db *gorm.DB) (map[string]*schema.Schema, map[str
|
||||
// createPostgresBackup 把全库导出成一个 gzip 压缩的 JSON Lines 文件。
|
||||
func (b *BackupService) createPostgresBackup(ctx context.Context, stamp string) (string, error) {
|
||||
dst := filepath.Join(b.backupDir(), "mebox_"+stamp+postgresBackupSuffix)
|
||||
file, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600)
|
||||
tmp := dst + ".tmp"
|
||||
file, err := os.OpenFile(tmp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -96,7 +98,7 @@ func (b *BackupService) createPostgresBackup(ctx context.Context, stamp string)
|
||||
_ = file.Close()
|
||||
if !complete {
|
||||
// 半截备份文件比没有备份更危险:留着会让人以为有可用的还原点。
|
||||
_ = os.Remove(dst)
|
||||
_ = os.Remove(tmp)
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -109,17 +111,30 @@ func (b *BackupService) createPostgresBackup(ctx context.Context, stamp string)
|
||||
}); err != nil {
|
||||
return "", fmt.Errorf("写入备份头失败: %w", err)
|
||||
}
|
||||
for _, m := range model.AllModels() {
|
||||
table, sch, err := parseModelSchema(b.db, m)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !b.db.Migrator().HasTable(table) {
|
||||
continue
|
||||
}
|
||||
if err := b.dumpPostgresTable(ctx, m, sch, table, enc); err != nil {
|
||||
return "", err
|
||||
var options []*sql.TxOptions
|
||||
if b.isPostgres() {
|
||||
options = append(options, &sql.TxOptions{Isolation: sql.LevelRepeatableRead, ReadOnly: true})
|
||||
}
|
||||
// Every table must come from the same snapshot, including while background
|
||||
// workers are inserting related rows during an automatic backup.
|
||||
if err := b.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
snapshot := *b
|
||||
snapshot.db = tx
|
||||
for _, m := range model.AllModels() {
|
||||
table, sch, err := parseModelSchema(tx, m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !tx.Migrator().HasTable(table) {
|
||||
continue
|
||||
}
|
||||
if err := snapshot.dumpPostgresTable(ctx, m, sch, table, enc); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}, options...); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := zw.Close(); err != nil {
|
||||
return "", fmt.Errorf("压缩备份失败: %w", err)
|
||||
@@ -127,6 +142,12 @@ func (b *BackupService) createPostgresBackup(ctx context.Context, stamp string)
|
||||
if err := file.Sync(); err != nil {
|
||||
return "", fmt.Errorf("写入备份失败: %w", err)
|
||||
}
|
||||
if err := file.Close(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := os.Rename(tmp, dst); err != nil {
|
||||
return "", err
|
||||
}
|
||||
complete = true
|
||||
return dst, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestBackupRetentionPreservesManualAndOtherDialectBackups(t *testing.T) {
|
||||
svc, _ := newBackupTestService(t)
|
||||
names := []string{"mebox_auto_new.db", "mebox_auto_old.db", "mebox_manual.db", "mebox_auto_postgres.json.gz"}
|
||||
for i, name := range names {
|
||||
path := filepath.Join(svc.backupDir(), name)
|
||||
if err := os.WriteFile(path, []byte("fixture"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
when := time.Now().Add(-time.Duration(i) * time.Hour)
|
||||
if err := os.Chtimes(path, when, when); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := svc.Prune(1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, name := range names {
|
||||
_, err := os.Stat(filepath.Join(svc.backupDir(), name))
|
||||
if name == "mebox_auto_old.db" {
|
||||
if !os.IsNotExist(err) {
|
||||
t.Fatalf("old automatic backup was retained: %v", err)
|
||||
}
|
||||
} else if err != nil {
|
||||
t.Fatalf("backup %s should have been preserved: %v", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackupCreateUsesUniquePrivateCompletedFiles(t *testing.T) {
|
||||
svc, _ := newBackupTestService(t)
|
||||
manual, err := svc.Create(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
automatic, err := svc.CreateAutomatic(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if manual.Filename == automatic.Filename || manual.Automatic || !automatic.Automatic {
|
||||
t.Fatalf("invalid backup identities: manual=%#v automatic=%#v", manual, automatic)
|
||||
}
|
||||
items, err := svc.List()
|
||||
if err != nil || len(items) != 2 {
|
||||
t.Fatalf("completed backup list: items=%#v err=%v", items, err)
|
||||
}
|
||||
for _, item := range items {
|
||||
info, err := os.Stat(item.Path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if runtime.GOOS != "windows" && info.Mode().Perm() != 0o600 {
|
||||
t.Fatalf("backup permissions are not private: %s err=%v", item.Path, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2,17 +2,71 @@ package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
"github.com/truewhile/MeBox/internal/repository"
|
||||
)
|
||||
|
||||
type MediaQualityItem struct {
|
||||
Issue string `json:"issue"`
|
||||
Media model.Media `json:"media"`
|
||||
}
|
||||
|
||||
func (s *MediaService) QualityIssues(ctx context.Context, issue string, page, pageSize int) ([]MediaQualityItem, int64, error) {
|
||||
if page < 1 {
|
||||
page = 1
|
||||
}
|
||||
if pageSize < 1 {
|
||||
pageSize = 50
|
||||
}
|
||||
if pageSize > 100 {
|
||||
pageSize = 100
|
||||
}
|
||||
q := s.repo.DB.WithContext(ctx).Model(&model.Media{})
|
||||
switch issue {
|
||||
case "missing":
|
||||
q = q.Where("availability_state = ?", "missing")
|
||||
case "unmatched":
|
||||
q = q.Where("scrape_status IN ? OR scrape_status = ''", []string{"pending", "no_match"})
|
||||
case "poster":
|
||||
q = q.Where("poster_url IS NULL OR TRIM(poster_url) = ''")
|
||||
default:
|
||||
return nil, 0, fmt.Errorf("unknown quality issue: %s", issue)
|
||||
}
|
||||
var total int64
|
||||
if err := q.Count(&total).Error; err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
var rows []model.Media
|
||||
if err := q.Order("updated_at DESC, id ASC").Offset((page - 1) * pageSize).Limit(pageSize).Find(&rows).Error; err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
s.attachLibraryMetadata(ctx, rows)
|
||||
items := make([]MediaQualityItem, 0, len(rows))
|
||||
for _, media := range rows {
|
||||
items = append(items, MediaQualityItem{Issue: issue, Media: media})
|
||||
}
|
||||
return items, total, nil
|
||||
}
|
||||
|
||||
type MediaSearchFilter struct {
|
||||
LibraryID string
|
||||
YearMin int
|
||||
YearMax int
|
||||
Genres []string
|
||||
}
|
||||
|
||||
// SearchMedia performs a simple LIKE search across titles.
|
||||
func (s *MediaService) SearchMedia(ctx context.Context, query string, limit int) ([]model.Media, error) {
|
||||
return s.SearchMediaVisible(ctx, query, limit, MediaVisibility{IncludeNSFW: true})
|
||||
}
|
||||
|
||||
func (s *MediaService) SearchMediaVisible(ctx context.Context, query string, limit int, visibility MediaVisibility) ([]model.Media, error) {
|
||||
return s.SearchMediaVisibleFiltered(ctx, query, limit, visibility, MediaSearchFilter{})
|
||||
}
|
||||
|
||||
func (s *MediaService) SearchMediaVisibleFiltered(ctx context.Context, query string, limit int, visibility MediaVisibility, filter MediaSearchFilter) ([]model.Media, error) {
|
||||
if limit <= 0 {
|
||||
limit = 50
|
||||
} else if limit > maxMediaSearchLimit {
|
||||
@@ -23,6 +77,10 @@ func (s *MediaService) SearchMediaVisible(ctx context.Context, query string, lim
|
||||
IncludeNSFW: visibility.IncludeNSFW,
|
||||
AllowedLibraryIDs: visibility.AllowedLibraryIDs,
|
||||
HiddenLibraryIDs: visibility.HiddenLibraryIDs,
|
||||
LibraryID: filter.LibraryID,
|
||||
YearMin: filter.YearMin,
|
||||
YearMax: filter.YearMax,
|
||||
Genres: filter.Genres,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -45,6 +103,10 @@ func (s *MediaService) SearchMediaVisibleGrouped(ctx context.Context, query stri
|
||||
}
|
||||
|
||||
func (s *MediaService) SearchMediaVisiblePage(ctx context.Context, query string, page, pageSize int, visibility MediaVisibility) ([]model.Media, int64, error) {
|
||||
return s.SearchMediaVisiblePageFiltered(ctx, query, page, pageSize, visibility, MediaSearchFilter{})
|
||||
}
|
||||
|
||||
func (s *MediaService) SearchMediaVisiblePageFiltered(ctx context.Context, query string, page, pageSize int, visibility MediaVisibility, filter MediaSearchFilter) ([]model.Media, int64, error) {
|
||||
if pageSize <= 0 {
|
||||
pageSize = 50
|
||||
}
|
||||
@@ -59,6 +121,10 @@ func (s *MediaService) SearchMediaVisiblePage(ctx context.Context, query string,
|
||||
IncludeNSFW: visibility.IncludeNSFW,
|
||||
AllowedLibraryIDs: visibility.AllowedLibraryIDs,
|
||||
HiddenLibraryIDs: visibility.HiddenLibraryIDs,
|
||||
LibraryID: filter.LibraryID,
|
||||
YearMin: filter.YearMin,
|
||||
YearMax: filter.YearMax,
|
||||
Genres: filter.Genres,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
|
||||
+29
-26
@@ -14,6 +14,7 @@ import (
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
@@ -144,32 +145,34 @@ type localMediaProbeTask struct {
|
||||
}
|
||||
|
||||
type existingLocalMedia struct {
|
||||
LibraryRootID string
|
||||
RelativePath string
|
||||
Title string
|
||||
OriginalName string
|
||||
EpisodeTitle string
|
||||
SizeBytes int64
|
||||
DurationSec int
|
||||
Width int
|
||||
Height int
|
||||
VideoCodec string
|
||||
AudioCodec string
|
||||
Container string
|
||||
STRMURL string
|
||||
FileID string
|
||||
PosterURL string
|
||||
BackdropURL string
|
||||
Overview string
|
||||
Year int
|
||||
ReleaseDate string
|
||||
Rating float32
|
||||
TMDbID int
|
||||
BangumiID int
|
||||
DoubanID string
|
||||
TheTVDBID string
|
||||
SeasonNum int
|
||||
EpisodeNum int
|
||||
AvailabilityState string
|
||||
MissingSince *time.Time
|
||||
LibraryRootID string
|
||||
RelativePath string
|
||||
Title string
|
||||
OriginalName string
|
||||
EpisodeTitle string
|
||||
SizeBytes int64
|
||||
DurationSec int
|
||||
Width int
|
||||
Height int
|
||||
VideoCodec string
|
||||
AudioCodec string
|
||||
Container string
|
||||
STRMURL string
|
||||
FileID string
|
||||
PosterURL string
|
||||
BackdropURL string
|
||||
Overview string
|
||||
Year int
|
||||
ReleaseDate string
|
||||
Rating float32
|
||||
TMDbID int
|
||||
BangumiID int
|
||||
DoubanID string
|
||||
TheTVDBID string
|
||||
SeasonNum int
|
||||
EpisodeNum int
|
||||
// EpisodeFraction 是集号的小数部分(S01E11.5 → 0.5);变化同样要触发刷新,
|
||||
// 否则已有行上的「半集」标记不会被写回。
|
||||
EpisodeFraction float64
|
||||
|
||||
@@ -33,7 +33,7 @@ func (s *ScannerService) existingLocalMediaSnapshotForRoot(ctx context.Context,
|
||||
query = query.Where("(library_root_id = ? OR library_root_id = '' OR library_root_id IS NULL)", rootID)
|
||||
}
|
||||
rows, err := query.
|
||||
Select("path", "library_root_id", "relative_path", "title", "original_name", "episode_title", "size_bytes", "duration_sec", "width", "height", "video_codec", "audio_codec", "container", "strm_url", "file_id", "poster_url", "backdrop_url", "overview", "year", "release_date", "rating", "tm_db_id", "bangumi_id", "douban_id", "thetvdb_id", "season_num", "episode_num", "episode_fraction", "genres", "countries", "languages", "nsfw", "scrape_status").
|
||||
Select("path", "library_root_id", "relative_path", "title", "original_name", "episode_title", "size_bytes", "duration_sec", "width", "height", "video_codec", "audio_codec", "container", "strm_url", "file_id", "poster_url", "backdrop_url", "overview", "year", "release_date", "rating", "tm_db_id", "bangumi_id", "douban_id", "thetvdb_id", "season_num", "episode_num", "episode_fraction", "genres", "countries", "languages", "nsfw", "scrape_status", "availability_state", "missing_since").
|
||||
Rows()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -80,6 +80,8 @@ func (s *ScannerService) existingLocalMediaSnapshotForRoot(ctx context.Context,
|
||||
&row.Languages,
|
||||
&row.NSFW,
|
||||
&row.ScrapeStatus,
|
||||
&row.AvailabilityState,
|
||||
&row.MissingSince,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -548,7 +548,7 @@ func TestScanLibraryMapsPersistedHostLibraryPath(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemovePathDeletesVanishedMedia(t *testing.T) {
|
||||
func TestRemovePathRetainsVanishedMedia(t *testing.T) {
|
||||
sc, repos := newScannerTestEnv(t)
|
||||
cache := NewRuntimeCacheService(&config.Config{}, zap.NewNop())
|
||||
sc.SetRuntimeCache(cache)
|
||||
@@ -589,8 +589,15 @@ func TestRemovePathDeletesVanishedMedia(t *testing.T) {
|
||||
if removed != 1 {
|
||||
t.Fatalf("removed = %d, want 1", removed)
|
||||
}
|
||||
if countMedia(t, repos) != 0 {
|
||||
t.Fatal("expected 0 media after removal")
|
||||
if countMedia(t, repos) != 1 {
|
||||
t.Fatal("missing media metadata must be retained")
|
||||
}
|
||||
var missing model.Media
|
||||
if err := repos.DB.Where("path = ?", file).First(&missing).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if missing.AvailabilityState != "missing" || missing.MissingSince == nil {
|
||||
t.Fatalf("missing state not recorded: %#v", missing)
|
||||
}
|
||||
if cache.GetJSON(t.Context(), "media:list:test", &cached) {
|
||||
t.Fatal("vanished media removal should invalidate media cache")
|
||||
|
||||
@@ -67,8 +67,16 @@ func TestScanLibraryScansMultipleRootsAndPrunesPerRoot(t *testing.T) {
|
||||
if err := db.Find(&rows).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(rows) != 1 || rows[0].Path != fileB {
|
||||
t.Fatalf("remaining rows = %#v, want offline root media preserved", rows)
|
||||
if len(rows) != 2 {
|
||||
t.Fatalf("remaining rows = %#v, want both media records preserved", rows)
|
||||
}
|
||||
for _, row := range rows {
|
||||
if row.Path == fileA && row.AvailabilityState != "missing" {
|
||||
t.Fatalf("vanished file was not marked missing: %#v", row)
|
||||
}
|
||||
if row.Path == fileB && row.AvailabilityState != "available" {
|
||||
t.Fatalf("offline root status was changed: %#v", row)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -122,7 +122,7 @@ func (s *ScannerService) localMediaScanState(in localMediaScanStateInput) (bool,
|
||||
}
|
||||
existing, exists := in.existingMedia[in.cleanPath]
|
||||
isNewMedia := !exists
|
||||
if exists && in.ext != ".strm" && existing.SizeBytes == in.size && !localMetadataNeedsRefresh(existing, in.localMeta) && !localDerivedMetadataNeedsRefresh(existing, in.incoming) {
|
||||
if exists && existing.AvailabilityState != "missing" && existing.MissingSince == nil && in.ext != ".strm" && existing.SizeBytes == in.size && !localMetadataNeedsRefresh(existing, in.localMeta) && !localDerivedMetadataNeedsRefresh(existing, in.incoming) {
|
||||
// Unchanged rows are skipped, but a row that never received ffprobe data
|
||||
// still needs a backfill probe so installing ffmpeg later repairs the library.
|
||||
s.queueProbeBackfillIfNeeded(in.path, in.ext, existing)
|
||||
|
||||
@@ -372,7 +372,14 @@ func TestScanLibraryPrunesMissingMedia(t *testing.T) {
|
||||
if err := db.Model(&model.Media{}).Where("library_id = ?", lib.ID).Count(&count).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if count != 1 {
|
||||
t.Fatalf("media count = %d, want 1", count)
|
||||
if count != 2 {
|
||||
t.Fatalf("media count = %d, want current and missing records preserved", count)
|
||||
}
|
||||
var missing model.Media
|
||||
if err := db.First(&missing, "id = ?", stale.ID).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if missing.AvailabilityState != "missing" {
|
||||
t.Fatalf("stale media state = %q, want missing", missing.AvailabilityState)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
)
|
||||
|
||||
func TestScanRestoresUnchangedMissingFileAndMetadata(t *testing.T) {
|
||||
scanner, repos := newScannerTestEnv(t)
|
||||
root := t.TempDir()
|
||||
library := model.Library{Name: "Movies", Path: root, Type: "movie", Enabled: true}
|
||||
if err := repos.Library.Create(t.Context(), &library); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
path := filepath.Join(root, "Example (2024).mkv")
|
||||
writeTestFile(t, path, "unchanged content")
|
||||
if _, err := scanner.ScanLibrary(t.Context(), library.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var original model.Media
|
||||
if err := repos.DB.Where("path = ?", path).First(&original).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := repos.DB.Model(&original).Updates(map[string]any{
|
||||
"title": "Manually matched title", "scrape_status": "matched", "tm_db_id": 42,
|
||||
}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Remove(path); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if changed, err := scanner.RemovePath(t.Context(), path); err != nil || changed != 1 {
|
||||
t.Fatalf("mark missing: changed=%d err=%v", changed, err)
|
||||
}
|
||||
if changed, err := scanner.RemovePath(t.Context(), path); err != nil || changed != 0 {
|
||||
t.Fatalf("repeated missing event: changed=%d err=%v", changed, err)
|
||||
}
|
||||
writeTestFile(t, path, "unchanged content")
|
||||
if _, err := scanner.ScanLibrary(t.Context(), library.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var restored model.Media
|
||||
if err := repos.DB.Where("path = ?", path).First(&restored).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if restored.ID != original.ID || restored.TMDbID != 42 || restored.Title != "Manually matched title" {
|
||||
t.Fatalf("restored file lost identity or metadata: %#v", restored)
|
||||
}
|
||||
if restored.AvailabilityState != "available" || restored.MissingSince != nil || restored.LastSeenAt == nil {
|
||||
t.Fatalf("unchanged restored file stayed missing: %#v", restored)
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -12,11 +13,15 @@ import (
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
)
|
||||
|
||||
// RemovePath 移除磁盘上已不存在的媒体记录。普通的删除仍做物理删除;
|
||||
// 可确认是 STRM 扩展名改名的路径则先保留软删除墓碑,供后续 ingest 继承元数据。
|
||||
// RemovePath 将确认不存在的媒体标记为失联;只有可确认的 STRM 扩展名改名
|
||||
// 才保留软删除墓碑,供后续 ingest 继承元数据。
|
||||
func (s *ScannerService) RemovePath(ctx context.Context, path string) (int64, error) {
|
||||
if _, err := os.Stat(path); err == nil {
|
||||
return 0, nil // still exists; nothing to remove
|
||||
} else if !errors.Is(err, os.ErrNotExist) {
|
||||
// Permission, I/O, and mount errors are not evidence that a file was
|
||||
// removed. Preserve records until the path can be inspected reliably.
|
||||
return 0, nil
|
||||
}
|
||||
// 目录整体消失(删除/改名离开):连同其子树下的媒体行一并移除。
|
||||
// 此前只删 path 精确匹配的行——目录本身通常没有 media 行,导致目录
|
||||
@@ -33,7 +38,7 @@ func (s *ScannerService) RemovePath(ctx context.Context, path string) (int64, er
|
||||
return 0, err
|
||||
}
|
||||
softIDs := make([]string, 0, len(rows))
|
||||
hardIDs := make([]string, 0, len(rows))
|
||||
missingIDs := make([]string, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
// LIKE 里的 % _ 是通配符(候选集只会偏大),用 Go 前缀精确过滤,
|
||||
// 避免对含 % / _ 的路径误删。
|
||||
@@ -43,12 +48,12 @@ func (s *ScannerService) RemovePath(ctx context.Context, path string) (int64, er
|
||||
// A vanished STRM with a live sibling (foo.strm <-> foo.mkv.strm)
|
||||
// is a rename. Keep a soft-deleted tombstone so the later ingest can
|
||||
// adopt its ID and scraped metadata even if fsnotify delivers the
|
||||
// remove event before the create event. A true deletion stays hard.
|
||||
// remove event before the create event.
|
||||
if row.Path == path && liveSTRMPathAlias(path) {
|
||||
softIDs = append(softIDs, row.ID)
|
||||
continue
|
||||
}
|
||||
hardIDs = append(hardIDs, row.ID)
|
||||
missingIDs = append(missingIDs, row.ID)
|
||||
}
|
||||
var removed int64
|
||||
if len(softIDs) > 0 {
|
||||
@@ -60,14 +65,12 @@ func (s *ScannerService) RemovePath(ctx context.Context, path string) (int64, er
|
||||
}
|
||||
removed += res.RowsAffected
|
||||
}
|
||||
if len(hardIDs) > 0 {
|
||||
res := s.repo.DB.WithContext(ctx).Unscoped().
|
||||
Where("id IN ?", hardIDs).
|
||||
Delete(&model.Media{})
|
||||
if res.Error != nil {
|
||||
return removed, res.Error
|
||||
if len(missingIDs) > 0 {
|
||||
count, err := s.markMediaMissingByIDs(ctx, missingIDs)
|
||||
if err != nil {
|
||||
return removed, err
|
||||
}
|
||||
removed += res.RowsAffected
|
||||
removed += count
|
||||
}
|
||||
if removed > 0 {
|
||||
s.invalidateMediaCache(ctx)
|
||||
@@ -82,7 +85,7 @@ const strmRenameTombstoneRetention = 7 * 24 * time.Hour
|
||||
|
||||
// purgeExpiredSTRMTombstones keeps the soft-delete grace period bounded.
|
||||
// It is deliberately restricted to deleted media rows under a STRM alias
|
||||
// workflow; normal deletions are still hard-deleted immediately.
|
||||
// workflow; ordinary missing files retain their metadata for recovery.
|
||||
func (s *ScannerService) purgeExpiredSTRMTombstones(ctx context.Context) {
|
||||
cutoff := time.Now().Add(-strmRenameTombstoneRetention)
|
||||
if err := s.repo.DB.WithContext(ctx).Unscoped().
|
||||
@@ -121,7 +124,7 @@ func (s *ScannerService) pruneMissingMedia(ctx context.Context, libraryID string
|
||||
}
|
||||
stale = append(stale, row.ID)
|
||||
}
|
||||
return s.deleteMediaByIDs(ctx, stale, true)
|
||||
return s.markMediaMissingByIDs(ctx, stale)
|
||||
}
|
||||
|
||||
func (s *ScannerService) pruneMissingMediaForRoot(ctx context.Context, libraryID, rootID, rootPath string, seen map[string]struct{}) (int64, error) {
|
||||
@@ -158,7 +161,33 @@ func (s *ScannerService) pruneMissingMediaForRoot(ctx context.Context, libraryID
|
||||
}
|
||||
stale = append(stale, row.ID)
|
||||
}
|
||||
return s.deleteMediaByIDs(ctx, stale, true)
|
||||
return s.markMediaMissingByIDs(ctx, stale)
|
||||
}
|
||||
|
||||
func (s *ScannerService) markMediaMissingByIDs(ctx context.Context, ids []string) (int64, error) {
|
||||
if len(ids) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
const batch = 500
|
||||
var changed int64
|
||||
now := time.Now()
|
||||
for i := 0; i < len(ids); i += batch {
|
||||
end := i + batch
|
||||
if end > len(ids) {
|
||||
end = len(ids)
|
||||
}
|
||||
result := s.repo.DB.WithContext(ctx).Model(&model.Media{}).
|
||||
Where("id IN ? AND (availability_state IS NULL OR availability_state <> ?)", ids[i:end], "missing").
|
||||
Updates(map[string]any{"availability_state": "missing", "missing_since": now})
|
||||
if result.Error != nil {
|
||||
return changed, result.Error
|
||||
}
|
||||
changed += result.RowsAffected
|
||||
}
|
||||
if changed > 0 {
|
||||
s.invalidateMediaCache(ctx)
|
||||
}
|
||||
return changed, nil
|
||||
}
|
||||
|
||||
func pathBelongsToRoot(pathValue, rootPath string) bool {
|
||||
|
||||
@@ -202,7 +202,7 @@ func (s *ScannerService) scanLocalLibraryFiles(ctx context.Context, lib *model.L
|
||||
func existingLocalMediaFileIDs(existingMedia map[string]existingLocalMedia) map[string]string {
|
||||
seenInodes := make(map[string]string)
|
||||
for path, existing := range existingMedia {
|
||||
if existing.FileID != "" {
|
||||
if existing.FileID != "" && existing.AvailabilityState != "missing" {
|
||||
seenInodes[existing.FileID] = path
|
||||
}
|
||||
}
|
||||
|
||||
@@ -175,7 +175,7 @@ func TestIngestPathAdoptsSoftDeletedSTRMRenameTombstone(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemovePathStillHardDeletesTrueDeletionWithoutAlias(t *testing.T) {
|
||||
func TestRemovePathRetainsTrueDeletionWithoutAlias(t *testing.T) {
|
||||
sc, repos := newScannerTestEnv(t)
|
||||
root := t.TempDir()
|
||||
lib := model.Library{Name: "Anime", Path: root, Type: "anime", Enabled: true}
|
||||
@@ -197,8 +197,15 @@ func TestRemovePathStillHardDeletesTrueDeletionWithoutAlias(t *testing.T) {
|
||||
if err := repos.DB.Unscoped().Model(&model.Media{}).Where("path = ?", path).Count(&count).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if count != 0 {
|
||||
t.Fatalf("true deletion left %d tombstone row(s)", count)
|
||||
if count != 1 {
|
||||
t.Fatalf("missing media count = %d, want metadata retained", count)
|
||||
}
|
||||
var media model.Media
|
||||
if err := repos.DB.Where("path = ?", path).First(&media).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if media.AvailabilityState != "missing" || media.MissingSince == nil {
|
||||
t.Fatalf("true deletion did not record missing state: %#v", media)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -50,6 +50,7 @@ type SchedulerService struct {
|
||||
readerFilesTTLHours int
|
||||
|
||||
segments *MediaSegmentService
|
||||
backup *BackupService
|
||||
|
||||
mu sync.Mutex
|
||||
stopCh chan struct{}
|
||||
@@ -65,6 +66,10 @@ func (s *SchedulerService) SetTaskTracker(tasks *TaskTrackerService) {
|
||||
s.tasks = tasks
|
||||
}
|
||||
|
||||
func (s *SchedulerService) SetBackupService(backup *BackupService) {
|
||||
s.backup = backup
|
||||
}
|
||||
|
||||
func (s *SchedulerService) SetOrganizePipeline(pipeline *OrganizePipelineService) {
|
||||
s.organizePipeline = pipeline
|
||||
}
|
||||
@@ -196,6 +201,13 @@ func (s *SchedulerService) Start(ctx context.Context) {
|
||||
run: s.jobTelegramExpiryWarning,
|
||||
})
|
||||
}
|
||||
if s.backup != nil {
|
||||
s.jobs = append(s.jobs, &scheduledJob{
|
||||
name: "database_backup",
|
||||
interval: 5 * time.Minute,
|
||||
run: s.jobDatabaseBackup,
|
||||
})
|
||||
}
|
||||
for _, j := range s.jobs {
|
||||
initialDelay := 15 * time.Second
|
||||
if j.name == "library_scan" || j.name == "organize_source" || j.name == "segment_prewarm" {
|
||||
|
||||
@@ -11,6 +11,53 @@ import (
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
const lastAutomaticBackupDateKey = "backup.last_automatic_date"
|
||||
|
||||
func (s *SchedulerService) jobDatabaseBackup(ctx context.Context) error {
|
||||
if s == nil || s.backup == nil || s.repo == nil || s.repo.Setting == nil {
|
||||
return nil
|
||||
}
|
||||
enabled, _ := s.repo.Setting.Get(ctx, "backup.auto_enabled")
|
||||
if !parseBoolSetting(enabled, false) {
|
||||
return nil
|
||||
}
|
||||
clock, _ := s.repo.Setting.Get(ctx, "backup.auto_time")
|
||||
if clock == "" {
|
||||
clock = "03:00"
|
||||
}
|
||||
parsed, err := time.Parse("15:04", clock)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
now := s.currentTime().In(time.Local)
|
||||
dueAt := time.Date(now.Year(), now.Month(), now.Day(), parsed.Hour(), parsed.Minute(), 0, 0, now.Location())
|
||||
if now.Before(dueAt) {
|
||||
return nil
|
||||
}
|
||||
date := now.Format("2006-01-02")
|
||||
last, _ := s.repo.Setting.Get(ctx, lastAutomaticBackupDateKey)
|
||||
if last == date {
|
||||
return nil
|
||||
}
|
||||
if _, err := s.backup.CreateAutomatic(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
// Record the successful backup before pruning. If retention cleanup fails,
|
||||
// the next scheduler tick must not create another copy for the same day.
|
||||
if err := s.repo.Setting.Set(ctx, lastAutomaticBackupDateKey, date); err != nil {
|
||||
return err
|
||||
}
|
||||
retentionValue, _ := s.repo.Setting.Get(ctx, "backup.retention")
|
||||
retention, err := strconv.Atoi(strings.TrimSpace(retentionValue))
|
||||
if err != nil || retention < 1 {
|
||||
retention = 7
|
||||
}
|
||||
if err := s.backup.Prune(retention); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// jobScanLibraries re-walks every enabled library.
|
||||
//
|
||||
// 默认关闭:文件变更由 WatcherService 增量入库,无需周期性全量重扫。
|
||||
|
||||
@@ -63,6 +63,9 @@ func (b *serviceContainerBuilder) startRealtimeServices() {
|
||||
b.c.WSHub = NewHub(b.log)
|
||||
helper.Go(b.log, "ws.hub", b.c.WSHub.Run)
|
||||
b.c.Tasks = NewTaskTrackerService(b.log, b.c.WSHub)
|
||||
if b.repos != nil {
|
||||
b.c.Tasks.SetPersistence(b.repos.DB)
|
||||
}
|
||||
b.c.SystemUpdate = NewSystemUpdateService(b.cfg, b.log, b.repos, b.c.Tasks, b.version)
|
||||
|
||||
b.c.SSEHub = NewSSEHub(b.log)
|
||||
@@ -206,6 +209,7 @@ func (b *serviceContainerBuilder) initAccessAndStorageServices() {
|
||||
b.c.Organizer, b.c.WSHub, b.cfg.Cache.CacheDir,
|
||||
)
|
||||
b.c.Scheduler.SetTaskTracker(b.c.Tasks)
|
||||
b.c.Scheduler.SetBackupService(b.c.Backup)
|
||||
b.c.Scheduler.SetOrganizePipeline(b.c.OrganizePipeline)
|
||||
b.c.Scheduler.SetSegments(b.c.Segments)
|
||||
b.c.Scheduler.SetImageCachePolicyProvider(func() ImageCachePolicy {
|
||||
|
||||
@@ -2,6 +2,7 @@ package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"html"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -9,12 +10,16 @@ import (
|
||||
|
||||
"github.com/google/uuid"
|
||||
"go.uber.org/zap"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
)
|
||||
|
||||
const (
|
||||
TaskStatusRunning = "running"
|
||||
TaskStatusCompleted = "completed"
|
||||
TaskStatusFailed = "failed"
|
||||
TaskStatusRunning = "running"
|
||||
TaskStatusCompleted = "completed"
|
||||
TaskStatusFailed = "failed"
|
||||
TaskStatusInterrupted = "interrupted"
|
||||
|
||||
TaskKindOrganize = "organize"
|
||||
TaskKindScan = "scan"
|
||||
@@ -70,6 +75,26 @@ type TaskTrackerService struct {
|
||||
recent []BackgroundTask
|
||||
maxRecent int
|
||||
now func() time.Time
|
||||
db *gorm.DB
|
||||
}
|
||||
|
||||
// SetPersistence enables durable task history and marks tasks left running by
|
||||
// a previous process as interrupted. Actual task execution is never resumed
|
||||
// automatically; this record is an operator-visible recovery point.
|
||||
func (t *TaskTrackerService) SetPersistence(db *gorm.DB) {
|
||||
if t == nil || db == nil {
|
||||
return
|
||||
}
|
||||
t.mu.Lock()
|
||||
t.db = db
|
||||
now := t.currentTime()
|
||||
t.mu.Unlock()
|
||||
err := db.Model(&model.BackgroundTaskRecord{}).
|
||||
Where("status = ?", TaskStatusRunning).
|
||||
Updates(map[string]any{"status": TaskStatusInterrupted, "error": "服务在任务完成前重启;请检查文件状态后重新执行", "updated_at": now, "finished_at": now}).Error
|
||||
if err != nil && t.log != nil {
|
||||
t.log.Warn("recover interrupted background tasks failed", zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
type TaskHandle struct {
|
||||
@@ -105,9 +130,11 @@ func (t *TaskTrackerService) Start(kind, name string, update TaskUpdate) *TaskHa
|
||||
StartedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
applyTaskUpdate(task, update)
|
||||
t.mu.Lock()
|
||||
t.active[task.ID] = task
|
||||
snapshot := cloneBackgroundTask(*task)
|
||||
t.persist(snapshot)
|
||||
t.mu.Unlock()
|
||||
t.publish(snapshot)
|
||||
return &TaskHandle{tracker: t, id: task.ID}
|
||||
@@ -132,18 +159,47 @@ func (t *TaskTrackerService) Snapshot() TaskSnapshot {
|
||||
return TaskSnapshot{}
|
||||
}
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
active := make([]BackgroundTask, 0, len(t.active))
|
||||
for _, task := range t.active {
|
||||
active = append(active, cloneBackgroundTask(*task))
|
||||
}
|
||||
recent := make([]BackgroundTask, 0, len(t.recent))
|
||||
db := t.db
|
||||
limit := t.maxRecent
|
||||
if limit < 1 {
|
||||
limit = 30
|
||||
}
|
||||
for _, task := range t.recent {
|
||||
recent = append(recent, cloneBackgroundTask(task))
|
||||
}
|
||||
t.mu.Unlock()
|
||||
if db != nil {
|
||||
var rows []model.BackgroundTaskRecord
|
||||
if err := db.Where("status <> ?", TaskStatusRunning).Order("started_at DESC").Limit(limit).Find(&rows).Error; err == nil {
|
||||
recent = make([]BackgroundTask, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
recent = append(recent, backgroundTaskFromRecord(row))
|
||||
}
|
||||
} else if t.log != nil {
|
||||
t.log.Warn("load background task history failed; using memory history", zap.Error(err))
|
||||
}
|
||||
}
|
||||
return TaskSnapshot{Active: active, Recent: recent}
|
||||
}
|
||||
|
||||
func backgroundTaskFromRecord(row model.BackgroundTaskRecord) BackgroundTask {
|
||||
task := BackgroundTask{
|
||||
ID: row.ID, Kind: row.Kind, Name: row.Name, Status: row.Status,
|
||||
Stage: row.Stage, SourcePath: row.SourcePath, DestPath: row.DestPath,
|
||||
Message: row.Message, Error: row.Error, StartedAt: row.StartedAt,
|
||||
UpdatedAt: row.UpdatedAt, FinishedAt: row.FinishedAt,
|
||||
}
|
||||
_ = json.Unmarshal([]byte(row.Details), &task.Details)
|
||||
_ = json.Unmarshal([]byte(row.Metrics), &task.Metrics)
|
||||
_ = json.Unmarshal([]byte(row.Items), &task.Items)
|
||||
return task
|
||||
}
|
||||
|
||||
func (t *TaskTrackerService) update(id string, update TaskUpdate) {
|
||||
now := t.currentTime()
|
||||
t.mu.Lock()
|
||||
@@ -155,6 +211,7 @@ func (t *TaskTrackerService) update(id string, update TaskUpdate) {
|
||||
applyTaskUpdate(task, update)
|
||||
task.UpdatedAt = now
|
||||
snapshot := cloneBackgroundTask(*task)
|
||||
t.persist(snapshot)
|
||||
t.mu.Unlock()
|
||||
t.publish(snapshot)
|
||||
}
|
||||
@@ -186,6 +243,7 @@ func (t *TaskTrackerService) finish(id string, err error, update TaskUpdate) {
|
||||
t.recent = t.recent[:t.maxRecent]
|
||||
}
|
||||
notifier := t.failureNotifier
|
||||
t.persist(snapshot)
|
||||
t.mu.Unlock()
|
||||
t.publish(snapshot)
|
||||
|
||||
@@ -195,6 +253,26 @@ func (t *TaskTrackerService) finish(id string, err error, update TaskUpdate) {
|
||||
}
|
||||
}
|
||||
|
||||
func (t *TaskTrackerService) persist(task BackgroundTask) {
|
||||
if t == nil || t.db == nil {
|
||||
return
|
||||
}
|
||||
encode := func(value any) string {
|
||||
data, _ := json.Marshal(value)
|
||||
return string(data)
|
||||
}
|
||||
row := model.BackgroundTaskRecord{
|
||||
ID: task.ID, Kind: task.Kind, Name: task.Name, Status: task.Status,
|
||||
Stage: task.Stage, SourcePath: task.SourcePath, DestPath: task.DestPath,
|
||||
Message: task.Message, Error: task.Error, Details: encode(task.Details),
|
||||
Metrics: encode(task.Metrics), Items: encode(task.Items), StartedAt: task.StartedAt,
|
||||
UpdatedAt: task.UpdatedAt, FinishedAt: task.FinishedAt,
|
||||
}
|
||||
if err := t.db.Save(&row).Error; err != nil && t.log != nil {
|
||||
t.log.Warn("persist background task failed", zap.String("task_id", task.ID), zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
// SetFailureNotifier 注入任务失败时的管理员通知回调。
|
||||
func (t *TaskTrackerService) SetFailureNotifier(fn func(ctx context.Context, text string)) {
|
||||
if t == nil {
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
)
|
||||
|
||||
func TestTaskTrackerPersistsHistoryAcrossRestart(t *testing.T) {
|
||||
db := newServiceTestDB(t)
|
||||
tracker := NewTaskTrackerService(zap.NewNop(), nil)
|
||||
tracker.SetPersistence(db)
|
||||
task := tracker.Start(TaskKindScan, "scan", TaskUpdate{Details: []string{"initial detail"}})
|
||||
task.Update(TaskUpdate{Metrics: map[string]int64{"visited": 42}})
|
||||
task.Finish(nil, TaskUpdate{Message: "done"})
|
||||
restarted := NewTaskTrackerService(zap.NewNop(), nil)
|
||||
restarted.SetPersistence(db)
|
||||
snapshot := restarted.Snapshot()
|
||||
if len(snapshot.Active) != 0 || len(snapshot.Recent) != 1 {
|
||||
t.Fatalf("unexpected history: %#v", snapshot)
|
||||
}
|
||||
recent := snapshot.Recent[0]
|
||||
if recent.Status != TaskStatusCompleted || recent.Metrics["visited"] != 42 || len(recent.Details) != 1 || recent.FinishedAt == nil {
|
||||
t.Fatalf("task history lost fields: %#v", recent)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTaskTrackerConcurrentUpdatesDoNotRegressFinishedTasks(t *testing.T) {
|
||||
db := newServiceTestDB(t)
|
||||
sqlDB, err := db.DB()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sqlDB.SetMaxOpenConns(1)
|
||||
tracker := NewTaskTrackerService(zap.NewNop(), nil)
|
||||
tracker.SetPersistence(db)
|
||||
var workers sync.WaitGroup
|
||||
for i := 0; i < 12; i++ {
|
||||
workers.Add(1)
|
||||
go func(index int) {
|
||||
defer workers.Done()
|
||||
task := tracker.Start(TaskKindScan, fmt.Sprintf("scan-%d", index), TaskUpdate{})
|
||||
task.Update(TaskUpdate{Metrics: map[string]int64{"visited": 1}})
|
||||
task.Finish(nil, TaskUpdate{})
|
||||
task.Update(TaskUpdate{Message: "late update"})
|
||||
_ = tracker.Snapshot()
|
||||
}(i)
|
||||
}
|
||||
workers.Wait()
|
||||
var completed int64
|
||||
if err := db.Model(&model.BackgroundTaskRecord{}).Where("status = ?", TaskStatusCompleted).Count(&completed).Error; err != nil || completed != 12 {
|
||||
t.Fatalf("finished task status regressed: completed=%d err=%v", completed, err)
|
||||
}
|
||||
if snapshot := tracker.Snapshot(); len(snapshot.Active) != 0 || len(snapshot.Recent) != 12 {
|
||||
t.Fatalf("inconsistent task snapshot: %#v", snapshot)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTaskTrackerMarksAbandonedTasksInterrupted(t *testing.T) {
|
||||
db := newServiceTestDB(t)
|
||||
tracker := NewTaskTrackerService(zap.NewNop(), nil)
|
||||
tracker.SetPersistence(db)
|
||||
tracker.Start(TaskKindOrganize, "unfinished", TaskUpdate{})
|
||||
restarted := NewTaskTrackerService(zap.NewNop(), nil)
|
||||
restarted.SetPersistence(db)
|
||||
snapshot := restarted.Snapshot()
|
||||
if len(snapshot.Recent) != 1 || snapshot.Recent[0].Status != TaskStatusInterrupted || snapshot.Recent[0].FinishedAt == nil {
|
||||
t.Fatalf("abandoned task not marked interrupted: %#v", snapshot)
|
||||
}
|
||||
var running int64
|
||||
if err := db.Model(&model.BackgroundTaskRecord{}).Where("status = ?", TaskStatusRunning).Count(&running).Error; err != nil || running != 0 {
|
||||
t.Fatalf("stale running tasks: %d err=%v", running, err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user