feat: 完善媒体维护、备份恢复与播放诊断

This commit is contained in:
jason jia
2026-10-20 14:01:26 +00:00
parent e34e81cce0
commit 1e8fe2396e
53 changed files with 2049 additions and 238 deletions
+81 -37
View File
@@ -5,7 +5,7 @@
//
// 两种方言的备份形态不同:
// - SQLite:用 "VACUUM INTO"(3.27.0 起可用)产出一个一致的库文件,
// 扩展名 .db;恢复即「替换库文件 + 重启」。
// 扩展名 .db;恢复先暂存,在下次启动打开数据库前替换文件。
// - PostgreSQL:没有单文件语义,用逻辑备份(见 backup_postgres.go),
// 扩展名 .json.gz;恢复是在单个事务内重建全部表。
package service
@@ -24,6 +24,7 @@ import (
"gorm.io/gorm"
"github.com/truewhile/MeBox/internal/config"
"github.com/truewhile/MeBox/internal/database"
)
// BackupService manages database backups.
@@ -44,6 +45,7 @@ type BackupInfo struct {
Path string `json:"path"`
Size int64 `json:"size"`
CreatedAt time.Time `json:"created_at"`
Automatic bool `json:"automatic"`
}
func (b *BackupService) backupDir() string {
@@ -69,6 +71,23 @@ func (b *BackupService) backupFilePath(filename string) (string, error) {
return path, nil
}
// ResolvePath returns the absolute path of a listed backup after applying the
// same filename and directory checks used by restore and delete operations.
func (b *BackupService) ResolvePath(filename string) (string, error) {
path, err := b.backupFilePath(filename)
if err != nil {
return "", err
}
info, err := os.Lstat(path)
if err != nil {
return "", err
}
if !info.Mode().IsRegular() {
return "", errors.New("backup is not a regular file")
}
return path, nil
}
// isPostgres reports whether the service is backed by PostgreSQL.
//
// PostgreSQL 不支持 SQLite 的 "VACUUM INTO",按文件替换恢复也会写错目标,
@@ -80,11 +99,23 @@ func (b *BackupService) isPostgres() bool {
// Create produces a new hot backup. SQLite uses "VACUUM INTO"; PostgreSQL uses
// a logical dump.
func (b *BackupService) Create(ctx context.Context) (*BackupInfo, error) {
return b.create(ctx, false)
}
func (b *BackupService) CreateAutomatic(ctx context.Context) (*BackupInfo, error) {
return b.create(ctx, true)
}
func (b *BackupService) create(ctx context.Context, automatic bool) (*BackupInfo, error) {
dir := b.backupDir()
if err := os.MkdirAll(dir, 0o750); err != nil {
return nil, err
}
ts := time.Now().Format("20060102_150405")
now := time.Now()
ts := fmt.Sprintf("%s_%09d", now.Format("20060102_150405"), now.Nanosecond())
if automatic {
ts = "auto_" + ts
}
var (
dst string
@@ -108,15 +139,24 @@ func (b *BackupService) Create(ctx context.Context) (*BackupInfo, error) {
Path: dst,
Size: stat.Size(),
CreatedAt: stat.ModTime(),
Automatic: automatic,
}, nil
}
func (b *BackupService) createSQLiteBackup(ctx context.Context, stamp string) (string, error) {
dst := filepath.Join(b.backupDir(), fmt.Sprintf("mebox_%s.db", stamp))
// VACUUM INTO creates a clean, non-WAL copy atomically.
if err := b.db.WithContext(ctx).Exec("VACUUM INTO ?", dst).Error; err != nil {
tmp := dst + ".tmp"
defer os.Remove(tmp)
// Publish only completed backups, with credentials protected by file mode.
if err := b.db.WithContext(ctx).Exec("VACUUM INTO ?", tmp).Error; err != nil {
return "", fmt.Errorf("VACUUM INTO failed: %w", err)
}
if err := os.Chmod(tmp, 0o600); err != nil {
return "", err
}
if err := os.Rename(tmp, dst); err != nil {
return "", err
}
return dst, nil
}
@@ -132,7 +172,7 @@ func (b *BackupService) List() ([]BackupInfo, error) {
}
out := make([]BackupInfo, 0, len(entries))
for _, e := range entries {
if e.IsDir() || !isBackupFilename(e.Name()) {
if !e.Type().IsRegular() || !isBackupFilename(e.Name()) {
continue
}
info, err := e.Info()
@@ -144,6 +184,7 @@ func (b *BackupService) List() ([]BackupInfo, error) {
Path: filepath.Join(dir, e.Name()),
Size: info.Size(),
CreatedAt: info.ModTime(),
Automatic: strings.HasPrefix(e.Name(), "mebox_auto_"),
})
}
sort.Slice(out, func(i, j int) bool {
@@ -152,6 +193,36 @@ func (b *BackupService) List() ([]BackupInfo, error) {
return out, nil
}
// Prune keeps maxCount automatic backups for the current database dialect.
// Manual backups and backups from another dialect are never pruned.
// A value below one disables pruning so an invalid setting cannot erase all
// recoverable copies.
func (b *BackupService) Prune(maxCount int) error {
if maxCount < 1 {
return nil
}
items, err := b.List()
if err != nil {
return err
}
eligible := items[:0]
for _, item := range items {
if item.Automatic && strings.HasSuffix(item.Filename, postgresBackupSuffix) == b.isPostgres() {
eligible = append(eligible, item)
}
}
items = eligible
if maxCount >= len(items) {
return nil
}
for _, item := range items[maxCount:] {
if err := b.Delete(item.Filename); err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
}
return nil
}
// Delete removes a single backup file.
func (b *BackupService) Delete(filename string) error {
path, err := b.backupFilePath(filename)
@@ -161,12 +232,10 @@ func (b *BackupService) Delete(filename string) error {
return os.Remove(path)
}
// Restore replaces the live database with the given backup.
// WARNING: this is destructive. SQLite restores by swapping the database file
// (the caller should shut down the server afterwards); PostgreSQL restores by
// rebuilding every table inside a single transaction.
// Restore stages SQLite backups for the next startup. PostgreSQL restores
// rebuild every table inside a single transaction.
func (b *BackupService) Restore(ctx context.Context, filename string) error {
src, err := b.backupFilePath(filename)
src, err := b.ResolvePath(filename)
if err != nil {
return err
}
@@ -182,35 +251,10 @@ func (b *BackupService) Restore(ctx context.Context, filename string) error {
if !strings.HasSuffix(filename, ".db") {
return errors.New("该备份是 PostgreSQL 逻辑备份,当前运行在 SQLite 上,无法恢复")
}
return b.restoreSQLiteBackup(filename, src)
}
func (b *BackupService) restoreSQLiteBackup(filename, src string) error {
dbPath, err := filepath.Abs(b.cfg.Database.DBPath)
if err != nil || dbPath == "" {
return errors.New("invalid database path")
}
// Strategy: rename live → live.old, copy backup → live, delete old.
old := dbPath + ".before_restore"
if err := os.Rename(dbPath, old); err != nil && !errors.Is(err, os.ErrNotExist) {
if err := database.StageSQLiteRestore(ctx, b.cfg, src); err != nil {
return err
}
data, err := os.ReadFile(src) // #nosec G304 -- src is constrained by backupFilePath to the configured backups directory.
if err != nil {
// Revert
_ = os.Rename(old, dbPath)
return err
}
if err := os.WriteFile(dbPath, data, 0o600); err != nil { // #nosec G703 -- database path is trusted process config and normalized before restore.
_ = os.Rename(old, dbPath)
return err
}
_ = os.Remove(old)
// Also remove WAL/SHM so SQLite opens the fresh file cleanly.
_ = os.Remove(dbPath + "-wal")
_ = os.Remove(dbPath + "-shm")
b.log.Warn("database restored from backup — restart the server",
zap.String("backup", filename))
b.log.Warn("SQLite backup staged; restart to apply", zap.String("backup", filename))
return nil
}
+33 -12
View File
@@ -3,6 +3,7 @@ package service
import (
"compress/gzip"
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
@@ -87,7 +88,8 @@ func postgresBackupModelSchemas(db *gorm.DB) (map[string]*schema.Schema, map[str
// createPostgresBackup 把全库导出成一个 gzip 压缩的 JSON Lines 文件。
func (b *BackupService) createPostgresBackup(ctx context.Context, stamp string) (string, error) {
dst := filepath.Join(b.backupDir(), "mebox_"+stamp+postgresBackupSuffix)
file, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600)
tmp := dst + ".tmp"
file, err := os.OpenFile(tmp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
if err != nil {
return "", err
}
@@ -96,7 +98,7 @@ func (b *BackupService) createPostgresBackup(ctx context.Context, stamp string)
_ = file.Close()
if !complete {
// 半截备份文件比没有备份更危险:留着会让人以为有可用的还原点。
_ = os.Remove(dst)
_ = os.Remove(tmp)
}
}()
@@ -109,17 +111,30 @@ func (b *BackupService) createPostgresBackup(ctx context.Context, stamp string)
}); err != nil {
return "", fmt.Errorf("写入备份头失败: %w", err)
}
for _, m := range model.AllModels() {
table, sch, err := parseModelSchema(b.db, m)
if err != nil {
return "", err
}
if !b.db.Migrator().HasTable(table) {
continue
}
if err := b.dumpPostgresTable(ctx, m, sch, table, enc); err != nil {
return "", err
var options []*sql.TxOptions
if b.isPostgres() {
options = append(options, &sql.TxOptions{Isolation: sql.LevelRepeatableRead, ReadOnly: true})
}
// Every table must come from the same snapshot, including while background
// workers are inserting related rows during an automatic backup.
if err := b.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
snapshot := *b
snapshot.db = tx
for _, m := range model.AllModels() {
table, sch, err := parseModelSchema(tx, m)
if err != nil {
return err
}
if !tx.Migrator().HasTable(table) {
continue
}
if err := snapshot.dumpPostgresTable(ctx, m, sch, table, enc); err != nil {
return err
}
}
return nil
}, options...); err != nil {
return "", err
}
if err := zw.Close(); err != nil {
return "", fmt.Errorf("压缩备份失败: %w", err)
@@ -127,6 +142,12 @@ func (b *BackupService) createPostgresBackup(ctx context.Context, stamp string)
if err := file.Sync(); err != nil {
return "", fmt.Errorf("写入备份失败: %w", err)
}
if err := file.Close(); err != nil {
return "", err
}
if err := os.Rename(tmp, dst); err != nil {
return "", err
}
complete = true
return dst, nil
}
+65
View File
@@ -0,0 +1,65 @@
package service
import (
"os"
"path/filepath"
"runtime"
"testing"
"time"
)
func TestBackupRetentionPreservesManualAndOtherDialectBackups(t *testing.T) {
svc, _ := newBackupTestService(t)
names := []string{"mebox_auto_new.db", "mebox_auto_old.db", "mebox_manual.db", "mebox_auto_postgres.json.gz"}
for i, name := range names {
path := filepath.Join(svc.backupDir(), name)
if err := os.WriteFile(path, []byte("fixture"), 0o600); err != nil {
t.Fatal(err)
}
when := time.Now().Add(-time.Duration(i) * time.Hour)
if err := os.Chtimes(path, when, when); err != nil {
t.Fatal(err)
}
}
if err := svc.Prune(1); err != nil {
t.Fatal(err)
}
for _, name := range names {
_, err := os.Stat(filepath.Join(svc.backupDir(), name))
if name == "mebox_auto_old.db" {
if !os.IsNotExist(err) {
t.Fatalf("old automatic backup was retained: %v", err)
}
} else if err != nil {
t.Fatalf("backup %s should have been preserved: %v", name, err)
}
}
}
func TestBackupCreateUsesUniquePrivateCompletedFiles(t *testing.T) {
svc, _ := newBackupTestService(t)
manual, err := svc.Create(t.Context())
if err != nil {
t.Fatal(err)
}
automatic, err := svc.CreateAutomatic(t.Context())
if err != nil {
t.Fatal(err)
}
if manual.Filename == automatic.Filename || manual.Automatic || !automatic.Automatic {
t.Fatalf("invalid backup identities: manual=%#v automatic=%#v", manual, automatic)
}
items, err := svc.List()
if err != nil || len(items) != 2 {
t.Fatalf("completed backup list: items=%#v err=%v", items, err)
}
for _, item := range items {
info, err := os.Stat(item.Path)
if err != nil {
t.Fatal(err)
}
if runtime.GOOS != "windows" && info.Mode().Perm() != 0o600 {
t.Fatalf("backup permissions are not private: %s err=%v", item.Path, err)
}
}
}
+66
View File
@@ -2,17 +2,71 @@ package service
import (
"context"
"fmt"
"github.com/truewhile/MeBox/internal/model"
"github.com/truewhile/MeBox/internal/repository"
)
type MediaQualityItem struct {
Issue string `json:"issue"`
Media model.Media `json:"media"`
}
func (s *MediaService) QualityIssues(ctx context.Context, issue string, page, pageSize int) ([]MediaQualityItem, int64, error) {
if page < 1 {
page = 1
}
if pageSize < 1 {
pageSize = 50
}
if pageSize > 100 {
pageSize = 100
}
q := s.repo.DB.WithContext(ctx).Model(&model.Media{})
switch issue {
case "missing":
q = q.Where("availability_state = ?", "missing")
case "unmatched":
q = q.Where("scrape_status IN ? OR scrape_status = ''", []string{"pending", "no_match"})
case "poster":
q = q.Where("poster_url IS NULL OR TRIM(poster_url) = ''")
default:
return nil, 0, fmt.Errorf("unknown quality issue: %s", issue)
}
var total int64
if err := q.Count(&total).Error; err != nil {
return nil, 0, err
}
var rows []model.Media
if err := q.Order("updated_at DESC, id ASC").Offset((page - 1) * pageSize).Limit(pageSize).Find(&rows).Error; err != nil {
return nil, 0, err
}
s.attachLibraryMetadata(ctx, rows)
items := make([]MediaQualityItem, 0, len(rows))
for _, media := range rows {
items = append(items, MediaQualityItem{Issue: issue, Media: media})
}
return items, total, nil
}
type MediaSearchFilter struct {
LibraryID string
YearMin int
YearMax int
Genres []string
}
// SearchMedia performs a simple LIKE search across titles.
func (s *MediaService) SearchMedia(ctx context.Context, query string, limit int) ([]model.Media, error) {
return s.SearchMediaVisible(ctx, query, limit, MediaVisibility{IncludeNSFW: true})
}
func (s *MediaService) SearchMediaVisible(ctx context.Context, query string, limit int, visibility MediaVisibility) ([]model.Media, error) {
return s.SearchMediaVisibleFiltered(ctx, query, limit, visibility, MediaSearchFilter{})
}
func (s *MediaService) SearchMediaVisibleFiltered(ctx context.Context, query string, limit int, visibility MediaVisibility, filter MediaSearchFilter) ([]model.Media, error) {
if limit <= 0 {
limit = 50
} else if limit > maxMediaSearchLimit {
@@ -23,6 +77,10 @@ func (s *MediaService) SearchMediaVisible(ctx context.Context, query string, lim
IncludeNSFW: visibility.IncludeNSFW,
AllowedLibraryIDs: visibility.AllowedLibraryIDs,
HiddenLibraryIDs: visibility.HiddenLibraryIDs,
LibraryID: filter.LibraryID,
YearMin: filter.YearMin,
YearMax: filter.YearMax,
Genres: filter.Genres,
})
if err != nil {
return nil, err
@@ -45,6 +103,10 @@ func (s *MediaService) SearchMediaVisibleGrouped(ctx context.Context, query stri
}
func (s *MediaService) SearchMediaVisiblePage(ctx context.Context, query string, page, pageSize int, visibility MediaVisibility) ([]model.Media, int64, error) {
return s.SearchMediaVisiblePageFiltered(ctx, query, page, pageSize, visibility, MediaSearchFilter{})
}
func (s *MediaService) SearchMediaVisiblePageFiltered(ctx context.Context, query string, page, pageSize int, visibility MediaVisibility, filter MediaSearchFilter) ([]model.Media, int64, error) {
if pageSize <= 0 {
pageSize = 50
}
@@ -59,6 +121,10 @@ func (s *MediaService) SearchMediaVisiblePage(ctx context.Context, query string,
IncludeNSFW: visibility.IncludeNSFW,
AllowedLibraryIDs: visibility.AllowedLibraryIDs,
HiddenLibraryIDs: visibility.HiddenLibraryIDs,
LibraryID: filter.LibraryID,
YearMin: filter.YearMin,
YearMax: filter.YearMax,
Genres: filter.Genres,
})
if err != nil {
return nil, 0, err
+29 -26
View File
@@ -14,6 +14,7 @@ import (
"errors"
"strings"
"sync"
"time"
"go.uber.org/zap"
@@ -144,32 +145,34 @@ type localMediaProbeTask struct {
}
type existingLocalMedia struct {
LibraryRootID string
RelativePath string
Title string
OriginalName string
EpisodeTitle string
SizeBytes int64
DurationSec int
Width int
Height int
VideoCodec string
AudioCodec string
Container string
STRMURL string
FileID string
PosterURL string
BackdropURL string
Overview string
Year int
ReleaseDate string
Rating float32
TMDbID int
BangumiID int
DoubanID string
TheTVDBID string
SeasonNum int
EpisodeNum int
AvailabilityState string
MissingSince *time.Time
LibraryRootID string
RelativePath string
Title string
OriginalName string
EpisodeTitle string
SizeBytes int64
DurationSec int
Width int
Height int
VideoCodec string
AudioCodec string
Container string
STRMURL string
FileID string
PosterURL string
BackdropURL string
Overview string
Year int
ReleaseDate string
Rating float32
TMDbID int
BangumiID int
DoubanID string
TheTVDBID string
SeasonNum int
EpisodeNum int
// EpisodeFraction 是集号的小数部分(S01E11.5 → 0.5);变化同样要触发刷新,
// 否则已有行上的「半集」标记不会被写回。
EpisodeFraction float64
+3 -1
View File
@@ -33,7 +33,7 @@ func (s *ScannerService) existingLocalMediaSnapshotForRoot(ctx context.Context,
query = query.Where("(library_root_id = ? OR library_root_id = '' OR library_root_id IS NULL)", rootID)
}
rows, err := query.
Select("path", "library_root_id", "relative_path", "title", "original_name", "episode_title", "size_bytes", "duration_sec", "width", "height", "video_codec", "audio_codec", "container", "strm_url", "file_id", "poster_url", "backdrop_url", "overview", "year", "release_date", "rating", "tm_db_id", "bangumi_id", "douban_id", "thetvdb_id", "season_num", "episode_num", "episode_fraction", "genres", "countries", "languages", "nsfw", "scrape_status").
Select("path", "library_root_id", "relative_path", "title", "original_name", "episode_title", "size_bytes", "duration_sec", "width", "height", "video_codec", "audio_codec", "container", "strm_url", "file_id", "poster_url", "backdrop_url", "overview", "year", "release_date", "rating", "tm_db_id", "bangumi_id", "douban_id", "thetvdb_id", "season_num", "episode_num", "episode_fraction", "genres", "countries", "languages", "nsfw", "scrape_status", "availability_state", "missing_since").
Rows()
if err != nil {
return nil, err
@@ -80,6 +80,8 @@ func (s *ScannerService) existingLocalMediaSnapshotForRoot(ctx context.Context,
&row.Languages,
&row.NSFW,
&row.ScrapeStatus,
&row.AvailabilityState,
&row.MissingSince,
); err != nil {
return nil, err
}
+10 -3
View File
@@ -548,7 +548,7 @@ func TestScanLibraryMapsPersistedHostLibraryPath(t *testing.T) {
}
}
func TestRemovePathDeletesVanishedMedia(t *testing.T) {
func TestRemovePathRetainsVanishedMedia(t *testing.T) {
sc, repos := newScannerTestEnv(t)
cache := NewRuntimeCacheService(&config.Config{}, zap.NewNop())
sc.SetRuntimeCache(cache)
@@ -589,8 +589,15 @@ func TestRemovePathDeletesVanishedMedia(t *testing.T) {
if removed != 1 {
t.Fatalf("removed = %d, want 1", removed)
}
if countMedia(t, repos) != 0 {
t.Fatal("expected 0 media after removal")
if countMedia(t, repos) != 1 {
t.Fatal("missing media metadata must be retained")
}
var missing model.Media
if err := repos.DB.Where("path = ?", file).First(&missing).Error; err != nil {
t.Fatal(err)
}
if missing.AvailabilityState != "missing" || missing.MissingSince == nil {
t.Fatalf("missing state not recorded: %#v", missing)
}
if cache.GetJSON(t.Context(), "media:list:test", &cached) {
t.Fatal("vanished media removal should invalidate media cache")
+10 -2
View File
@@ -67,8 +67,16 @@ func TestScanLibraryScansMultipleRootsAndPrunesPerRoot(t *testing.T) {
if err := db.Find(&rows).Error; err != nil {
t.Fatal(err)
}
if len(rows) != 1 || rows[0].Path != fileB {
t.Fatalf("remaining rows = %#v, want offline root media preserved", rows)
if len(rows) != 2 {
t.Fatalf("remaining rows = %#v, want both media records preserved", rows)
}
for _, row := range rows {
if row.Path == fileA && row.AvailabilityState != "missing" {
t.Fatalf("vanished file was not marked missing: %#v", row)
}
if row.Path == fileB && row.AvailabilityState != "available" {
t.Fatalf("offline root status was changed: %#v", row)
}
}
}
+1 -1
View File
@@ -122,7 +122,7 @@ func (s *ScannerService) localMediaScanState(in localMediaScanStateInput) (bool,
}
existing, exists := in.existingMedia[in.cleanPath]
isNewMedia := !exists
if exists && in.ext != ".strm" && existing.SizeBytes == in.size && !localMetadataNeedsRefresh(existing, in.localMeta) && !localDerivedMetadataNeedsRefresh(existing, in.incoming) {
if exists && existing.AvailabilityState != "missing" && existing.MissingSince == nil && in.ext != ".strm" && existing.SizeBytes == in.size && !localMetadataNeedsRefresh(existing, in.localMeta) && !localDerivedMetadataNeedsRefresh(existing, in.incoming) {
// Unchanged rows are skipped, but a row that never received ffprobe data
// still needs a backfill probe so installing ffmpeg later repairs the library.
s.queueProbeBackfillIfNeeded(in.path, in.ext, existing)
@@ -372,7 +372,14 @@ func TestScanLibraryPrunesMissingMedia(t *testing.T) {
if err := db.Model(&model.Media{}).Where("library_id = ?", lib.ID).Count(&count).Error; err != nil {
t.Fatal(err)
}
if count != 1 {
t.Fatalf("media count = %d, want 1", count)
if count != 2 {
t.Fatalf("media count = %d, want current and missing records preserved", count)
}
var missing model.Media
if err := db.First(&missing, "id = ?", stale.ID).Error; err != nil {
t.Fatal(err)
}
if missing.AvailabilityState != "missing" {
t.Fatalf("stale media state = %q, want missing", missing.AvailabilityState)
}
}
+55
View File
@@ -0,0 +1,55 @@
package service
import (
"os"
"path/filepath"
"testing"
"github.com/truewhile/MeBox/internal/model"
)
func TestScanRestoresUnchangedMissingFileAndMetadata(t *testing.T) {
scanner, repos := newScannerTestEnv(t)
root := t.TempDir()
library := model.Library{Name: "Movies", Path: root, Type: "movie", Enabled: true}
if err := repos.Library.Create(t.Context(), &library); err != nil {
t.Fatal(err)
}
path := filepath.Join(root, "Example (2024).mkv")
writeTestFile(t, path, "unchanged content")
if _, err := scanner.ScanLibrary(t.Context(), library.ID); err != nil {
t.Fatal(err)
}
var original model.Media
if err := repos.DB.Where("path = ?", path).First(&original).Error; err != nil {
t.Fatal(err)
}
if err := repos.DB.Model(&original).Updates(map[string]any{
"title": "Manually matched title", "scrape_status": "matched", "tm_db_id": 42,
}).Error; err != nil {
t.Fatal(err)
}
if err := os.Remove(path); err != nil {
t.Fatal(err)
}
if changed, err := scanner.RemovePath(t.Context(), path); err != nil || changed != 1 {
t.Fatalf("mark missing: changed=%d err=%v", changed, err)
}
if changed, err := scanner.RemovePath(t.Context(), path); err != nil || changed != 0 {
t.Fatalf("repeated missing event: changed=%d err=%v", changed, err)
}
writeTestFile(t, path, "unchanged content")
if _, err := scanner.ScanLibrary(t.Context(), library.ID); err != nil {
t.Fatal(err)
}
var restored model.Media
if err := repos.DB.Where("path = ?", path).First(&restored).Error; err != nil {
t.Fatal(err)
}
if restored.ID != original.ID || restored.TMDbID != 42 || restored.Title != "Manually matched title" {
t.Fatalf("restored file lost identity or metadata: %#v", restored)
}
if restored.AvailabilityState != "available" || restored.MissingSince != nil || restored.LastSeenAt == nil {
t.Fatalf("unchanged restored file stayed missing: %#v", restored)
}
}
+44 -15
View File
@@ -2,6 +2,7 @@ package service
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
@@ -12,11 +13,15 @@ import (
"github.com/truewhile/MeBox/internal/model"
)
// RemovePath 移除磁盘上已不存在的媒体记录。普通的删除仍做物理删除;
// 可确认是 STRM 扩展名改名的路径则先保留软删除墓碑,供后续 ingest 继承元数据。
// RemovePath 将确认不存在的媒体标记为失联;只有可确认的 STRM 扩展名改名
// 才保留软删除墓碑,供后续 ingest 继承元数据。
func (s *ScannerService) RemovePath(ctx context.Context, path string) (int64, error) {
if _, err := os.Stat(path); err == nil {
return 0, nil // still exists; nothing to remove
} else if !errors.Is(err, os.ErrNotExist) {
// Permission, I/O, and mount errors are not evidence that a file was
// removed. Preserve records until the path can be inspected reliably.
return 0, nil
}
// 目录整体消失(删除/改名离开):连同其子树下的媒体行一并移除。
// 此前只删 path 精确匹配的行——目录本身通常没有 media 行,导致目录
@@ -33,7 +38,7 @@ func (s *ScannerService) RemovePath(ctx context.Context, path string) (int64, er
return 0, err
}
softIDs := make([]string, 0, len(rows))
hardIDs := make([]string, 0, len(rows))
missingIDs := make([]string, 0, len(rows))
for _, row := range rows {
// LIKE 里的 % _ 是通配符(候选集只会偏大),用 Go 前缀精确过滤,
// 避免对含 % / _ 的路径误删。
@@ -43,12 +48,12 @@ func (s *ScannerService) RemovePath(ctx context.Context, path string) (int64, er
// A vanished STRM with a live sibling (foo.strm <-> foo.mkv.strm)
// is a rename. Keep a soft-deleted tombstone so the later ingest can
// adopt its ID and scraped metadata even if fsnotify delivers the
// remove event before the create event. A true deletion stays hard.
// remove event before the create event.
if row.Path == path && liveSTRMPathAlias(path) {
softIDs = append(softIDs, row.ID)
continue
}
hardIDs = append(hardIDs, row.ID)
missingIDs = append(missingIDs, row.ID)
}
var removed int64
if len(softIDs) > 0 {
@@ -60,14 +65,12 @@ func (s *ScannerService) RemovePath(ctx context.Context, path string) (int64, er
}
removed += res.RowsAffected
}
if len(hardIDs) > 0 {
res := s.repo.DB.WithContext(ctx).Unscoped().
Where("id IN ?", hardIDs).
Delete(&model.Media{})
if res.Error != nil {
return removed, res.Error
if len(missingIDs) > 0 {
count, err := s.markMediaMissingByIDs(ctx, missingIDs)
if err != nil {
return removed, err
}
removed += res.RowsAffected
removed += count
}
if removed > 0 {
s.invalidateMediaCache(ctx)
@@ -82,7 +85,7 @@ const strmRenameTombstoneRetention = 7 * 24 * time.Hour
// purgeExpiredSTRMTombstones keeps the soft-delete grace period bounded.
// It is deliberately restricted to deleted media rows under a STRM alias
// workflow; normal deletions are still hard-deleted immediately.
// workflow; ordinary missing files retain their metadata for recovery.
func (s *ScannerService) purgeExpiredSTRMTombstones(ctx context.Context) {
cutoff := time.Now().Add(-strmRenameTombstoneRetention)
if err := s.repo.DB.WithContext(ctx).Unscoped().
@@ -121,7 +124,7 @@ func (s *ScannerService) pruneMissingMedia(ctx context.Context, libraryID string
}
stale = append(stale, row.ID)
}
return s.deleteMediaByIDs(ctx, stale, true)
return s.markMediaMissingByIDs(ctx, stale)
}
func (s *ScannerService) pruneMissingMediaForRoot(ctx context.Context, libraryID, rootID, rootPath string, seen map[string]struct{}) (int64, error) {
@@ -158,7 +161,33 @@ func (s *ScannerService) pruneMissingMediaForRoot(ctx context.Context, libraryID
}
stale = append(stale, row.ID)
}
return s.deleteMediaByIDs(ctx, stale, true)
return s.markMediaMissingByIDs(ctx, stale)
}
func (s *ScannerService) markMediaMissingByIDs(ctx context.Context, ids []string) (int64, error) {
if len(ids) == 0 {
return 0, nil
}
const batch = 500
var changed int64
now := time.Now()
for i := 0; i < len(ids); i += batch {
end := i + batch
if end > len(ids) {
end = len(ids)
}
result := s.repo.DB.WithContext(ctx).Model(&model.Media{}).
Where("id IN ? AND (availability_state IS NULL OR availability_state <> ?)", ids[i:end], "missing").
Updates(map[string]any{"availability_state": "missing", "missing_since": now})
if result.Error != nil {
return changed, result.Error
}
changed += result.RowsAffected
}
if changed > 0 {
s.invalidateMediaCache(ctx)
}
return changed, nil
}
func pathBelongsToRoot(pathValue, rootPath string) bool {
+1 -1
View File
@@ -202,7 +202,7 @@ func (s *ScannerService) scanLocalLibraryFiles(ctx context.Context, lib *model.L
func existingLocalMediaFileIDs(existingMedia map[string]existingLocalMedia) map[string]string {
seenInodes := make(map[string]string)
for path, existing := range existingMedia {
if existing.FileID != "" {
if existing.FileID != "" && existing.AvailabilityState != "missing" {
seenInodes[existing.FileID] = path
}
}
+10 -3
View File
@@ -175,7 +175,7 @@ func TestIngestPathAdoptsSoftDeletedSTRMRenameTombstone(t *testing.T) {
}
}
func TestRemovePathStillHardDeletesTrueDeletionWithoutAlias(t *testing.T) {
func TestRemovePathRetainsTrueDeletionWithoutAlias(t *testing.T) {
sc, repos := newScannerTestEnv(t)
root := t.TempDir()
lib := model.Library{Name: "Anime", Path: root, Type: "anime", Enabled: true}
@@ -197,8 +197,15 @@ func TestRemovePathStillHardDeletesTrueDeletionWithoutAlias(t *testing.T) {
if err := repos.DB.Unscoped().Model(&model.Media{}).Where("path = ?", path).Count(&count).Error; err != nil {
t.Fatal(err)
}
if count != 0 {
t.Fatalf("true deletion left %d tombstone row(s)", count)
if count != 1 {
t.Fatalf("missing media count = %d, want metadata retained", count)
}
var media model.Media
if err := repos.DB.Where("path = ?", path).First(&media).Error; err != nil {
t.Fatal(err)
}
if media.AvailabilityState != "missing" || media.MissingSince == nil {
t.Fatalf("true deletion did not record missing state: %#v", media)
}
}
+12
View File
@@ -50,6 +50,7 @@ type SchedulerService struct {
readerFilesTTLHours int
segments *MediaSegmentService
backup *BackupService
mu sync.Mutex
stopCh chan struct{}
@@ -65,6 +66,10 @@ func (s *SchedulerService) SetTaskTracker(tasks *TaskTrackerService) {
s.tasks = tasks
}
func (s *SchedulerService) SetBackupService(backup *BackupService) {
s.backup = backup
}
func (s *SchedulerService) SetOrganizePipeline(pipeline *OrganizePipelineService) {
s.organizePipeline = pipeline
}
@@ -196,6 +201,13 @@ func (s *SchedulerService) Start(ctx context.Context) {
run: s.jobTelegramExpiryWarning,
})
}
if s.backup != nil {
s.jobs = append(s.jobs, &scheduledJob{
name: "database_backup",
interval: 5 * time.Minute,
run: s.jobDatabaseBackup,
})
}
for _, j := range s.jobs {
initialDelay := 15 * time.Second
if j.name == "library_scan" || j.name == "organize_source" || j.name == "segment_prewarm" {
+47
View File
@@ -11,6 +11,53 @@ import (
"go.uber.org/zap"
)
const lastAutomaticBackupDateKey = "backup.last_automatic_date"
func (s *SchedulerService) jobDatabaseBackup(ctx context.Context) error {
if s == nil || s.backup == nil || s.repo == nil || s.repo.Setting == nil {
return nil
}
enabled, _ := s.repo.Setting.Get(ctx, "backup.auto_enabled")
if !parseBoolSetting(enabled, false) {
return nil
}
clock, _ := s.repo.Setting.Get(ctx, "backup.auto_time")
if clock == "" {
clock = "03:00"
}
parsed, err := time.Parse("15:04", clock)
if err != nil {
return nil
}
now := s.currentTime().In(time.Local)
dueAt := time.Date(now.Year(), now.Month(), now.Day(), parsed.Hour(), parsed.Minute(), 0, 0, now.Location())
if now.Before(dueAt) {
return nil
}
date := now.Format("2006-01-02")
last, _ := s.repo.Setting.Get(ctx, lastAutomaticBackupDateKey)
if last == date {
return nil
}
if _, err := s.backup.CreateAutomatic(ctx); err != nil {
return err
}
// Record the successful backup before pruning. If retention cleanup fails,
// the next scheduler tick must not create another copy for the same day.
if err := s.repo.Setting.Set(ctx, lastAutomaticBackupDateKey, date); err != nil {
return err
}
retentionValue, _ := s.repo.Setting.Get(ctx, "backup.retention")
retention, err := strconv.Atoi(strings.TrimSpace(retentionValue))
if err != nil || retention < 1 {
retention = 7
}
if err := s.backup.Prune(retention); err != nil {
return err
}
return nil
}
// jobScanLibraries re-walks every enabled library.
//
// 默认关闭:文件变更由 WatcherService 增量入库,无需周期性全量重扫。
+4
View File
@@ -63,6 +63,9 @@ func (b *serviceContainerBuilder) startRealtimeServices() {
b.c.WSHub = NewHub(b.log)
helper.Go(b.log, "ws.hub", b.c.WSHub.Run)
b.c.Tasks = NewTaskTrackerService(b.log, b.c.WSHub)
if b.repos != nil {
b.c.Tasks.SetPersistence(b.repos.DB)
}
b.c.SystemUpdate = NewSystemUpdateService(b.cfg, b.log, b.repos, b.c.Tasks, b.version)
b.c.SSEHub = NewSSEHub(b.log)
@@ -206,6 +209,7 @@ func (b *serviceContainerBuilder) initAccessAndStorageServices() {
b.c.Organizer, b.c.WSHub, b.cfg.Cache.CacheDir,
)
b.c.Scheduler.SetTaskTracker(b.c.Tasks)
b.c.Scheduler.SetBackupService(b.c.Backup)
b.c.Scheduler.SetOrganizePipeline(b.c.OrganizePipeline)
b.c.Scheduler.SetSegments(b.c.Segments)
b.c.Scheduler.SetImageCachePolicyProvider(func() ImageCachePolicy {
+82 -4
View File
@@ -2,6 +2,7 @@ package service
import (
"context"
"encoding/json"
"html"
"strings"
"sync"
@@ -9,12 +10,16 @@ import (
"github.com/google/uuid"
"go.uber.org/zap"
"gorm.io/gorm"
"github.com/truewhile/MeBox/internal/model"
)
const (
TaskStatusRunning = "running"
TaskStatusCompleted = "completed"
TaskStatusFailed = "failed"
TaskStatusRunning = "running"
TaskStatusCompleted = "completed"
TaskStatusFailed = "failed"
TaskStatusInterrupted = "interrupted"
TaskKindOrganize = "organize"
TaskKindScan = "scan"
@@ -70,6 +75,26 @@ type TaskTrackerService struct {
recent []BackgroundTask
maxRecent int
now func() time.Time
db *gorm.DB
}
// SetPersistence enables durable task history and marks tasks left running by
// a previous process as interrupted. Actual task execution is never resumed
// automatically; this record is an operator-visible recovery point.
func (t *TaskTrackerService) SetPersistence(db *gorm.DB) {
if t == nil || db == nil {
return
}
t.mu.Lock()
t.db = db
now := t.currentTime()
t.mu.Unlock()
err := db.Model(&model.BackgroundTaskRecord{}).
Where("status = ?", TaskStatusRunning).
Updates(map[string]any{"status": TaskStatusInterrupted, "error": "服务在任务完成前重启;请检查文件状态后重新执行", "updated_at": now, "finished_at": now}).Error
if err != nil && t.log != nil {
t.log.Warn("recover interrupted background tasks failed", zap.Error(err))
}
}
type TaskHandle struct {
@@ -105,9 +130,11 @@ func (t *TaskTrackerService) Start(kind, name string, update TaskUpdate) *TaskHa
StartedAt: now,
UpdatedAt: now,
}
applyTaskUpdate(task, update)
t.mu.Lock()
t.active[task.ID] = task
snapshot := cloneBackgroundTask(*task)
t.persist(snapshot)
t.mu.Unlock()
t.publish(snapshot)
return &TaskHandle{tracker: t, id: task.ID}
@@ -132,18 +159,47 @@ func (t *TaskTrackerService) Snapshot() TaskSnapshot {
return TaskSnapshot{}
}
t.mu.Lock()
defer t.mu.Unlock()
active := make([]BackgroundTask, 0, len(t.active))
for _, task := range t.active {
active = append(active, cloneBackgroundTask(*task))
}
recent := make([]BackgroundTask, 0, len(t.recent))
db := t.db
limit := t.maxRecent
if limit < 1 {
limit = 30
}
for _, task := range t.recent {
recent = append(recent, cloneBackgroundTask(task))
}
t.mu.Unlock()
if db != nil {
var rows []model.BackgroundTaskRecord
if err := db.Where("status <> ?", TaskStatusRunning).Order("started_at DESC").Limit(limit).Find(&rows).Error; err == nil {
recent = make([]BackgroundTask, 0, len(rows))
for _, row := range rows {
recent = append(recent, backgroundTaskFromRecord(row))
}
} else if t.log != nil {
t.log.Warn("load background task history failed; using memory history", zap.Error(err))
}
}
return TaskSnapshot{Active: active, Recent: recent}
}
func backgroundTaskFromRecord(row model.BackgroundTaskRecord) BackgroundTask {
task := BackgroundTask{
ID: row.ID, Kind: row.Kind, Name: row.Name, Status: row.Status,
Stage: row.Stage, SourcePath: row.SourcePath, DestPath: row.DestPath,
Message: row.Message, Error: row.Error, StartedAt: row.StartedAt,
UpdatedAt: row.UpdatedAt, FinishedAt: row.FinishedAt,
}
_ = json.Unmarshal([]byte(row.Details), &task.Details)
_ = json.Unmarshal([]byte(row.Metrics), &task.Metrics)
_ = json.Unmarshal([]byte(row.Items), &task.Items)
return task
}
func (t *TaskTrackerService) update(id string, update TaskUpdate) {
now := t.currentTime()
t.mu.Lock()
@@ -155,6 +211,7 @@ func (t *TaskTrackerService) update(id string, update TaskUpdate) {
applyTaskUpdate(task, update)
task.UpdatedAt = now
snapshot := cloneBackgroundTask(*task)
t.persist(snapshot)
t.mu.Unlock()
t.publish(snapshot)
}
@@ -186,6 +243,7 @@ func (t *TaskTrackerService) finish(id string, err error, update TaskUpdate) {
t.recent = t.recent[:t.maxRecent]
}
notifier := t.failureNotifier
t.persist(snapshot)
t.mu.Unlock()
t.publish(snapshot)
@@ -195,6 +253,26 @@ func (t *TaskTrackerService) finish(id string, err error, update TaskUpdate) {
}
}
func (t *TaskTrackerService) persist(task BackgroundTask) {
if t == nil || t.db == nil {
return
}
encode := func(value any) string {
data, _ := json.Marshal(value)
return string(data)
}
row := model.BackgroundTaskRecord{
ID: task.ID, Kind: task.Kind, Name: task.Name, Status: task.Status,
Stage: task.Stage, SourcePath: task.SourcePath, DestPath: task.DestPath,
Message: task.Message, Error: task.Error, Details: encode(task.Details),
Metrics: encode(task.Metrics), Items: encode(task.Items), StartedAt: task.StartedAt,
UpdatedAt: task.UpdatedAt, FinishedAt: task.FinishedAt,
}
if err := t.db.Save(&row).Error; err != nil && t.log != nil {
t.log.Warn("persist background task failed", zap.String("task_id", task.ID), zap.Error(err))
}
}
// SetFailureNotifier 注入任务失败时的管理员通知回调。
func (t *TaskTrackerService) SetFailureNotifier(fn func(ctx context.Context, text string)) {
if t == nil {
@@ -0,0 +1,78 @@
package service
import (
"fmt"
"sync"
"testing"
"go.uber.org/zap"
"github.com/truewhile/MeBox/internal/model"
)
func TestTaskTrackerPersistsHistoryAcrossRestart(t *testing.T) {
db := newServiceTestDB(t)
tracker := NewTaskTrackerService(zap.NewNop(), nil)
tracker.SetPersistence(db)
task := tracker.Start(TaskKindScan, "scan", TaskUpdate{Details: []string{"initial detail"}})
task.Update(TaskUpdate{Metrics: map[string]int64{"visited": 42}})
task.Finish(nil, TaskUpdate{Message: "done"})
restarted := NewTaskTrackerService(zap.NewNop(), nil)
restarted.SetPersistence(db)
snapshot := restarted.Snapshot()
if len(snapshot.Active) != 0 || len(snapshot.Recent) != 1 {
t.Fatalf("unexpected history: %#v", snapshot)
}
recent := snapshot.Recent[0]
if recent.Status != TaskStatusCompleted || recent.Metrics["visited"] != 42 || len(recent.Details) != 1 || recent.FinishedAt == nil {
t.Fatalf("task history lost fields: %#v", recent)
}
}
func TestTaskTrackerConcurrentUpdatesDoNotRegressFinishedTasks(t *testing.T) {
db := newServiceTestDB(t)
sqlDB, err := db.DB()
if err != nil {
t.Fatal(err)
}
sqlDB.SetMaxOpenConns(1)
tracker := NewTaskTrackerService(zap.NewNop(), nil)
tracker.SetPersistence(db)
var workers sync.WaitGroup
for i := 0; i < 12; i++ {
workers.Add(1)
go func(index int) {
defer workers.Done()
task := tracker.Start(TaskKindScan, fmt.Sprintf("scan-%d", index), TaskUpdate{})
task.Update(TaskUpdate{Metrics: map[string]int64{"visited": 1}})
task.Finish(nil, TaskUpdate{})
task.Update(TaskUpdate{Message: "late update"})
_ = tracker.Snapshot()
}(i)
}
workers.Wait()
var completed int64
if err := db.Model(&model.BackgroundTaskRecord{}).Where("status = ?", TaskStatusCompleted).Count(&completed).Error; err != nil || completed != 12 {
t.Fatalf("finished task status regressed: completed=%d err=%v", completed, err)
}
if snapshot := tracker.Snapshot(); len(snapshot.Active) != 0 || len(snapshot.Recent) != 12 {
t.Fatalf("inconsistent task snapshot: %#v", snapshot)
}
}
func TestTaskTrackerMarksAbandonedTasksInterrupted(t *testing.T) {
db := newServiceTestDB(t)
tracker := NewTaskTrackerService(zap.NewNop(), nil)
tracker.SetPersistence(db)
tracker.Start(TaskKindOrganize, "unfinished", TaskUpdate{})
restarted := NewTaskTrackerService(zap.NewNop(), nil)
restarted.SetPersistence(db)
snapshot := restarted.Snapshot()
if len(snapshot.Recent) != 1 || snapshot.Recent[0].Status != TaskStatusInterrupted || snapshot.Recent[0].FinishedAt == nil {
t.Fatalf("abandoned task not marked interrupted: %#v", snapshot)
}
var running int64
if err := db.Model(&model.BackgroundTaskRecord{}).Where("status = ?", TaskStatusRunning).Count(&running).Error; err != nil || running != 0 {
t.Fatalf("stale running tasks: %d err=%v", running, err)
}
}