mirror of
https://github.com/truewhile/MeBox.git
synced 2026-11-16 12:54:22 +08:00
feat: 完善媒体维护、备份恢复与播放诊断
This commit is contained in:
@@ -0,0 +1,189 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/config"
|
||||
)
|
||||
|
||||
var sqliteRestoreMu sync.Mutex
|
||||
|
||||
func sqliteRestorePath(cfg *config.Config) (string, error) {
|
||||
if cfg == nil || strings.TrimSpace(cfg.Database.DBPath) == "" || cfg.Database.DBPath == ":memory:" || strings.HasPrefix(cfg.Database.DBPath, "file:") {
|
||||
return "", errors.New("恢复需要配置有效的 SQLite 数据库文件路径")
|
||||
}
|
||||
return filepath.Abs(cfg.Database.DBPath)
|
||||
}
|
||||
|
||||
func sqliteRestoreDSN(path, mode string) string {
|
||||
u := url.URL{Scheme: "file", Path: filepath.ToSlash(path)}
|
||||
q := url.Values{"mode": {mode}, "_pragma": {"busy_timeout(5000)"}}
|
||||
u.RawQuery = q.Encode()
|
||||
return u.String()
|
||||
}
|
||||
|
||||
func validateSQLiteRestore(ctx context.Context, path string) error {
|
||||
info, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
return errors.New("备份必须是常规文件")
|
||||
}
|
||||
db, err := sql.Open("sqlite", sqliteRestoreDSN(path, "ro"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer db.Close()
|
||||
var result string
|
||||
if err := db.QueryRowContext(ctx, "PRAGMA quick_check").Scan(&result); err != nil {
|
||||
return fmt.Errorf("SQLite 备份校验失败: %w", err)
|
||||
}
|
||||
if result != "ok" {
|
||||
return fmt.Errorf("SQLite 备份校验失败: %s", result)
|
||||
}
|
||||
var tables int
|
||||
if err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name IN ('users', 'media', 'settings')").Scan(&tables); err != nil {
|
||||
return err
|
||||
}
|
||||
if tables == 0 {
|
||||
return errors.New("备份不包含 MeBox 数据表")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// StageSQLiteRestore leaves the live database and its WAL untouched. The
|
||||
// validated copy is applied on the next startup, before any database is opened.
|
||||
func StageSQLiteRestore(ctx context.Context, cfg *config.Config, source string) error {
|
||||
sqliteRestoreMu.Lock()
|
||||
defer sqliteRestoreMu.Unlock()
|
||||
dbPath, err := sqliteRestorePath(cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pending := dbPath + ".restore-pending"
|
||||
if _, err := os.Lstat(pending); err == nil {
|
||||
return errors.New("已有等待恢复的备份,请重启 MeBox 后再操作")
|
||||
} else if !errors.Is(err, os.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
if err := validateSQLiteRestore(ctx, source); err != nil {
|
||||
return err
|
||||
}
|
||||
in, err := os.Open(source)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer in.Close()
|
||||
out, err := os.CreateTemp(filepath.Dir(dbPath), ".mebox-restore-*")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tmp := out.Name()
|
||||
defer os.Remove(tmp)
|
||||
if _, err := io.Copy(out, in); err != nil {
|
||||
_ = out.Close()
|
||||
return err
|
||||
}
|
||||
if err := out.Sync(); err != nil {
|
||||
_ = out.Close()
|
||||
return err
|
||||
}
|
||||
if err := out.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateSQLiteRestore(ctx, tmp); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, pending)
|
||||
}
|
||||
|
||||
// ApplyPendingSQLiteRestore must run before database.Open and before workers or
|
||||
// HTTP listeners start. The previous database is retained alongside the file.
|
||||
func ApplyPendingSQLiteRestore(ctx context.Context, cfg *config.Config, log *zap.Logger) error {
|
||||
if cfg == nil {
|
||||
return nil
|
||||
}
|
||||
dialect := normalizeDatabaseType(cfg.Database.Type)
|
||||
if dialect == "auto" {
|
||||
dialect = effectiveAutoDatabaseType(cfg)
|
||||
}
|
||||
if dialect != "sqlite" {
|
||||
return nil
|
||||
}
|
||||
sqliteRestoreMu.Lock()
|
||||
defer sqliteRestoreMu.Unlock()
|
||||
dbPath, err := sqliteRestorePath(cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pending := dbPath + ".restore-pending"
|
||||
if _, err := os.Lstat(pending); errors.Is(err, os.ErrNotExist) {
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateSQLiteRestore(ctx, pending); err != nil {
|
||||
return err
|
||||
}
|
||||
previous := ""
|
||||
if _, err := os.Lstat(dbPath); err == nil {
|
||||
// A clean checkpoint prevents an old WAL from being replayed into the
|
||||
// replacement database. A busy checkpoint leaves all files untouched.
|
||||
live, err := sql.Open("sqlite", sqliteRestoreDSN(dbPath, "rw"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var busy, pages, copied int
|
||||
err = live.QueryRowContext(ctx, "PRAGMA wal_checkpoint(TRUNCATE)").Scan(&busy, &pages, &copied)
|
||||
closeErr := live.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if closeErr != nil {
|
||||
return closeErr
|
||||
}
|
||||
if busy != 0 {
|
||||
return errors.New("SQLite 数据库仍被占用,恢复已保留等待下次启动")
|
||||
}
|
||||
old, err := os.CreateTemp(filepath.Dir(dbPath), filepath.Base(dbPath)+".before-restore-*")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
previous = old.Name()
|
||||
if err := old.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Remove(previous); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(dbPath, previous); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if !errors.Is(err, os.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(pending, dbPath); err != nil {
|
||||
if previous != "" {
|
||||
if rollbackErr := os.Rename(previous, dbPath); rollbackErr != nil {
|
||||
return fmt.Errorf("恢复失败: %w;回退失败: %v,原数据库保留于 %s", err, rollbackErr, previous)
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
if log != nil {
|
||||
log.Warn("SQLite backup restored before startup", zap.String("previous_database", previous))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/config"
|
||||
)
|
||||
|
||||
func TestSQLiteRestoreStagesWithoutChangingLiveDatabase(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
cfg := &config.Config{}
|
||||
cfg.Database.Type = "sqlite"
|
||||
cfg.Database.DBPath = filepath.Join(root, "live.db")
|
||||
live, err := sql.Open("sqlite", sqliteRestoreDSN(cfg.Database.DBPath, "rwc"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer live.Close()
|
||||
for _, statement := range []string{
|
||||
"PRAGMA journal_mode=WAL", "CREATE TABLE settings (key TEXT PRIMARY KEY, value TEXT)",
|
||||
"INSERT INTO settings VALUES ('state', 'backup')",
|
||||
} {
|
||||
if _, err := live.Exec(statement); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
source := filepath.Join(root, "backup.db")
|
||||
if _, err := live.Exec("VACUUM INTO ?", source); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := live.Exec("UPDATE settings SET value = 'live'"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := StageSQLiteRestore(t.Context(), cfg, source); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var value string
|
||||
if err := live.QueryRow("SELECT value FROM settings").Scan(&value); err != nil || value != "live" {
|
||||
t.Fatalf("staging changed live database: value=%q err=%v", value, err)
|
||||
}
|
||||
if _, err := live.Exec("UPDATE settings SET value = 'after-stage'"); err != nil {
|
||||
t.Fatalf("database could not write after staging: %v", err)
|
||||
}
|
||||
if err := StageSQLiteRestore(t.Context(), cfg, source); err == nil {
|
||||
t.Fatal("second staging should not replace the pending restore")
|
||||
}
|
||||
if err := os.Remove(source); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := live.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ApplyPendingSQLiteRestore(t.Context(), cfg, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
restored, err := sql.Open("sqlite", sqliteRestoreDSN(cfg.Database.DBPath, "ro"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer restored.Close()
|
||||
if err := restored.QueryRow("SELECT value FROM settings").Scan(&value); err != nil || value != "backup" {
|
||||
t.Fatalf("restore failed: value=%q err=%v", value, err)
|
||||
}
|
||||
previous, err := filepath.Glob(cfg.Database.DBPath + ".before-restore-*")
|
||||
if err != nil || len(previous) != 1 {
|
||||
t.Fatalf("previous database not preserved: files=%v err=%v", previous, err)
|
||||
}
|
||||
old, err := sql.Open("sqlite", sqliteRestoreDSN(previous[0], "ro"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer old.Close()
|
||||
if err := old.QueryRow("SELECT value FROM settings").Scan(&value); err != nil || value != "after-stage" {
|
||||
t.Fatalf("old database lost latest writes: value=%q err=%v", value, err)
|
||||
}
|
||||
if err := ApplyPendingSQLiteRestore(t.Context(), cfg, nil); err != nil {
|
||||
t.Fatalf("repeat startup: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSQLiteRestoreRejectsCorruptAndSymlinkBackups(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
cfg := &config.Config{}
|
||||
cfg.Database.DBPath = filepath.Join(root, "live.db")
|
||||
source := filepath.Join(root, "corrupt.db")
|
||||
if err := os.WriteFile(source, []byte("not a SQLite database"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := StageSQLiteRestore(t.Context(), cfg, source); err == nil {
|
||||
t.Fatal("corrupt backup accepted")
|
||||
}
|
||||
link := filepath.Join(root, "link.db")
|
||||
if err := os.Symlink(source, link); err == nil {
|
||||
if err := StageSQLiteRestore(t.Context(), cfg, link); err == nil {
|
||||
t.Fatal("symlink backup accepted")
|
||||
}
|
||||
}
|
||||
if _, err := os.Stat(cfg.Database.DBPath + ".restore-pending"); !os.IsNotExist(err) {
|
||||
t.Fatalf("failed validation left pending data: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/truewhile/MeBox/internal/service"
|
||||
)
|
||||
|
||||
func backgroundTasksHandler(svc *service.Container) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
if svc == nil || svc.Tasks == nil {
|
||||
c.JSON(http.StatusOK, service.TaskSnapshot{Active: []service.BackgroundTask{}, Recent: []service.BackgroundTask{}})
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, svc.Tasks.Snapshot())
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package handler
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
@@ -41,12 +42,28 @@ func deleteBackupHandler(svc *service.Container) gin.HandlerFunc {
|
||||
}
|
||||
}
|
||||
|
||||
func downloadBackupHandler(svc *service.Container) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
filename := c.Query("filename")
|
||||
path, err := svc.Backup.ResolvePath(filename)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
c.FileAttachment(path, filepath.Base(path))
|
||||
}
|
||||
}
|
||||
|
||||
func restoreBackupHandler(svc *service.Container) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
if err := svc.Backup.Restore(c.Request.Context(), c.Query("filename")); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"message": "restored — restart the server to apply"})
|
||||
message := "数据库已恢复,请重启 MeBox 以刷新运行时配置"
|
||||
if svc.Repo.DB.Dialector.Name() == "sqlite" {
|
||||
message = "备份已校验并暂存,请重启 MeBox 后应用恢复"
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"message": message, "restart_required": true})
|
||||
}
|
||||
}
|
||||
|
||||
+146
-105
@@ -6,6 +6,7 @@ import (
|
||||
"errors"
|
||||
"math"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -654,6 +655,9 @@ func paginateSlice[T any](items []T, page, size int) []T {
|
||||
if len(items) == 0 {
|
||||
return []T{}
|
||||
}
|
||||
if page-1 > len(items)/size {
|
||||
return []T{}
|
||||
}
|
||||
start := (page - 1) * size
|
||||
if start >= len(items) {
|
||||
return []T{}
|
||||
@@ -668,127 +672,164 @@ func paginateSlice[T any](items []T, page, size int) []T {
|
||||
func searchMediaHandler(svc *service.Container) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
ctx := c.Request.Context()
|
||||
q := c.Query("q")
|
||||
query := strings.TrimSpace(c.Query("q"))
|
||||
visibility := mediaVisibilityForRequest(c, svc)
|
||||
groupVersions := c.DefaultQuery("group_versions", "1") != "0"
|
||||
|
||||
fetchRemote := func(limit int) []model.Media {
|
||||
if svc.EmbyRemote == nil || strings.TrimSpace(q) == "" {
|
||||
return nil
|
||||
}
|
||||
remoteItems, _ := svc.EmbyRemote.RemoteSearchMedia(ctx, q, limit, visibility)
|
||||
return remoteItems
|
||||
filter := service.MediaSearchFilter{
|
||||
LibraryID: strings.TrimSpace(c.Query("library_id")),
|
||||
YearMin: boundedSearchYear(c.Query("year_min")),
|
||||
YearMax: boundedSearchYear(c.Query("year_max")),
|
||||
}
|
||||
|
||||
if c.DefaultQuery("group_series", "0") != "0" {
|
||||
localItems, err := svc.Media.SearchMediaVisible(ctx, q, 50000, visibility)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
for _, genre := range c.QueryArray("genre") {
|
||||
if genre = strings.TrimSpace(genre); genre != "" && len(filter.Genres) < 8 {
|
||||
filter.Genres = append(filter.Genres, genre)
|
||||
}
|
||||
remoteItems := fetchRemote(50000)
|
||||
all := service.GroupMediaSeriesItems(append(localItems, remoteItems...))
|
||||
|
||||
if c.Query("page") != "" || c.Query("page_size") != "" {
|
||||
page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
|
||||
size, _ := strconv.Atoi(c.DefaultQuery("page_size", "50"))
|
||||
paged := paginateSlice(all, page, size)
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"items": paged,
|
||||
"total": len(all),
|
||||
"page": page,
|
||||
"page_size": size,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
limit, _ := strconv.Atoi(c.DefaultQuery("limit", "50"))
|
||||
if limit <= 0 {
|
||||
limit = 50
|
||||
}
|
||||
if len(all) > limit {
|
||||
all = all[:limit]
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"items": all})
|
||||
return
|
||||
}
|
||||
|
||||
if c.Query("page") != "" || c.Query("page_size") != "" {
|
||||
page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
|
||||
size, _ := strconv.Atoi(c.DefaultQuery("page_size", "50"))
|
||||
if !groupVersions {
|
||||
localItems, _, err := svc.Media.SearchMediaVisiblePage(ctx, q, 1, 50000, visibility)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
remoteItems := fetchRemote(size * 2)
|
||||
all := append(localItems, remoteItems...)
|
||||
paged := paginateSlice(all, page, size)
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"items": paged,
|
||||
"total": len(all),
|
||||
"page": page,
|
||||
"page_size": size,
|
||||
})
|
||||
return
|
||||
}
|
||||
localItems, err := svc.Media.SearchMediaVisible(ctx, q, 50000, visibility)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
remoteItems := fetchRemote(size * 2)
|
||||
all := append(localItems, remoteItems...)
|
||||
grouped := service.GroupMediaVersions(all)
|
||||
paged := service.PaginateMediaItems(grouped, page, size)
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"items": paged,
|
||||
"total": len(grouped),
|
||||
"page": page,
|
||||
"page_size": size,
|
||||
})
|
||||
return
|
||||
if filter.YearMin > 0 && filter.YearMax > 0 && filter.YearMin > filter.YearMax {
|
||||
filter.YearMin, filter.YearMax = filter.YearMax, filter.YearMin
|
||||
}
|
||||
page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
|
||||
if page < 1 {
|
||||
page = 1
|
||||
}
|
||||
size, _ := strconv.Atoi(c.DefaultQuery("page_size", "50"))
|
||||
if size < 1 {
|
||||
size = 50
|
||||
} else if size > 2000 {
|
||||
size = 2000
|
||||
}
|
||||
|
||||
limit, _ := strconv.Atoi(c.DefaultQuery("limit", "50"))
|
||||
if limit <= 0 {
|
||||
if limit < 1 {
|
||||
limit = 50
|
||||
} else if limit > 50000 {
|
||||
limit = 50000
|
||||
}
|
||||
if !groupVersions {
|
||||
localItems, err := svc.Media.SearchMediaVisible(ctx, q, limit, visibility)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
remoteItems := fetchRemote(limit)
|
||||
all := append(localItems, remoteItems...)
|
||||
if len(all) > limit {
|
||||
all = all[:limit]
|
||||
}
|
||||
if all == nil {
|
||||
all = []model.Media{}
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"items": all})
|
||||
return
|
||||
}
|
||||
|
||||
localItems, err := svc.Media.SearchMediaVisible(ctx, q, 50000, visibility)
|
||||
paginated := c.Query("page") != "" || c.Query("page_size") != ""
|
||||
sortBy := normalizeMediaSearchSort(c.Query("sort"))
|
||||
// Sorting and grouping must operate on the same candidate set before
|
||||
// either a page size or the caller's output limit is applied.
|
||||
items, err := svc.Media.SearchMediaVisibleFiltered(ctx, query, 50000, visibility, filter)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
remoteItems := fetchRemote(limit)
|
||||
all := append(localItems, remoteItems...)
|
||||
grouped := service.GroupMediaVersions(all)
|
||||
items := service.FirstMediaItems(grouped, limit)
|
||||
if items == nil {
|
||||
items = []service.MediaItem{}
|
||||
if svc.EmbyRemote != nil && query != "" && (filter.LibraryID == "" || service.IsEmbyRemoteID(filter.LibraryID)) {
|
||||
remote, _ := svc.EmbyRemote.RemoteSearchMedia(ctx, query, 50000, visibility)
|
||||
for _, item := range remote {
|
||||
if remoteSearchMatchesFilter(item, filter) {
|
||||
items = append(items, item)
|
||||
}
|
||||
}
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"items": items})
|
||||
if c.DefaultQuery("group_series", "0") != "0" {
|
||||
items = service.GroupMediaSeriesItems(items)
|
||||
sortSearchMedia(items, sortBy)
|
||||
writeMediaSearchResults(c, items, page, size, limit, paginated)
|
||||
return
|
||||
}
|
||||
if c.DefaultQuery("group_versions", "1") != "0" {
|
||||
grouped := service.GroupMediaVersions(items)
|
||||
if sortBy != "" {
|
||||
sort.SliceStable(grouped, func(i, j int) bool {
|
||||
return mediaSearchSortLess(grouped[i].Media, grouped[j].Media, sortBy)
|
||||
})
|
||||
}
|
||||
writeMediaSearchResults(c, grouped, page, size, limit, paginated)
|
||||
return
|
||||
}
|
||||
sortSearchMedia(items, sortBy)
|
||||
writeMediaSearchResults(c, items, page, size, limit, paginated)
|
||||
}
|
||||
}
|
||||
|
||||
func writeMediaSearchResults[T any](c *gin.Context, items []T, page, size, limit int, paginated bool) {
|
||||
if items == nil {
|
||||
items = []T{}
|
||||
}
|
||||
if paginated {
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"items": paginateSlice(items, page, size),
|
||||
"total": len(items), "page": page, "page_size": size,
|
||||
})
|
||||
return
|
||||
}
|
||||
if len(items) > limit {
|
||||
items = items[:limit]
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"items": items})
|
||||
}
|
||||
|
||||
func remoteSearchMatchesFilter(item model.Media, filter service.MediaSearchFilter) bool {
|
||||
if filter.LibraryID != "" && item.LibraryID != filter.LibraryID && item.DisplayLibraryID != filter.LibraryID {
|
||||
return false
|
||||
}
|
||||
if filter.YearMin > 0 && item.Year < filter.YearMin || filter.YearMax > 0 && item.Year > filter.YearMax {
|
||||
return false
|
||||
}
|
||||
if len(filter.Genres) == 0 {
|
||||
return true
|
||||
}
|
||||
normalize := func(value string) string {
|
||||
return strings.ToLower(strings.ReplaceAll(strings.TrimSpace(value), " ", ""))
|
||||
}
|
||||
for _, genre := range strings.Split(strings.ReplaceAll(item.Genres, ",", ","), ",") {
|
||||
for _, wanted := range filter.Genres {
|
||||
if normalize(genre) == normalize(wanted) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func normalizeMediaSearchSort(value string) string {
|
||||
value = strings.TrimSpace(value)
|
||||
switch value {
|
||||
case "title_asc", "year_desc", "rating_desc":
|
||||
return value
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func sortSearchMedia(items []model.Media, sortBy string) {
|
||||
if sortBy == "" {
|
||||
return
|
||||
}
|
||||
sort.SliceStable(items, func(i, j int) bool {
|
||||
return mediaSearchSortLess(items[i], items[j], sortBy)
|
||||
})
|
||||
}
|
||||
|
||||
func mediaSearchSortLess(a, b model.Media, sortBy string) bool {
|
||||
switch sortBy {
|
||||
case "title_asc":
|
||||
left, right := strings.ToLower(strings.TrimSpace(a.Title)), strings.ToLower(strings.TrimSpace(b.Title))
|
||||
if left != right {
|
||||
return left < right
|
||||
}
|
||||
case "year_desc":
|
||||
if a.Year != b.Year {
|
||||
return a.Year > b.Year
|
||||
}
|
||||
case "rating_desc":
|
||||
if a.Rating != b.Rating {
|
||||
return a.Rating > b.Rating
|
||||
}
|
||||
}
|
||||
if !a.CreatedAt.Equal(b.CreatedAt) {
|
||||
return a.CreatedAt.After(b.CreatedAt)
|
||||
}
|
||||
return a.ID < b.ID
|
||||
}
|
||||
|
||||
func boundedSearchYear(value string) int {
|
||||
year, err := strconv.Atoi(strings.TrimSpace(value))
|
||||
if err != nil || year < 1870 || year > 2200 {
|
||||
return 0
|
||||
}
|
||||
return year
|
||||
}
|
||||
|
||||
func streamHandler(svc *service.Container) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
ctx := c.Request.Context()
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/truewhile/MeBox/internal/service"
|
||||
)
|
||||
|
||||
func mediaQualityHandler(svc *service.Container) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
|
||||
pageSize, _ := strconv.Atoi(c.DefaultQuery("page_size", "50"))
|
||||
if page < 1 {
|
||||
page = 1
|
||||
} else if page > 1000000 {
|
||||
page = 1000000
|
||||
}
|
||||
if pageSize < 1 {
|
||||
pageSize = 50
|
||||
} else if pageSize > 100 {
|
||||
pageSize = 100
|
||||
}
|
||||
issue := c.DefaultQuery("issue", "unmatched")
|
||||
if issue != "unmatched" && issue != "poster" && issue != "missing" {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "未知检查分类"})
|
||||
return
|
||||
}
|
||||
items, total, err := svc.Media.QualityIssues(c.Request.Context(), issue, page, pageSize)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"items": items, "total": total, "page": page, "page_size": pageSize})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/glebarez/sqlite"
|
||||
"go.uber.org/zap"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/config"
|
||||
"github.com/truewhile/MeBox/internal/middleware"
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
"github.com/truewhile/MeBox/internal/repository"
|
||||
"github.com/truewhile/MeBox/internal/service"
|
||||
)
|
||||
|
||||
func TestSearchMediaSortLimitFiltersAndDeepPagination(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sqlDB, _ := db.DB()
|
||||
defer sqlDB.Close()
|
||||
if err := db.AutoMigrate(&model.Library{}, &model.Media{}, &model.User{}, &model.Setting{}, &model.PlayProfile{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
repos := repository.New(db)
|
||||
lib := model.Library{Name: "Review", Path: "/review", Type: "movie", Enabled: true}
|
||||
if err := repos.Library.Create(t.Context(), &lib); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rows := make([]model.Media, 2001)
|
||||
for i := range rows {
|
||||
rows[i] = model.Media{
|
||||
Base: model.Base{ID: fmt.Sprintf("review-%04d", i)}, LibraryID: lib.ID,
|
||||
Title: fmt.Sprintf("Review %04d", i), Path: fmt.Sprintf("/review/%04d.mp4", i), Year: 2020, Genres: "Comedy",
|
||||
}
|
||||
}
|
||||
rows[2000].Year = 2025
|
||||
rows[2000].Genres = "Drama"
|
||||
if err := db.CreateInBatches(rows, 100).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
svc := &service.Container{Repo: repos, Media: service.NewMediaService(&config.Config{}, zap.NewNop(), repos)}
|
||||
cases := []struct {
|
||||
name, query, expectedID string
|
||||
count, total, page, size int
|
||||
}{
|
||||
{name: "sorted output limit", query: "sort=year_desc&limit=1", expectedID: "review-2000", count: 1},
|
||||
{name: "page past old 2000 cap", query: "sort=title_asc&page=2001&page_size=1", expectedID: "review-2000", count: 1, total: 2001, page: 2001, size: 1},
|
||||
{name: "genre filtering", query: "genre=Drama&library_id=" + lib.ID, expectedID: "review-2000", count: 1},
|
||||
{name: "normalized page", query: "page=0&page_size=0", count: 50, total: 2001, page: 1, size: 50},
|
||||
{name: "bounded page size", query: "page=1&page_size=99999", count: 2000, total: 2001, page: 1, size: 2000},
|
||||
{name: "huge page does not overflow", query: "page=" + strconv.Itoa(int(^uint(0)>>1)) + "&page_size=2000", count: 0, total: 2001, page: int(^uint(0) >> 1), size: 2000},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
w := httptest.NewRecorder()
|
||||
ctx, _ := gin.CreateTestContext(w)
|
||||
ctx.Set(middleware.CtxUserID, "admin")
|
||||
ctx.Set(middleware.CtxUserRole, "admin")
|
||||
ctx.Request = httptest.NewRequest(http.MethodGet, "/api/media?q=Review&group_versions=0&"+tc.query, nil)
|
||||
searchMediaHandler(svc)(ctx)
|
||||
var response struct {
|
||||
Items []model.Media `json:"items"`
|
||||
Total int `json:"total"`
|
||||
Page int `json:"page"`
|
||||
Size int `json:"page_size"`
|
||||
}
|
||||
if w.Code != http.StatusOK || json.Unmarshal(w.Body.Bytes(), &response) != nil {
|
||||
t.Fatalf("search failed: status=%d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
if len(response.Items) != tc.count || response.Total != tc.total || response.Page != tc.page || response.Size != tc.size {
|
||||
t.Fatalf("response count=%d total=%d page=%d size=%d, want %#v", len(response.Items), response.Total, response.Page, response.Size, tc)
|
||||
}
|
||||
if tc.expectedID != "" && response.Items[0].ID != tc.expectedID {
|
||||
t.Fatalf("first ID=%q want %q", response.Items[0].ID, tc.expectedID)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"testing"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
@@ -27,7 +28,8 @@ func TestSearchMediaSeriesKeepsRemoteLibrariesSeparate(t *testing.T) {
|
||||
"TotalRecordCount": 1,
|
||||
"Items": []map[string]any{{
|
||||
"Id": "same-series", "Name": "龙珠改", "Type": "Series",
|
||||
"ProviderIds": map[string]string{"Tmdb": "61709"},
|
||||
"ProviderIds": map[string]string{"Tmdb": "61709"},
|
||||
"ProductionYear": 2024, "Genres": []string{"Science Fiction"},
|
||||
}},
|
||||
})
|
||||
}))
|
||||
@@ -108,4 +110,9 @@ func TestSearchMediaSeriesKeepsRemoteLibrariesSeparate(t *testing.T) {
|
||||
}
|
||||
seen[item.DisplayLibraryID] = true
|
||||
}
|
||||
remoteLibraryID := service.EncodeEmbyRemoteID("mount-a", "view-a")
|
||||
items, total := request("page=1&page_size=8&library_id=" + url.QueryEscape(remoteLibraryID) + "&year_min=2020&genre=Science+Fiction")
|
||||
if total != 1 || len(items) != 1 || items[0].DisplayLibraryID != remoteLibraryID {
|
||||
t.Fatalf("remote library/year/genre filters lost results: total=%d items=%#v", total, items)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -149,10 +149,10 @@ func readerDeleteSourceHandler(svc *service.Container) gin.HandlerFunc {
|
||||
}
|
||||
|
||||
func readerDebugSourceHandler(svc *service.Container) gin.HandlerFunc {
|
||||
var body struct {
|
||||
Key string `json:"key" binding:"required"`
|
||||
}
|
||||
return func(c *gin.Context) {
|
||||
var body struct {
|
||||
Key string `json:"key" binding:"required"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&body); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
|
||||
@@ -13,6 +13,8 @@ func registerAdminRoutes(api *gin.RouterGroup, cfg *config.Config, svc *service.
|
||||
admin := api.Group("/admin")
|
||||
admin.Use(middleware.AuthRequired(cfg.Secrets.JWTSecret), middleware.AdminRequired())
|
||||
registerAdminUserRoutes(admin, svc)
|
||||
admin.GET("/tasks", backgroundTasksHandler(svc))
|
||||
admin.GET("/quality", mediaQualityHandler(svc))
|
||||
registerAdminPermissionRoutes(admin, svc)
|
||||
registerAdminSystemRoutes(admin, svc)
|
||||
registerAdminBackupRoutes(admin, svc)
|
||||
@@ -143,6 +145,7 @@ func registerAdminSystemRoutes(admin *gin.RouterGroup, svc *service.Container) {
|
||||
func registerAdminBackupRoutes(admin *gin.RouterGroup, svc *service.Container) {
|
||||
admin.GET("/backups", listBackupsHandler(svc))
|
||||
admin.POST("/backups", createBackupHandler(svc))
|
||||
admin.GET("/backups/download", downloadBackupHandler(svc))
|
||||
admin.DELETE("/backups", deleteBackupHandler(svc))
|
||||
admin.POST("/backups/restore", restoreBackupHandler(svc))
|
||||
}
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
package model
|
||||
|
||||
import "time"
|
||||
|
||||
// BackgroundTaskRecord persists the operator-facing lifecycle of scan,
|
||||
// organize, and other long-running application tasks.
|
||||
type BackgroundTaskRecord struct {
|
||||
ID string `gorm:"primaryKey;size:36" json:"id"`
|
||||
Kind string `gorm:"index;size:32;not null" json:"kind"`
|
||||
Name string `gorm:"size:255;not null" json:"name"`
|
||||
Status string `gorm:"index;size:24;not null" json:"status"`
|
||||
Stage string `gorm:"size:64" json:"stage"`
|
||||
SourcePath string `gorm:"size:2048" json:"source_path"`
|
||||
DestPath string `gorm:"size:2048" json:"dest_path"`
|
||||
Message string `gorm:"type:text" json:"message"`
|
||||
Error string `gorm:"type:text" json:"error"`
|
||||
Details string `gorm:"type:text" json:"details"`
|
||||
Metrics string `gorm:"type:text" json:"metrics"`
|
||||
Items string `gorm:"type:text" json:"items"`
|
||||
StartedAt time.Time `gorm:"index" json:"started_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
FinishedAt *time.Time `json:"finished_at,omitempty"`
|
||||
}
|
||||
@@ -1,5 +1,7 @@
|
||||
package model
|
||||
|
||||
import "time"
|
||||
|
||||
// Library 表示一个逻辑媒体库。Path 保留为兼容字段,指向第一条 LibraryRoot。
|
||||
type Library struct {
|
||||
Base
|
||||
@@ -26,12 +28,15 @@ type LibraryRoot struct {
|
||||
// Media 是单个可播放项。剧集链接到 SeriesID;电影 SeriesID == ""。
|
||||
type Media struct {
|
||||
Base
|
||||
LibraryID string `gorm:"index;size:36;index:idx_media_library_release,priority:1" json:"library_id"`
|
||||
LibraryRootID string `gorm:"index;size:36" json:"library_root_id,omitempty"`
|
||||
SeriesID string `gorm:"index;size:128" json:"series_id,omitempty"`
|
||||
Title string `gorm:"size:255;not null" json:"title"`
|
||||
OriginalName string `gorm:"size:255" json:"original_name,omitempty"`
|
||||
EpisodeTitle string `gorm:"size:255" json:"episode_title,omitempty"`
|
||||
AvailabilityState string `gorm:"index;size:16;not null;default:available" json:"availability_state"`
|
||||
MissingSince *time.Time `json:"missing_since,omitempty"`
|
||||
LastSeenAt *time.Time `json:"last_seen_at,omitempty"`
|
||||
LibraryID string `gorm:"index;size:36;index:idx_media_library_release,priority:1" json:"library_id"`
|
||||
LibraryRootID string `gorm:"index;size:36" json:"library_root_id,omitempty"`
|
||||
SeriesID string `gorm:"index;size:128" json:"series_id,omitempty"`
|
||||
Title string `gorm:"size:255;not null" json:"title"`
|
||||
OriginalName string `gorm:"size:255" json:"original_name,omitempty"`
|
||||
EpisodeTitle string `gorm:"size:255" json:"episode_title,omitempty"`
|
||||
// Path 的唯一性由 database 包按方言显式建索引,这里刻意不用 uniqueIndex:
|
||||
// PostgreSQL 的 btree 单条索引项上限 2704 字节,而 varchar(1024) 在中文路径
|
||||
// 下可达 3072 字节,直接对整列建唯一索引会让长路径的扫描插入直接失败
|
||||
|
||||
@@ -61,6 +61,7 @@ func AllModels() []interface{} {
|
||||
&StrmUploadTask{},
|
||||
&StrmDirCache{},
|
||||
&ScrapeTask{},
|
||||
&BackgroundTaskRecord{},
|
||||
&EmbyMount{},
|
||||
&ReaderBookSource{},
|
||||
&ReaderSourceState{},
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
@@ -163,6 +164,12 @@ func (r *MediaRepository) upsertWithDB(ctx context.Context, db *gorm.DB, m *mode
|
||||
existing.Path = m.Path
|
||||
existing.DeletedAt = gorm.DeletedAt{}
|
||||
}
|
||||
if _, ok := updates["availability_state"]; ok {
|
||||
existing.AvailabilityState = "available"
|
||||
existing.MissingSince = nil
|
||||
now := updates["last_seen_at"].(time.Time)
|
||||
existing.LastSeenAt = &now
|
||||
}
|
||||
*m = existing
|
||||
return existing.ID, nil
|
||||
}
|
||||
@@ -216,6 +223,10 @@ func (r *MediaRepository) findOrCreateMediaByPath(ctx context.Context, db *gorm.
|
||||
if m.ScrapeStatus == "" {
|
||||
m.ScrapeStatus = "pending"
|
||||
}
|
||||
now := time.Now()
|
||||
m.AvailabilityState = "available"
|
||||
m.MissingSince = nil
|
||||
m.LastSeenAt = &now
|
||||
// PostgreSQL 唯一键异常会中止整个事务,之后重查也会失败。
|
||||
// DO NOTHING 保持事务可用,并兼容 path 的表达式唯一索引。
|
||||
result := db.WithContext(ctx).Clauses(clause.OnConflict{DoNothing: true}).Create(m)
|
||||
@@ -265,6 +276,11 @@ func (r *MediaRepository) findMediaByAlias(ctx context.Context, db *gorm.DB, cur
|
||||
|
||||
func mediaUpsertUpdates(existing, incoming model.Media, reclaimLibrary bool) map[string]any {
|
||||
updates := map[string]any{}
|
||||
if existing.AvailabilityState != "available" || existing.MissingSince != nil {
|
||||
updates["availability_state"] = "available"
|
||||
updates["missing_since"] = nil
|
||||
updates["last_seen_at"] = time.Now()
|
||||
}
|
||||
addMediaFileScanUpdates(updates, existing, incoming)
|
||||
addMediaTitleUpdates(updates, existing, incoming)
|
||||
addMediaExternalIDUpdates(updates, existing, incoming)
|
||||
|
||||
+81
-37
@@ -5,7 +5,7 @@
|
||||
//
|
||||
// 两种方言的备份形态不同:
|
||||
// - SQLite:用 "VACUUM INTO"(3.27.0 起可用)产出一个一致的库文件,
|
||||
// 扩展名 .db;恢复即「替换库文件 + 重启」。
|
||||
// 扩展名 .db;恢复先暂存,在下次启动打开数据库前替换文件。
|
||||
// - PostgreSQL:没有单文件语义,用逻辑备份(见 backup_postgres.go),
|
||||
// 扩展名 .json.gz;恢复是在单个事务内重建全部表。
|
||||
package service
|
||||
@@ -24,6 +24,7 @@ import (
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/config"
|
||||
"github.com/truewhile/MeBox/internal/database"
|
||||
)
|
||||
|
||||
// BackupService manages database backups.
|
||||
@@ -44,6 +45,7 @@ type BackupInfo struct {
|
||||
Path string `json:"path"`
|
||||
Size int64 `json:"size"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
Automatic bool `json:"automatic"`
|
||||
}
|
||||
|
||||
func (b *BackupService) backupDir() string {
|
||||
@@ -69,6 +71,23 @@ func (b *BackupService) backupFilePath(filename string) (string, error) {
|
||||
return path, nil
|
||||
}
|
||||
|
||||
// ResolvePath returns the absolute path of a listed backup after applying the
|
||||
// same filename and directory checks used by restore and delete operations.
|
||||
func (b *BackupService) ResolvePath(filename string) (string, error) {
|
||||
path, err := b.backupFilePath(filename)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
info, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
return "", errors.New("backup is not a regular file")
|
||||
}
|
||||
return path, nil
|
||||
}
|
||||
|
||||
// isPostgres reports whether the service is backed by PostgreSQL.
|
||||
//
|
||||
// PostgreSQL 不支持 SQLite 的 "VACUUM INTO",按文件替换恢复也会写错目标,
|
||||
@@ -80,11 +99,23 @@ func (b *BackupService) isPostgres() bool {
|
||||
// Create produces a new hot backup. SQLite uses "VACUUM INTO"; PostgreSQL uses
|
||||
// a logical dump.
|
||||
func (b *BackupService) Create(ctx context.Context) (*BackupInfo, error) {
|
||||
return b.create(ctx, false)
|
||||
}
|
||||
|
||||
func (b *BackupService) CreateAutomatic(ctx context.Context) (*BackupInfo, error) {
|
||||
return b.create(ctx, true)
|
||||
}
|
||||
|
||||
func (b *BackupService) create(ctx context.Context, automatic bool) (*BackupInfo, error) {
|
||||
dir := b.backupDir()
|
||||
if err := os.MkdirAll(dir, 0o750); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ts := time.Now().Format("20060102_150405")
|
||||
now := time.Now()
|
||||
ts := fmt.Sprintf("%s_%09d", now.Format("20060102_150405"), now.Nanosecond())
|
||||
if automatic {
|
||||
ts = "auto_" + ts
|
||||
}
|
||||
|
||||
var (
|
||||
dst string
|
||||
@@ -108,15 +139,24 @@ func (b *BackupService) Create(ctx context.Context) (*BackupInfo, error) {
|
||||
Path: dst,
|
||||
Size: stat.Size(),
|
||||
CreatedAt: stat.ModTime(),
|
||||
Automatic: automatic,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (b *BackupService) createSQLiteBackup(ctx context.Context, stamp string) (string, error) {
|
||||
dst := filepath.Join(b.backupDir(), fmt.Sprintf("mebox_%s.db", stamp))
|
||||
// VACUUM INTO creates a clean, non-WAL copy atomically.
|
||||
if err := b.db.WithContext(ctx).Exec("VACUUM INTO ?", dst).Error; err != nil {
|
||||
tmp := dst + ".tmp"
|
||||
defer os.Remove(tmp)
|
||||
// Publish only completed backups, with credentials protected by file mode.
|
||||
if err := b.db.WithContext(ctx).Exec("VACUUM INTO ?", tmp).Error; err != nil {
|
||||
return "", fmt.Errorf("VACUUM INTO failed: %w", err)
|
||||
}
|
||||
if err := os.Chmod(tmp, 0o600); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := os.Rename(tmp, dst); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return dst, nil
|
||||
}
|
||||
|
||||
@@ -132,7 +172,7 @@ func (b *BackupService) List() ([]BackupInfo, error) {
|
||||
}
|
||||
out := make([]BackupInfo, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
if e.IsDir() || !isBackupFilename(e.Name()) {
|
||||
if !e.Type().IsRegular() || !isBackupFilename(e.Name()) {
|
||||
continue
|
||||
}
|
||||
info, err := e.Info()
|
||||
@@ -144,6 +184,7 @@ func (b *BackupService) List() ([]BackupInfo, error) {
|
||||
Path: filepath.Join(dir, e.Name()),
|
||||
Size: info.Size(),
|
||||
CreatedAt: info.ModTime(),
|
||||
Automatic: strings.HasPrefix(e.Name(), "mebox_auto_"),
|
||||
})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
@@ -152,6 +193,36 @@ func (b *BackupService) List() ([]BackupInfo, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Prune keeps maxCount automatic backups for the current database dialect.
|
||||
// Manual backups and backups from another dialect are never pruned.
|
||||
// A value below one disables pruning so an invalid setting cannot erase all
|
||||
// recoverable copies.
|
||||
func (b *BackupService) Prune(maxCount int) error {
|
||||
if maxCount < 1 {
|
||||
return nil
|
||||
}
|
||||
items, err := b.List()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
eligible := items[:0]
|
||||
for _, item := range items {
|
||||
if item.Automatic && strings.HasSuffix(item.Filename, postgresBackupSuffix) == b.isPostgres() {
|
||||
eligible = append(eligible, item)
|
||||
}
|
||||
}
|
||||
items = eligible
|
||||
if maxCount >= len(items) {
|
||||
return nil
|
||||
}
|
||||
for _, item := range items[maxCount:] {
|
||||
if err := b.Delete(item.Filename); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete removes a single backup file.
|
||||
func (b *BackupService) Delete(filename string) error {
|
||||
path, err := b.backupFilePath(filename)
|
||||
@@ -161,12 +232,10 @@ func (b *BackupService) Delete(filename string) error {
|
||||
return os.Remove(path)
|
||||
}
|
||||
|
||||
// Restore replaces the live database with the given backup.
|
||||
// WARNING: this is destructive. SQLite restores by swapping the database file
|
||||
// (the caller should shut down the server afterwards); PostgreSQL restores by
|
||||
// rebuilding every table inside a single transaction.
|
||||
// Restore stages SQLite backups for the next startup. PostgreSQL restores
|
||||
// rebuild every table inside a single transaction.
|
||||
func (b *BackupService) Restore(ctx context.Context, filename string) error {
|
||||
src, err := b.backupFilePath(filename)
|
||||
src, err := b.ResolvePath(filename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -182,35 +251,10 @@ func (b *BackupService) Restore(ctx context.Context, filename string) error {
|
||||
if !strings.HasSuffix(filename, ".db") {
|
||||
return errors.New("该备份是 PostgreSQL 逻辑备份,当前运行在 SQLite 上,无法恢复")
|
||||
}
|
||||
return b.restoreSQLiteBackup(filename, src)
|
||||
}
|
||||
|
||||
func (b *BackupService) restoreSQLiteBackup(filename, src string) error {
|
||||
dbPath, err := filepath.Abs(b.cfg.Database.DBPath)
|
||||
if err != nil || dbPath == "" {
|
||||
return errors.New("invalid database path")
|
||||
}
|
||||
// Strategy: rename live → live.old, copy backup → live, delete old.
|
||||
old := dbPath + ".before_restore"
|
||||
if err := os.Rename(dbPath, old); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
if err := database.StageSQLiteRestore(ctx, b.cfg, src); err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := os.ReadFile(src) // #nosec G304 -- src is constrained by backupFilePath to the configured backups directory.
|
||||
if err != nil {
|
||||
// Revert
|
||||
_ = os.Rename(old, dbPath)
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(dbPath, data, 0o600); err != nil { // #nosec G703 -- database path is trusted process config and normalized before restore.
|
||||
_ = os.Rename(old, dbPath)
|
||||
return err
|
||||
}
|
||||
_ = os.Remove(old)
|
||||
// Also remove WAL/SHM so SQLite opens the fresh file cleanly.
|
||||
_ = os.Remove(dbPath + "-wal")
|
||||
_ = os.Remove(dbPath + "-shm")
|
||||
b.log.Warn("database restored from backup — restart the server",
|
||||
zap.String("backup", filename))
|
||||
b.log.Warn("SQLite backup staged; restart to apply", zap.String("backup", filename))
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ package service
|
||||
import (
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -87,7 +88,8 @@ func postgresBackupModelSchemas(db *gorm.DB) (map[string]*schema.Schema, map[str
|
||||
// createPostgresBackup 把全库导出成一个 gzip 压缩的 JSON Lines 文件。
|
||||
func (b *BackupService) createPostgresBackup(ctx context.Context, stamp string) (string, error) {
|
||||
dst := filepath.Join(b.backupDir(), "mebox_"+stamp+postgresBackupSuffix)
|
||||
file, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600)
|
||||
tmp := dst + ".tmp"
|
||||
file, err := os.OpenFile(tmp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -96,7 +98,7 @@ func (b *BackupService) createPostgresBackup(ctx context.Context, stamp string)
|
||||
_ = file.Close()
|
||||
if !complete {
|
||||
// 半截备份文件比没有备份更危险:留着会让人以为有可用的还原点。
|
||||
_ = os.Remove(dst)
|
||||
_ = os.Remove(tmp)
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -109,17 +111,30 @@ func (b *BackupService) createPostgresBackup(ctx context.Context, stamp string)
|
||||
}); err != nil {
|
||||
return "", fmt.Errorf("写入备份头失败: %w", err)
|
||||
}
|
||||
for _, m := range model.AllModels() {
|
||||
table, sch, err := parseModelSchema(b.db, m)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !b.db.Migrator().HasTable(table) {
|
||||
continue
|
||||
}
|
||||
if err := b.dumpPostgresTable(ctx, m, sch, table, enc); err != nil {
|
||||
return "", err
|
||||
var options []*sql.TxOptions
|
||||
if b.isPostgres() {
|
||||
options = append(options, &sql.TxOptions{Isolation: sql.LevelRepeatableRead, ReadOnly: true})
|
||||
}
|
||||
// Every table must come from the same snapshot, including while background
|
||||
// workers are inserting related rows during an automatic backup.
|
||||
if err := b.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
snapshot := *b
|
||||
snapshot.db = tx
|
||||
for _, m := range model.AllModels() {
|
||||
table, sch, err := parseModelSchema(tx, m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !tx.Migrator().HasTable(table) {
|
||||
continue
|
||||
}
|
||||
if err := snapshot.dumpPostgresTable(ctx, m, sch, table, enc); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}, options...); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := zw.Close(); err != nil {
|
||||
return "", fmt.Errorf("压缩备份失败: %w", err)
|
||||
@@ -127,6 +142,12 @@ func (b *BackupService) createPostgresBackup(ctx context.Context, stamp string)
|
||||
if err := file.Sync(); err != nil {
|
||||
return "", fmt.Errorf("写入备份失败: %w", err)
|
||||
}
|
||||
if err := file.Close(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := os.Rename(tmp, dst); err != nil {
|
||||
return "", err
|
||||
}
|
||||
complete = true
|
||||
return dst, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestBackupRetentionPreservesManualAndOtherDialectBackups(t *testing.T) {
|
||||
svc, _ := newBackupTestService(t)
|
||||
names := []string{"mebox_auto_new.db", "mebox_auto_old.db", "mebox_manual.db", "mebox_auto_postgres.json.gz"}
|
||||
for i, name := range names {
|
||||
path := filepath.Join(svc.backupDir(), name)
|
||||
if err := os.WriteFile(path, []byte("fixture"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
when := time.Now().Add(-time.Duration(i) * time.Hour)
|
||||
if err := os.Chtimes(path, when, when); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := svc.Prune(1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, name := range names {
|
||||
_, err := os.Stat(filepath.Join(svc.backupDir(), name))
|
||||
if name == "mebox_auto_old.db" {
|
||||
if !os.IsNotExist(err) {
|
||||
t.Fatalf("old automatic backup was retained: %v", err)
|
||||
}
|
||||
} else if err != nil {
|
||||
t.Fatalf("backup %s should have been preserved: %v", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackupCreateUsesUniquePrivateCompletedFiles(t *testing.T) {
|
||||
svc, _ := newBackupTestService(t)
|
||||
manual, err := svc.Create(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
automatic, err := svc.CreateAutomatic(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if manual.Filename == automatic.Filename || manual.Automatic || !automatic.Automatic {
|
||||
t.Fatalf("invalid backup identities: manual=%#v automatic=%#v", manual, automatic)
|
||||
}
|
||||
items, err := svc.List()
|
||||
if err != nil || len(items) != 2 {
|
||||
t.Fatalf("completed backup list: items=%#v err=%v", items, err)
|
||||
}
|
||||
for _, item := range items {
|
||||
info, err := os.Stat(item.Path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if runtime.GOOS != "windows" && info.Mode().Perm() != 0o600 {
|
||||
t.Fatalf("backup permissions are not private: %s err=%v", item.Path, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2,17 +2,71 @@ package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
"github.com/truewhile/MeBox/internal/repository"
|
||||
)
|
||||
|
||||
type MediaQualityItem struct {
|
||||
Issue string `json:"issue"`
|
||||
Media model.Media `json:"media"`
|
||||
}
|
||||
|
||||
func (s *MediaService) QualityIssues(ctx context.Context, issue string, page, pageSize int) ([]MediaQualityItem, int64, error) {
|
||||
if page < 1 {
|
||||
page = 1
|
||||
}
|
||||
if pageSize < 1 {
|
||||
pageSize = 50
|
||||
}
|
||||
if pageSize > 100 {
|
||||
pageSize = 100
|
||||
}
|
||||
q := s.repo.DB.WithContext(ctx).Model(&model.Media{})
|
||||
switch issue {
|
||||
case "missing":
|
||||
q = q.Where("availability_state = ?", "missing")
|
||||
case "unmatched":
|
||||
q = q.Where("scrape_status IN ? OR scrape_status = ''", []string{"pending", "no_match"})
|
||||
case "poster":
|
||||
q = q.Where("poster_url IS NULL OR TRIM(poster_url) = ''")
|
||||
default:
|
||||
return nil, 0, fmt.Errorf("unknown quality issue: %s", issue)
|
||||
}
|
||||
var total int64
|
||||
if err := q.Count(&total).Error; err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
var rows []model.Media
|
||||
if err := q.Order("updated_at DESC, id ASC").Offset((page - 1) * pageSize).Limit(pageSize).Find(&rows).Error; err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
s.attachLibraryMetadata(ctx, rows)
|
||||
items := make([]MediaQualityItem, 0, len(rows))
|
||||
for _, media := range rows {
|
||||
items = append(items, MediaQualityItem{Issue: issue, Media: media})
|
||||
}
|
||||
return items, total, nil
|
||||
}
|
||||
|
||||
type MediaSearchFilter struct {
|
||||
LibraryID string
|
||||
YearMin int
|
||||
YearMax int
|
||||
Genres []string
|
||||
}
|
||||
|
||||
// SearchMedia performs a simple LIKE search across titles.
|
||||
func (s *MediaService) SearchMedia(ctx context.Context, query string, limit int) ([]model.Media, error) {
|
||||
return s.SearchMediaVisible(ctx, query, limit, MediaVisibility{IncludeNSFW: true})
|
||||
}
|
||||
|
||||
func (s *MediaService) SearchMediaVisible(ctx context.Context, query string, limit int, visibility MediaVisibility) ([]model.Media, error) {
|
||||
return s.SearchMediaVisibleFiltered(ctx, query, limit, visibility, MediaSearchFilter{})
|
||||
}
|
||||
|
||||
func (s *MediaService) SearchMediaVisibleFiltered(ctx context.Context, query string, limit int, visibility MediaVisibility, filter MediaSearchFilter) ([]model.Media, error) {
|
||||
if limit <= 0 {
|
||||
limit = 50
|
||||
} else if limit > maxMediaSearchLimit {
|
||||
@@ -23,6 +77,10 @@ func (s *MediaService) SearchMediaVisible(ctx context.Context, query string, lim
|
||||
IncludeNSFW: visibility.IncludeNSFW,
|
||||
AllowedLibraryIDs: visibility.AllowedLibraryIDs,
|
||||
HiddenLibraryIDs: visibility.HiddenLibraryIDs,
|
||||
LibraryID: filter.LibraryID,
|
||||
YearMin: filter.YearMin,
|
||||
YearMax: filter.YearMax,
|
||||
Genres: filter.Genres,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -45,6 +103,10 @@ func (s *MediaService) SearchMediaVisibleGrouped(ctx context.Context, query stri
|
||||
}
|
||||
|
||||
func (s *MediaService) SearchMediaVisiblePage(ctx context.Context, query string, page, pageSize int, visibility MediaVisibility) ([]model.Media, int64, error) {
|
||||
return s.SearchMediaVisiblePageFiltered(ctx, query, page, pageSize, visibility, MediaSearchFilter{})
|
||||
}
|
||||
|
||||
func (s *MediaService) SearchMediaVisiblePageFiltered(ctx context.Context, query string, page, pageSize int, visibility MediaVisibility, filter MediaSearchFilter) ([]model.Media, int64, error) {
|
||||
if pageSize <= 0 {
|
||||
pageSize = 50
|
||||
}
|
||||
@@ -59,6 +121,10 @@ func (s *MediaService) SearchMediaVisiblePage(ctx context.Context, query string,
|
||||
IncludeNSFW: visibility.IncludeNSFW,
|
||||
AllowedLibraryIDs: visibility.AllowedLibraryIDs,
|
||||
HiddenLibraryIDs: visibility.HiddenLibraryIDs,
|
||||
LibraryID: filter.LibraryID,
|
||||
YearMin: filter.YearMin,
|
||||
YearMax: filter.YearMax,
|
||||
Genres: filter.Genres,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
|
||||
+29
-26
@@ -14,6 +14,7 @@ import (
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
@@ -144,32 +145,34 @@ type localMediaProbeTask struct {
|
||||
}
|
||||
|
||||
type existingLocalMedia struct {
|
||||
LibraryRootID string
|
||||
RelativePath string
|
||||
Title string
|
||||
OriginalName string
|
||||
EpisodeTitle string
|
||||
SizeBytes int64
|
||||
DurationSec int
|
||||
Width int
|
||||
Height int
|
||||
VideoCodec string
|
||||
AudioCodec string
|
||||
Container string
|
||||
STRMURL string
|
||||
FileID string
|
||||
PosterURL string
|
||||
BackdropURL string
|
||||
Overview string
|
||||
Year int
|
||||
ReleaseDate string
|
||||
Rating float32
|
||||
TMDbID int
|
||||
BangumiID int
|
||||
DoubanID string
|
||||
TheTVDBID string
|
||||
SeasonNum int
|
||||
EpisodeNum int
|
||||
AvailabilityState string
|
||||
MissingSince *time.Time
|
||||
LibraryRootID string
|
||||
RelativePath string
|
||||
Title string
|
||||
OriginalName string
|
||||
EpisodeTitle string
|
||||
SizeBytes int64
|
||||
DurationSec int
|
||||
Width int
|
||||
Height int
|
||||
VideoCodec string
|
||||
AudioCodec string
|
||||
Container string
|
||||
STRMURL string
|
||||
FileID string
|
||||
PosterURL string
|
||||
BackdropURL string
|
||||
Overview string
|
||||
Year int
|
||||
ReleaseDate string
|
||||
Rating float32
|
||||
TMDbID int
|
||||
BangumiID int
|
||||
DoubanID string
|
||||
TheTVDBID string
|
||||
SeasonNum int
|
||||
EpisodeNum int
|
||||
// EpisodeFraction 是集号的小数部分(S01E11.5 → 0.5);变化同样要触发刷新,
|
||||
// 否则已有行上的「半集」标记不会被写回。
|
||||
EpisodeFraction float64
|
||||
|
||||
@@ -33,7 +33,7 @@ func (s *ScannerService) existingLocalMediaSnapshotForRoot(ctx context.Context,
|
||||
query = query.Where("(library_root_id = ? OR library_root_id = '' OR library_root_id IS NULL)", rootID)
|
||||
}
|
||||
rows, err := query.
|
||||
Select("path", "library_root_id", "relative_path", "title", "original_name", "episode_title", "size_bytes", "duration_sec", "width", "height", "video_codec", "audio_codec", "container", "strm_url", "file_id", "poster_url", "backdrop_url", "overview", "year", "release_date", "rating", "tm_db_id", "bangumi_id", "douban_id", "thetvdb_id", "season_num", "episode_num", "episode_fraction", "genres", "countries", "languages", "nsfw", "scrape_status").
|
||||
Select("path", "library_root_id", "relative_path", "title", "original_name", "episode_title", "size_bytes", "duration_sec", "width", "height", "video_codec", "audio_codec", "container", "strm_url", "file_id", "poster_url", "backdrop_url", "overview", "year", "release_date", "rating", "tm_db_id", "bangumi_id", "douban_id", "thetvdb_id", "season_num", "episode_num", "episode_fraction", "genres", "countries", "languages", "nsfw", "scrape_status", "availability_state", "missing_since").
|
||||
Rows()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -80,6 +80,8 @@ func (s *ScannerService) existingLocalMediaSnapshotForRoot(ctx context.Context,
|
||||
&row.Languages,
|
||||
&row.NSFW,
|
||||
&row.ScrapeStatus,
|
||||
&row.AvailabilityState,
|
||||
&row.MissingSince,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -548,7 +548,7 @@ func TestScanLibraryMapsPersistedHostLibraryPath(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemovePathDeletesVanishedMedia(t *testing.T) {
|
||||
func TestRemovePathRetainsVanishedMedia(t *testing.T) {
|
||||
sc, repos := newScannerTestEnv(t)
|
||||
cache := NewRuntimeCacheService(&config.Config{}, zap.NewNop())
|
||||
sc.SetRuntimeCache(cache)
|
||||
@@ -589,8 +589,15 @@ func TestRemovePathDeletesVanishedMedia(t *testing.T) {
|
||||
if removed != 1 {
|
||||
t.Fatalf("removed = %d, want 1", removed)
|
||||
}
|
||||
if countMedia(t, repos) != 0 {
|
||||
t.Fatal("expected 0 media after removal")
|
||||
if countMedia(t, repos) != 1 {
|
||||
t.Fatal("missing media metadata must be retained")
|
||||
}
|
||||
var missing model.Media
|
||||
if err := repos.DB.Where("path = ?", file).First(&missing).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if missing.AvailabilityState != "missing" || missing.MissingSince == nil {
|
||||
t.Fatalf("missing state not recorded: %#v", missing)
|
||||
}
|
||||
if cache.GetJSON(t.Context(), "media:list:test", &cached) {
|
||||
t.Fatal("vanished media removal should invalidate media cache")
|
||||
|
||||
@@ -67,8 +67,16 @@ func TestScanLibraryScansMultipleRootsAndPrunesPerRoot(t *testing.T) {
|
||||
if err := db.Find(&rows).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(rows) != 1 || rows[0].Path != fileB {
|
||||
t.Fatalf("remaining rows = %#v, want offline root media preserved", rows)
|
||||
if len(rows) != 2 {
|
||||
t.Fatalf("remaining rows = %#v, want both media records preserved", rows)
|
||||
}
|
||||
for _, row := range rows {
|
||||
if row.Path == fileA && row.AvailabilityState != "missing" {
|
||||
t.Fatalf("vanished file was not marked missing: %#v", row)
|
||||
}
|
||||
if row.Path == fileB && row.AvailabilityState != "available" {
|
||||
t.Fatalf("offline root status was changed: %#v", row)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -122,7 +122,7 @@ func (s *ScannerService) localMediaScanState(in localMediaScanStateInput) (bool,
|
||||
}
|
||||
existing, exists := in.existingMedia[in.cleanPath]
|
||||
isNewMedia := !exists
|
||||
if exists && in.ext != ".strm" && existing.SizeBytes == in.size && !localMetadataNeedsRefresh(existing, in.localMeta) && !localDerivedMetadataNeedsRefresh(existing, in.incoming) {
|
||||
if exists && existing.AvailabilityState != "missing" && existing.MissingSince == nil && in.ext != ".strm" && existing.SizeBytes == in.size && !localMetadataNeedsRefresh(existing, in.localMeta) && !localDerivedMetadataNeedsRefresh(existing, in.incoming) {
|
||||
// Unchanged rows are skipped, but a row that never received ffprobe data
|
||||
// still needs a backfill probe so installing ffmpeg later repairs the library.
|
||||
s.queueProbeBackfillIfNeeded(in.path, in.ext, existing)
|
||||
|
||||
@@ -372,7 +372,14 @@ func TestScanLibraryPrunesMissingMedia(t *testing.T) {
|
||||
if err := db.Model(&model.Media{}).Where("library_id = ?", lib.ID).Count(&count).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if count != 1 {
|
||||
t.Fatalf("media count = %d, want 1", count)
|
||||
if count != 2 {
|
||||
t.Fatalf("media count = %d, want current and missing records preserved", count)
|
||||
}
|
||||
var missing model.Media
|
||||
if err := db.First(&missing, "id = ?", stale.ID).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if missing.AvailabilityState != "missing" {
|
||||
t.Fatalf("stale media state = %q, want missing", missing.AvailabilityState)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
)
|
||||
|
||||
func TestScanRestoresUnchangedMissingFileAndMetadata(t *testing.T) {
|
||||
scanner, repos := newScannerTestEnv(t)
|
||||
root := t.TempDir()
|
||||
library := model.Library{Name: "Movies", Path: root, Type: "movie", Enabled: true}
|
||||
if err := repos.Library.Create(t.Context(), &library); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
path := filepath.Join(root, "Example (2024).mkv")
|
||||
writeTestFile(t, path, "unchanged content")
|
||||
if _, err := scanner.ScanLibrary(t.Context(), library.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var original model.Media
|
||||
if err := repos.DB.Where("path = ?", path).First(&original).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := repos.DB.Model(&original).Updates(map[string]any{
|
||||
"title": "Manually matched title", "scrape_status": "matched", "tm_db_id": 42,
|
||||
}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Remove(path); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if changed, err := scanner.RemovePath(t.Context(), path); err != nil || changed != 1 {
|
||||
t.Fatalf("mark missing: changed=%d err=%v", changed, err)
|
||||
}
|
||||
if changed, err := scanner.RemovePath(t.Context(), path); err != nil || changed != 0 {
|
||||
t.Fatalf("repeated missing event: changed=%d err=%v", changed, err)
|
||||
}
|
||||
writeTestFile(t, path, "unchanged content")
|
||||
if _, err := scanner.ScanLibrary(t.Context(), library.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var restored model.Media
|
||||
if err := repos.DB.Where("path = ?", path).First(&restored).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if restored.ID != original.ID || restored.TMDbID != 42 || restored.Title != "Manually matched title" {
|
||||
t.Fatalf("restored file lost identity or metadata: %#v", restored)
|
||||
}
|
||||
if restored.AvailabilityState != "available" || restored.MissingSince != nil || restored.LastSeenAt == nil {
|
||||
t.Fatalf("unchanged restored file stayed missing: %#v", restored)
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -12,11 +13,15 @@ import (
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
)
|
||||
|
||||
// RemovePath 移除磁盘上已不存在的媒体记录。普通的删除仍做物理删除;
|
||||
// 可确认是 STRM 扩展名改名的路径则先保留软删除墓碑,供后续 ingest 继承元数据。
|
||||
// RemovePath 将确认不存在的媒体标记为失联;只有可确认的 STRM 扩展名改名
|
||||
// 才保留软删除墓碑,供后续 ingest 继承元数据。
|
||||
func (s *ScannerService) RemovePath(ctx context.Context, path string) (int64, error) {
|
||||
if _, err := os.Stat(path); err == nil {
|
||||
return 0, nil // still exists; nothing to remove
|
||||
} else if !errors.Is(err, os.ErrNotExist) {
|
||||
// Permission, I/O, and mount errors are not evidence that a file was
|
||||
// removed. Preserve records until the path can be inspected reliably.
|
||||
return 0, nil
|
||||
}
|
||||
// 目录整体消失(删除/改名离开):连同其子树下的媒体行一并移除。
|
||||
// 此前只删 path 精确匹配的行——目录本身通常没有 media 行,导致目录
|
||||
@@ -33,7 +38,7 @@ func (s *ScannerService) RemovePath(ctx context.Context, path string) (int64, er
|
||||
return 0, err
|
||||
}
|
||||
softIDs := make([]string, 0, len(rows))
|
||||
hardIDs := make([]string, 0, len(rows))
|
||||
missingIDs := make([]string, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
// LIKE 里的 % _ 是通配符(候选集只会偏大),用 Go 前缀精确过滤,
|
||||
// 避免对含 % / _ 的路径误删。
|
||||
@@ -43,12 +48,12 @@ func (s *ScannerService) RemovePath(ctx context.Context, path string) (int64, er
|
||||
// A vanished STRM with a live sibling (foo.strm <-> foo.mkv.strm)
|
||||
// is a rename. Keep a soft-deleted tombstone so the later ingest can
|
||||
// adopt its ID and scraped metadata even if fsnotify delivers the
|
||||
// remove event before the create event. A true deletion stays hard.
|
||||
// remove event before the create event.
|
||||
if row.Path == path && liveSTRMPathAlias(path) {
|
||||
softIDs = append(softIDs, row.ID)
|
||||
continue
|
||||
}
|
||||
hardIDs = append(hardIDs, row.ID)
|
||||
missingIDs = append(missingIDs, row.ID)
|
||||
}
|
||||
var removed int64
|
||||
if len(softIDs) > 0 {
|
||||
@@ -60,14 +65,12 @@ func (s *ScannerService) RemovePath(ctx context.Context, path string) (int64, er
|
||||
}
|
||||
removed += res.RowsAffected
|
||||
}
|
||||
if len(hardIDs) > 0 {
|
||||
res := s.repo.DB.WithContext(ctx).Unscoped().
|
||||
Where("id IN ?", hardIDs).
|
||||
Delete(&model.Media{})
|
||||
if res.Error != nil {
|
||||
return removed, res.Error
|
||||
if len(missingIDs) > 0 {
|
||||
count, err := s.markMediaMissingByIDs(ctx, missingIDs)
|
||||
if err != nil {
|
||||
return removed, err
|
||||
}
|
||||
removed += res.RowsAffected
|
||||
removed += count
|
||||
}
|
||||
if removed > 0 {
|
||||
s.invalidateMediaCache(ctx)
|
||||
@@ -82,7 +85,7 @@ const strmRenameTombstoneRetention = 7 * 24 * time.Hour
|
||||
|
||||
// purgeExpiredSTRMTombstones keeps the soft-delete grace period bounded.
|
||||
// It is deliberately restricted to deleted media rows under a STRM alias
|
||||
// workflow; normal deletions are still hard-deleted immediately.
|
||||
// workflow; ordinary missing files retain their metadata for recovery.
|
||||
func (s *ScannerService) purgeExpiredSTRMTombstones(ctx context.Context) {
|
||||
cutoff := time.Now().Add(-strmRenameTombstoneRetention)
|
||||
if err := s.repo.DB.WithContext(ctx).Unscoped().
|
||||
@@ -121,7 +124,7 @@ func (s *ScannerService) pruneMissingMedia(ctx context.Context, libraryID string
|
||||
}
|
||||
stale = append(stale, row.ID)
|
||||
}
|
||||
return s.deleteMediaByIDs(ctx, stale, true)
|
||||
return s.markMediaMissingByIDs(ctx, stale)
|
||||
}
|
||||
|
||||
func (s *ScannerService) pruneMissingMediaForRoot(ctx context.Context, libraryID, rootID, rootPath string, seen map[string]struct{}) (int64, error) {
|
||||
@@ -158,7 +161,33 @@ func (s *ScannerService) pruneMissingMediaForRoot(ctx context.Context, libraryID
|
||||
}
|
||||
stale = append(stale, row.ID)
|
||||
}
|
||||
return s.deleteMediaByIDs(ctx, stale, true)
|
||||
return s.markMediaMissingByIDs(ctx, stale)
|
||||
}
|
||||
|
||||
func (s *ScannerService) markMediaMissingByIDs(ctx context.Context, ids []string) (int64, error) {
|
||||
if len(ids) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
const batch = 500
|
||||
var changed int64
|
||||
now := time.Now()
|
||||
for i := 0; i < len(ids); i += batch {
|
||||
end := i + batch
|
||||
if end > len(ids) {
|
||||
end = len(ids)
|
||||
}
|
||||
result := s.repo.DB.WithContext(ctx).Model(&model.Media{}).
|
||||
Where("id IN ? AND (availability_state IS NULL OR availability_state <> ?)", ids[i:end], "missing").
|
||||
Updates(map[string]any{"availability_state": "missing", "missing_since": now})
|
||||
if result.Error != nil {
|
||||
return changed, result.Error
|
||||
}
|
||||
changed += result.RowsAffected
|
||||
}
|
||||
if changed > 0 {
|
||||
s.invalidateMediaCache(ctx)
|
||||
}
|
||||
return changed, nil
|
||||
}
|
||||
|
||||
func pathBelongsToRoot(pathValue, rootPath string) bool {
|
||||
|
||||
@@ -202,7 +202,7 @@ func (s *ScannerService) scanLocalLibraryFiles(ctx context.Context, lib *model.L
|
||||
func existingLocalMediaFileIDs(existingMedia map[string]existingLocalMedia) map[string]string {
|
||||
seenInodes := make(map[string]string)
|
||||
for path, existing := range existingMedia {
|
||||
if existing.FileID != "" {
|
||||
if existing.FileID != "" && existing.AvailabilityState != "missing" {
|
||||
seenInodes[existing.FileID] = path
|
||||
}
|
||||
}
|
||||
|
||||
@@ -175,7 +175,7 @@ func TestIngestPathAdoptsSoftDeletedSTRMRenameTombstone(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemovePathStillHardDeletesTrueDeletionWithoutAlias(t *testing.T) {
|
||||
func TestRemovePathRetainsTrueDeletionWithoutAlias(t *testing.T) {
|
||||
sc, repos := newScannerTestEnv(t)
|
||||
root := t.TempDir()
|
||||
lib := model.Library{Name: "Anime", Path: root, Type: "anime", Enabled: true}
|
||||
@@ -197,8 +197,15 @@ func TestRemovePathStillHardDeletesTrueDeletionWithoutAlias(t *testing.T) {
|
||||
if err := repos.DB.Unscoped().Model(&model.Media{}).Where("path = ?", path).Count(&count).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if count != 0 {
|
||||
t.Fatalf("true deletion left %d tombstone row(s)", count)
|
||||
if count != 1 {
|
||||
t.Fatalf("missing media count = %d, want metadata retained", count)
|
||||
}
|
||||
var media model.Media
|
||||
if err := repos.DB.Where("path = ?", path).First(&media).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if media.AvailabilityState != "missing" || media.MissingSince == nil {
|
||||
t.Fatalf("true deletion did not record missing state: %#v", media)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -50,6 +50,7 @@ type SchedulerService struct {
|
||||
readerFilesTTLHours int
|
||||
|
||||
segments *MediaSegmentService
|
||||
backup *BackupService
|
||||
|
||||
mu sync.Mutex
|
||||
stopCh chan struct{}
|
||||
@@ -65,6 +66,10 @@ func (s *SchedulerService) SetTaskTracker(tasks *TaskTrackerService) {
|
||||
s.tasks = tasks
|
||||
}
|
||||
|
||||
func (s *SchedulerService) SetBackupService(backup *BackupService) {
|
||||
s.backup = backup
|
||||
}
|
||||
|
||||
func (s *SchedulerService) SetOrganizePipeline(pipeline *OrganizePipelineService) {
|
||||
s.organizePipeline = pipeline
|
||||
}
|
||||
@@ -196,6 +201,13 @@ func (s *SchedulerService) Start(ctx context.Context) {
|
||||
run: s.jobTelegramExpiryWarning,
|
||||
})
|
||||
}
|
||||
if s.backup != nil {
|
||||
s.jobs = append(s.jobs, &scheduledJob{
|
||||
name: "database_backup",
|
||||
interval: 5 * time.Minute,
|
||||
run: s.jobDatabaseBackup,
|
||||
})
|
||||
}
|
||||
for _, j := range s.jobs {
|
||||
initialDelay := 15 * time.Second
|
||||
if j.name == "library_scan" || j.name == "organize_source" || j.name == "segment_prewarm" {
|
||||
|
||||
@@ -11,6 +11,53 @@ import (
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
const lastAutomaticBackupDateKey = "backup.last_automatic_date"
|
||||
|
||||
func (s *SchedulerService) jobDatabaseBackup(ctx context.Context) error {
|
||||
if s == nil || s.backup == nil || s.repo == nil || s.repo.Setting == nil {
|
||||
return nil
|
||||
}
|
||||
enabled, _ := s.repo.Setting.Get(ctx, "backup.auto_enabled")
|
||||
if !parseBoolSetting(enabled, false) {
|
||||
return nil
|
||||
}
|
||||
clock, _ := s.repo.Setting.Get(ctx, "backup.auto_time")
|
||||
if clock == "" {
|
||||
clock = "03:00"
|
||||
}
|
||||
parsed, err := time.Parse("15:04", clock)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
now := s.currentTime().In(time.Local)
|
||||
dueAt := time.Date(now.Year(), now.Month(), now.Day(), parsed.Hour(), parsed.Minute(), 0, 0, now.Location())
|
||||
if now.Before(dueAt) {
|
||||
return nil
|
||||
}
|
||||
date := now.Format("2006-01-02")
|
||||
last, _ := s.repo.Setting.Get(ctx, lastAutomaticBackupDateKey)
|
||||
if last == date {
|
||||
return nil
|
||||
}
|
||||
if _, err := s.backup.CreateAutomatic(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
// Record the successful backup before pruning. If retention cleanup fails,
|
||||
// the next scheduler tick must not create another copy for the same day.
|
||||
if err := s.repo.Setting.Set(ctx, lastAutomaticBackupDateKey, date); err != nil {
|
||||
return err
|
||||
}
|
||||
retentionValue, _ := s.repo.Setting.Get(ctx, "backup.retention")
|
||||
retention, err := strconv.Atoi(strings.TrimSpace(retentionValue))
|
||||
if err != nil || retention < 1 {
|
||||
retention = 7
|
||||
}
|
||||
if err := s.backup.Prune(retention); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// jobScanLibraries re-walks every enabled library.
|
||||
//
|
||||
// 默认关闭:文件变更由 WatcherService 增量入库,无需周期性全量重扫。
|
||||
|
||||
@@ -63,6 +63,9 @@ func (b *serviceContainerBuilder) startRealtimeServices() {
|
||||
b.c.WSHub = NewHub(b.log)
|
||||
helper.Go(b.log, "ws.hub", b.c.WSHub.Run)
|
||||
b.c.Tasks = NewTaskTrackerService(b.log, b.c.WSHub)
|
||||
if b.repos != nil {
|
||||
b.c.Tasks.SetPersistence(b.repos.DB)
|
||||
}
|
||||
b.c.SystemUpdate = NewSystemUpdateService(b.cfg, b.log, b.repos, b.c.Tasks, b.version)
|
||||
|
||||
b.c.SSEHub = NewSSEHub(b.log)
|
||||
@@ -206,6 +209,7 @@ func (b *serviceContainerBuilder) initAccessAndStorageServices() {
|
||||
b.c.Organizer, b.c.WSHub, b.cfg.Cache.CacheDir,
|
||||
)
|
||||
b.c.Scheduler.SetTaskTracker(b.c.Tasks)
|
||||
b.c.Scheduler.SetBackupService(b.c.Backup)
|
||||
b.c.Scheduler.SetOrganizePipeline(b.c.OrganizePipeline)
|
||||
b.c.Scheduler.SetSegments(b.c.Segments)
|
||||
b.c.Scheduler.SetImageCachePolicyProvider(func() ImageCachePolicy {
|
||||
|
||||
@@ -2,6 +2,7 @@ package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"html"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -9,12 +10,16 @@ import (
|
||||
|
||||
"github.com/google/uuid"
|
||||
"go.uber.org/zap"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
)
|
||||
|
||||
const (
|
||||
TaskStatusRunning = "running"
|
||||
TaskStatusCompleted = "completed"
|
||||
TaskStatusFailed = "failed"
|
||||
TaskStatusRunning = "running"
|
||||
TaskStatusCompleted = "completed"
|
||||
TaskStatusFailed = "failed"
|
||||
TaskStatusInterrupted = "interrupted"
|
||||
|
||||
TaskKindOrganize = "organize"
|
||||
TaskKindScan = "scan"
|
||||
@@ -70,6 +75,26 @@ type TaskTrackerService struct {
|
||||
recent []BackgroundTask
|
||||
maxRecent int
|
||||
now func() time.Time
|
||||
db *gorm.DB
|
||||
}
|
||||
|
||||
// SetPersistence enables durable task history and marks tasks left running by
|
||||
// a previous process as interrupted. Actual task execution is never resumed
|
||||
// automatically; this record is an operator-visible recovery point.
|
||||
func (t *TaskTrackerService) SetPersistence(db *gorm.DB) {
|
||||
if t == nil || db == nil {
|
||||
return
|
||||
}
|
||||
t.mu.Lock()
|
||||
t.db = db
|
||||
now := t.currentTime()
|
||||
t.mu.Unlock()
|
||||
err := db.Model(&model.BackgroundTaskRecord{}).
|
||||
Where("status = ?", TaskStatusRunning).
|
||||
Updates(map[string]any{"status": TaskStatusInterrupted, "error": "服务在任务完成前重启;请检查文件状态后重新执行", "updated_at": now, "finished_at": now}).Error
|
||||
if err != nil && t.log != nil {
|
||||
t.log.Warn("recover interrupted background tasks failed", zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
type TaskHandle struct {
|
||||
@@ -105,9 +130,11 @@ func (t *TaskTrackerService) Start(kind, name string, update TaskUpdate) *TaskHa
|
||||
StartedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
applyTaskUpdate(task, update)
|
||||
t.mu.Lock()
|
||||
t.active[task.ID] = task
|
||||
snapshot := cloneBackgroundTask(*task)
|
||||
t.persist(snapshot)
|
||||
t.mu.Unlock()
|
||||
t.publish(snapshot)
|
||||
return &TaskHandle{tracker: t, id: task.ID}
|
||||
@@ -132,18 +159,47 @@ func (t *TaskTrackerService) Snapshot() TaskSnapshot {
|
||||
return TaskSnapshot{}
|
||||
}
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
active := make([]BackgroundTask, 0, len(t.active))
|
||||
for _, task := range t.active {
|
||||
active = append(active, cloneBackgroundTask(*task))
|
||||
}
|
||||
recent := make([]BackgroundTask, 0, len(t.recent))
|
||||
db := t.db
|
||||
limit := t.maxRecent
|
||||
if limit < 1 {
|
||||
limit = 30
|
||||
}
|
||||
for _, task := range t.recent {
|
||||
recent = append(recent, cloneBackgroundTask(task))
|
||||
}
|
||||
t.mu.Unlock()
|
||||
if db != nil {
|
||||
var rows []model.BackgroundTaskRecord
|
||||
if err := db.Where("status <> ?", TaskStatusRunning).Order("started_at DESC").Limit(limit).Find(&rows).Error; err == nil {
|
||||
recent = make([]BackgroundTask, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
recent = append(recent, backgroundTaskFromRecord(row))
|
||||
}
|
||||
} else if t.log != nil {
|
||||
t.log.Warn("load background task history failed; using memory history", zap.Error(err))
|
||||
}
|
||||
}
|
||||
return TaskSnapshot{Active: active, Recent: recent}
|
||||
}
|
||||
|
||||
func backgroundTaskFromRecord(row model.BackgroundTaskRecord) BackgroundTask {
|
||||
task := BackgroundTask{
|
||||
ID: row.ID, Kind: row.Kind, Name: row.Name, Status: row.Status,
|
||||
Stage: row.Stage, SourcePath: row.SourcePath, DestPath: row.DestPath,
|
||||
Message: row.Message, Error: row.Error, StartedAt: row.StartedAt,
|
||||
UpdatedAt: row.UpdatedAt, FinishedAt: row.FinishedAt,
|
||||
}
|
||||
_ = json.Unmarshal([]byte(row.Details), &task.Details)
|
||||
_ = json.Unmarshal([]byte(row.Metrics), &task.Metrics)
|
||||
_ = json.Unmarshal([]byte(row.Items), &task.Items)
|
||||
return task
|
||||
}
|
||||
|
||||
func (t *TaskTrackerService) update(id string, update TaskUpdate) {
|
||||
now := t.currentTime()
|
||||
t.mu.Lock()
|
||||
@@ -155,6 +211,7 @@ func (t *TaskTrackerService) update(id string, update TaskUpdate) {
|
||||
applyTaskUpdate(task, update)
|
||||
task.UpdatedAt = now
|
||||
snapshot := cloneBackgroundTask(*task)
|
||||
t.persist(snapshot)
|
||||
t.mu.Unlock()
|
||||
t.publish(snapshot)
|
||||
}
|
||||
@@ -186,6 +243,7 @@ func (t *TaskTrackerService) finish(id string, err error, update TaskUpdate) {
|
||||
t.recent = t.recent[:t.maxRecent]
|
||||
}
|
||||
notifier := t.failureNotifier
|
||||
t.persist(snapshot)
|
||||
t.mu.Unlock()
|
||||
t.publish(snapshot)
|
||||
|
||||
@@ -195,6 +253,26 @@ func (t *TaskTrackerService) finish(id string, err error, update TaskUpdate) {
|
||||
}
|
||||
}
|
||||
|
||||
func (t *TaskTrackerService) persist(task BackgroundTask) {
|
||||
if t == nil || t.db == nil {
|
||||
return
|
||||
}
|
||||
encode := func(value any) string {
|
||||
data, _ := json.Marshal(value)
|
||||
return string(data)
|
||||
}
|
||||
row := model.BackgroundTaskRecord{
|
||||
ID: task.ID, Kind: task.Kind, Name: task.Name, Status: task.Status,
|
||||
Stage: task.Stage, SourcePath: task.SourcePath, DestPath: task.DestPath,
|
||||
Message: task.Message, Error: task.Error, Details: encode(task.Details),
|
||||
Metrics: encode(task.Metrics), Items: encode(task.Items), StartedAt: task.StartedAt,
|
||||
UpdatedAt: task.UpdatedAt, FinishedAt: task.FinishedAt,
|
||||
}
|
||||
if err := t.db.Save(&row).Error; err != nil && t.log != nil {
|
||||
t.log.Warn("persist background task failed", zap.String("task_id", task.ID), zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
// SetFailureNotifier 注入任务失败时的管理员通知回调。
|
||||
func (t *TaskTrackerService) SetFailureNotifier(fn func(ctx context.Context, text string)) {
|
||||
if t == nil {
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
)
|
||||
|
||||
func TestTaskTrackerPersistsHistoryAcrossRestart(t *testing.T) {
|
||||
db := newServiceTestDB(t)
|
||||
tracker := NewTaskTrackerService(zap.NewNop(), nil)
|
||||
tracker.SetPersistence(db)
|
||||
task := tracker.Start(TaskKindScan, "scan", TaskUpdate{Details: []string{"initial detail"}})
|
||||
task.Update(TaskUpdate{Metrics: map[string]int64{"visited": 42}})
|
||||
task.Finish(nil, TaskUpdate{Message: "done"})
|
||||
restarted := NewTaskTrackerService(zap.NewNop(), nil)
|
||||
restarted.SetPersistence(db)
|
||||
snapshot := restarted.Snapshot()
|
||||
if len(snapshot.Active) != 0 || len(snapshot.Recent) != 1 {
|
||||
t.Fatalf("unexpected history: %#v", snapshot)
|
||||
}
|
||||
recent := snapshot.Recent[0]
|
||||
if recent.Status != TaskStatusCompleted || recent.Metrics["visited"] != 42 || len(recent.Details) != 1 || recent.FinishedAt == nil {
|
||||
t.Fatalf("task history lost fields: %#v", recent)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTaskTrackerConcurrentUpdatesDoNotRegressFinishedTasks(t *testing.T) {
|
||||
db := newServiceTestDB(t)
|
||||
sqlDB, err := db.DB()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sqlDB.SetMaxOpenConns(1)
|
||||
tracker := NewTaskTrackerService(zap.NewNop(), nil)
|
||||
tracker.SetPersistence(db)
|
||||
var workers sync.WaitGroup
|
||||
for i := 0; i < 12; i++ {
|
||||
workers.Add(1)
|
||||
go func(index int) {
|
||||
defer workers.Done()
|
||||
task := tracker.Start(TaskKindScan, fmt.Sprintf("scan-%d", index), TaskUpdate{})
|
||||
task.Update(TaskUpdate{Metrics: map[string]int64{"visited": 1}})
|
||||
task.Finish(nil, TaskUpdate{})
|
||||
task.Update(TaskUpdate{Message: "late update"})
|
||||
_ = tracker.Snapshot()
|
||||
}(i)
|
||||
}
|
||||
workers.Wait()
|
||||
var completed int64
|
||||
if err := db.Model(&model.BackgroundTaskRecord{}).Where("status = ?", TaskStatusCompleted).Count(&completed).Error; err != nil || completed != 12 {
|
||||
t.Fatalf("finished task status regressed: completed=%d err=%v", completed, err)
|
||||
}
|
||||
if snapshot := tracker.Snapshot(); len(snapshot.Active) != 0 || len(snapshot.Recent) != 12 {
|
||||
t.Fatalf("inconsistent task snapshot: %#v", snapshot)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTaskTrackerMarksAbandonedTasksInterrupted(t *testing.T) {
|
||||
db := newServiceTestDB(t)
|
||||
tracker := NewTaskTrackerService(zap.NewNop(), nil)
|
||||
tracker.SetPersistence(db)
|
||||
tracker.Start(TaskKindOrganize, "unfinished", TaskUpdate{})
|
||||
restarted := NewTaskTrackerService(zap.NewNop(), nil)
|
||||
restarted.SetPersistence(db)
|
||||
snapshot := restarted.Snapshot()
|
||||
if len(snapshot.Recent) != 1 || snapshot.Recent[0].Status != TaskStatusInterrupted || snapshot.Recent[0].FinishedAt == nil {
|
||||
t.Fatalf("abandoned task not marked interrupted: %#v", snapshot)
|
||||
}
|
||||
var running int64
|
||||
if err := db.Model(&model.BackgroundTaskRecord{}).Where("status = ?", TaskStatusRunning).Count(&running).Error; err != nil || running != 0 {
|
||||
t.Fatalf("stale running tasks: %d err=%v", running, err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user