feat: 完善媒体维护、备份恢复与播放诊断

This commit is contained in:
jason jia
2026-10-20 14:01:26 +00:00
parent e34e81cce0
commit 1e8fe2396e
53 changed files with 2049 additions and 238 deletions
+189
View File
@@ -0,0 +1,189 @@
package database
import (
"context"
"database/sql"
"errors"
"fmt"
"io"
"net/url"
"os"
"path/filepath"
"strings"
"sync"
"go.uber.org/zap"
"github.com/truewhile/MeBox/internal/config"
)
var sqliteRestoreMu sync.Mutex
func sqliteRestorePath(cfg *config.Config) (string, error) {
if cfg == nil || strings.TrimSpace(cfg.Database.DBPath) == "" || cfg.Database.DBPath == ":memory:" || strings.HasPrefix(cfg.Database.DBPath, "file:") {
return "", errors.New("恢复需要配置有效的 SQLite 数据库文件路径")
}
return filepath.Abs(cfg.Database.DBPath)
}
func sqliteRestoreDSN(path, mode string) string {
u := url.URL{Scheme: "file", Path: filepath.ToSlash(path)}
q := url.Values{"mode": {mode}, "_pragma": {"busy_timeout(5000)"}}
u.RawQuery = q.Encode()
return u.String()
}
func validateSQLiteRestore(ctx context.Context, path string) error {
info, err := os.Lstat(path)
if err != nil {
return err
}
if !info.Mode().IsRegular() {
return errors.New("备份必须是常规文件")
}
db, err := sql.Open("sqlite", sqliteRestoreDSN(path, "ro"))
if err != nil {
return err
}
defer db.Close()
var result string
if err := db.QueryRowContext(ctx, "PRAGMA quick_check").Scan(&result); err != nil {
return fmt.Errorf("SQLite 备份校验失败: %w", err)
}
if result != "ok" {
return fmt.Errorf("SQLite 备份校验失败: %s", result)
}
var tables int
if err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name IN ('users', 'media', 'settings')").Scan(&tables); err != nil {
return err
}
if tables == 0 {
return errors.New("备份不包含 MeBox 数据表")
}
return nil
}
// StageSQLiteRestore leaves the live database and its WAL untouched. The
// validated copy is applied on the next startup, before any database is opened.
func StageSQLiteRestore(ctx context.Context, cfg *config.Config, source string) error {
sqliteRestoreMu.Lock()
defer sqliteRestoreMu.Unlock()
dbPath, err := sqliteRestorePath(cfg)
if err != nil {
return err
}
pending := dbPath + ".restore-pending"
if _, err := os.Lstat(pending); err == nil {
return errors.New("已有等待恢复的备份,请重启 MeBox 后再操作")
} else if !errors.Is(err, os.ErrNotExist) {
return err
}
if err := validateSQLiteRestore(ctx, source); err != nil {
return err
}
in, err := os.Open(source)
if err != nil {
return err
}
defer in.Close()
out, err := os.CreateTemp(filepath.Dir(dbPath), ".mebox-restore-*")
if err != nil {
return err
}
tmp := out.Name()
defer os.Remove(tmp)
if _, err := io.Copy(out, in); err != nil {
_ = out.Close()
return err
}
if err := out.Sync(); err != nil {
_ = out.Close()
return err
}
if err := out.Close(); err != nil {
return err
}
if err := validateSQLiteRestore(ctx, tmp); err != nil {
return err
}
return os.Rename(tmp, pending)
}
// ApplyPendingSQLiteRestore must run before database.Open and before workers or
// HTTP listeners start. The previous database is retained alongside the file.
func ApplyPendingSQLiteRestore(ctx context.Context, cfg *config.Config, log *zap.Logger) error {
if cfg == nil {
return nil
}
dialect := normalizeDatabaseType(cfg.Database.Type)
if dialect == "auto" {
dialect = effectiveAutoDatabaseType(cfg)
}
if dialect != "sqlite" {
return nil
}
sqliteRestoreMu.Lock()
defer sqliteRestoreMu.Unlock()
dbPath, err := sqliteRestorePath(cfg)
if err != nil {
return err
}
pending := dbPath + ".restore-pending"
if _, err := os.Lstat(pending); errors.Is(err, os.ErrNotExist) {
return nil
} else if err != nil {
return err
}
if err := validateSQLiteRestore(ctx, pending); err != nil {
return err
}
previous := ""
if _, err := os.Lstat(dbPath); err == nil {
// A clean checkpoint prevents an old WAL from being replayed into the
// replacement database. A busy checkpoint leaves all files untouched.
live, err := sql.Open("sqlite", sqliteRestoreDSN(dbPath, "rw"))
if err != nil {
return err
}
var busy, pages, copied int
err = live.QueryRowContext(ctx, "PRAGMA wal_checkpoint(TRUNCATE)").Scan(&busy, &pages, &copied)
closeErr := live.Close()
if err != nil {
return err
}
if closeErr != nil {
return closeErr
}
if busy != 0 {
return errors.New("SQLite 数据库仍被占用,恢复已保留等待下次启动")
}
old, err := os.CreateTemp(filepath.Dir(dbPath), filepath.Base(dbPath)+".before-restore-*")
if err != nil {
return err
}
previous = old.Name()
if err := old.Close(); err != nil {
return err
}
if err := os.Remove(previous); err != nil {
return err
}
if err := os.Rename(dbPath, previous); err != nil {
return err
}
} else if !errors.Is(err, os.ErrNotExist) {
return err
}
if err := os.Rename(pending, dbPath); err != nil {
if previous != "" {
if rollbackErr := os.Rename(previous, dbPath); rollbackErr != nil {
return fmt.Errorf("恢复失败: %w;回退失败: %v,原数据库保留于 %s", err, rollbackErr, previous)
}
}
return err
}
if log != nil {
log.Warn("SQLite backup restored before startup", zap.String("previous_database", previous))
}
return nil
}
+104
View File
@@ -0,0 +1,104 @@
package database
import (
"database/sql"
"os"
"path/filepath"
"testing"
"github.com/truewhile/MeBox/internal/config"
)
func TestSQLiteRestoreStagesWithoutChangingLiveDatabase(t *testing.T) {
root := t.TempDir()
cfg := &config.Config{}
cfg.Database.Type = "sqlite"
cfg.Database.DBPath = filepath.Join(root, "live.db")
live, err := sql.Open("sqlite", sqliteRestoreDSN(cfg.Database.DBPath, "rwc"))
if err != nil {
t.Fatal(err)
}
defer live.Close()
for _, statement := range []string{
"PRAGMA journal_mode=WAL", "CREATE TABLE settings (key TEXT PRIMARY KEY, value TEXT)",
"INSERT INTO settings VALUES ('state', 'backup')",
} {
if _, err := live.Exec(statement); err != nil {
t.Fatal(err)
}
}
source := filepath.Join(root, "backup.db")
if _, err := live.Exec("VACUUM INTO ?", source); err != nil {
t.Fatal(err)
}
if _, err := live.Exec("UPDATE settings SET value = 'live'"); err != nil {
t.Fatal(err)
}
if err := StageSQLiteRestore(t.Context(), cfg, source); err != nil {
t.Fatal(err)
}
var value string
if err := live.QueryRow("SELECT value FROM settings").Scan(&value); err != nil || value != "live" {
t.Fatalf("staging changed live database: value=%q err=%v", value, err)
}
if _, err := live.Exec("UPDATE settings SET value = 'after-stage'"); err != nil {
t.Fatalf("database could not write after staging: %v", err)
}
if err := StageSQLiteRestore(t.Context(), cfg, source); err == nil {
t.Fatal("second staging should not replace the pending restore")
}
if err := os.Remove(source); err != nil {
t.Fatal(err)
}
if err := live.Close(); err != nil {
t.Fatal(err)
}
if err := ApplyPendingSQLiteRestore(t.Context(), cfg, nil); err != nil {
t.Fatal(err)
}
restored, err := sql.Open("sqlite", sqliteRestoreDSN(cfg.Database.DBPath, "ro"))
if err != nil {
t.Fatal(err)
}
defer restored.Close()
if err := restored.QueryRow("SELECT value FROM settings").Scan(&value); err != nil || value != "backup" {
t.Fatalf("restore failed: value=%q err=%v", value, err)
}
previous, err := filepath.Glob(cfg.Database.DBPath + ".before-restore-*")
if err != nil || len(previous) != 1 {
t.Fatalf("previous database not preserved: files=%v err=%v", previous, err)
}
old, err := sql.Open("sqlite", sqliteRestoreDSN(previous[0], "ro"))
if err != nil {
t.Fatal(err)
}
defer old.Close()
if err := old.QueryRow("SELECT value FROM settings").Scan(&value); err != nil || value != "after-stage" {
t.Fatalf("old database lost latest writes: value=%q err=%v", value, err)
}
if err := ApplyPendingSQLiteRestore(t.Context(), cfg, nil); err != nil {
t.Fatalf("repeat startup: %v", err)
}
}
func TestSQLiteRestoreRejectsCorruptAndSymlinkBackups(t *testing.T) {
root := t.TempDir()
cfg := &config.Config{}
cfg.Database.DBPath = filepath.Join(root, "live.db")
source := filepath.Join(root, "corrupt.db")
if err := os.WriteFile(source, []byte("not a SQLite database"), 0o600); err != nil {
t.Fatal(err)
}
if err := StageSQLiteRestore(t.Context(), cfg, source); err == nil {
t.Fatal("corrupt backup accepted")
}
link := filepath.Join(root, "link.db")
if err := os.Symlink(source, link); err == nil {
if err := StageSQLiteRestore(t.Context(), cfg, link); err == nil {
t.Fatal("symlink backup accepted")
}
}
if _, err := os.Stat(cfg.Database.DBPath + ".restore-pending"); !os.IsNotExist(err) {
t.Fatalf("failed validation left pending data: %v", err)
}
}
+18
View File
@@ -0,0 +1,18 @@
package handler
import (
"net/http"
"github.com/gin-gonic/gin"
"github.com/truewhile/MeBox/internal/service"
)
func backgroundTasksHandler(svc *service.Container) gin.HandlerFunc {
return func(c *gin.Context) {
if svc == nil || svc.Tasks == nil {
c.JSON(http.StatusOK, service.TaskSnapshot{Active: []service.BackgroundTask{}, Recent: []service.BackgroundTask{}})
return
}
c.JSON(http.StatusOK, svc.Tasks.Snapshot())
}
}
+18 -1
View File
@@ -3,6 +3,7 @@ package handler
import (
"net/http"
"path/filepath"
"github.com/gin-gonic/gin"
@@ -41,12 +42,28 @@ func deleteBackupHandler(svc *service.Container) gin.HandlerFunc {
}
}
func downloadBackupHandler(svc *service.Container) gin.HandlerFunc {
return func(c *gin.Context) {
filename := c.Query("filename")
path, err := svc.Backup.ResolvePath(filename)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
c.FileAttachment(path, filepath.Base(path))
}
}
func restoreBackupHandler(svc *service.Container) gin.HandlerFunc {
return func(c *gin.Context) {
if err := svc.Backup.Restore(c.Request.Context(), c.Query("filename")); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
c.JSON(http.StatusOK, gin.H{"message": "restored — restart the server to apply"})
message := "数据库已恢复,请重启 MeBox 以刷新运行时配置"
if svc.Repo.DB.Dialector.Name() == "sqlite" {
message = "备份已校验并暂存,请重启 MeBox 后应用恢复"
}
c.JSON(http.StatusOK, gin.H{"message": message, "restart_required": true})
}
}
+146 -105
View File
@@ -6,6 +6,7 @@ import (
"errors"
"math"
"net/http"
"sort"
"strconv"
"strings"
"sync"
@@ -654,6 +655,9 @@ func paginateSlice[T any](items []T, page, size int) []T {
if len(items) == 0 {
return []T{}
}
if page-1 > len(items)/size {
return []T{}
}
start := (page - 1) * size
if start >= len(items) {
return []T{}
@@ -668,127 +672,164 @@ func paginateSlice[T any](items []T, page, size int) []T {
func searchMediaHandler(svc *service.Container) gin.HandlerFunc {
return func(c *gin.Context) {
ctx := c.Request.Context()
q := c.Query("q")
query := strings.TrimSpace(c.Query("q"))
visibility := mediaVisibilityForRequest(c, svc)
groupVersions := c.DefaultQuery("group_versions", "1") != "0"
fetchRemote := func(limit int) []model.Media {
if svc.EmbyRemote == nil || strings.TrimSpace(q) == "" {
return nil
}
remoteItems, _ := svc.EmbyRemote.RemoteSearchMedia(ctx, q, limit, visibility)
return remoteItems
filter := service.MediaSearchFilter{
LibraryID: strings.TrimSpace(c.Query("library_id")),
YearMin: boundedSearchYear(c.Query("year_min")),
YearMax: boundedSearchYear(c.Query("year_max")),
}
if c.DefaultQuery("group_series", "0") != "0" {
localItems, err := svc.Media.SearchMediaVisible(ctx, q, 50000, visibility)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
for _, genre := range c.QueryArray("genre") {
if genre = strings.TrimSpace(genre); genre != "" && len(filter.Genres) < 8 {
filter.Genres = append(filter.Genres, genre)
}
remoteItems := fetchRemote(50000)
all := service.GroupMediaSeriesItems(append(localItems, remoteItems...))
if c.Query("page") != "" || c.Query("page_size") != "" {
page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
size, _ := strconv.Atoi(c.DefaultQuery("page_size", "50"))
paged := paginateSlice(all, page, size)
c.JSON(http.StatusOK, gin.H{
"items": paged,
"total": len(all),
"page": page,
"page_size": size,
})
return
}
limit, _ := strconv.Atoi(c.DefaultQuery("limit", "50"))
if limit <= 0 {
limit = 50
}
if len(all) > limit {
all = all[:limit]
}
c.JSON(http.StatusOK, gin.H{"items": all})
return
}
if c.Query("page") != "" || c.Query("page_size") != "" {
page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
size, _ := strconv.Atoi(c.DefaultQuery("page_size", "50"))
if !groupVersions {
localItems, _, err := svc.Media.SearchMediaVisiblePage(ctx, q, 1, 50000, visibility)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
remoteItems := fetchRemote(size * 2)
all := append(localItems, remoteItems...)
paged := paginateSlice(all, page, size)
c.JSON(http.StatusOK, gin.H{
"items": paged,
"total": len(all),
"page": page,
"page_size": size,
})
return
}
localItems, err := svc.Media.SearchMediaVisible(ctx, q, 50000, visibility)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
remoteItems := fetchRemote(size * 2)
all := append(localItems, remoteItems...)
grouped := service.GroupMediaVersions(all)
paged := service.PaginateMediaItems(grouped, page, size)
c.JSON(http.StatusOK, gin.H{
"items": paged,
"total": len(grouped),
"page": page,
"page_size": size,
})
return
if filter.YearMin > 0 && filter.YearMax > 0 && filter.YearMin > filter.YearMax {
filter.YearMin, filter.YearMax = filter.YearMax, filter.YearMin
}
page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
if page < 1 {
page = 1
}
size, _ := strconv.Atoi(c.DefaultQuery("page_size", "50"))
if size < 1 {
size = 50
} else if size > 2000 {
size = 2000
}
limit, _ := strconv.Atoi(c.DefaultQuery("limit", "50"))
if limit <= 0 {
if limit < 1 {
limit = 50
} else if limit > 50000 {
limit = 50000
}
if !groupVersions {
localItems, err := svc.Media.SearchMediaVisible(ctx, q, limit, visibility)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
remoteItems := fetchRemote(limit)
all := append(localItems, remoteItems...)
if len(all) > limit {
all = all[:limit]
}
if all == nil {
all = []model.Media{}
}
c.JSON(http.StatusOK, gin.H{"items": all})
return
}
localItems, err := svc.Media.SearchMediaVisible(ctx, q, 50000, visibility)
paginated := c.Query("page") != "" || c.Query("page_size") != ""
sortBy := normalizeMediaSearchSort(c.Query("sort"))
// Sorting and grouping must operate on the same candidate set before
// either a page size or the caller's output limit is applied.
items, err := svc.Media.SearchMediaVisibleFiltered(ctx, query, 50000, visibility, filter)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
remoteItems := fetchRemote(limit)
all := append(localItems, remoteItems...)
grouped := service.GroupMediaVersions(all)
items := service.FirstMediaItems(grouped, limit)
if items == nil {
items = []service.MediaItem{}
if svc.EmbyRemote != nil && query != "" && (filter.LibraryID == "" || service.IsEmbyRemoteID(filter.LibraryID)) {
remote, _ := svc.EmbyRemote.RemoteSearchMedia(ctx, query, 50000, visibility)
for _, item := range remote {
if remoteSearchMatchesFilter(item, filter) {
items = append(items, item)
}
}
}
c.JSON(http.StatusOK, gin.H{"items": items})
if c.DefaultQuery("group_series", "0") != "0" {
items = service.GroupMediaSeriesItems(items)
sortSearchMedia(items, sortBy)
writeMediaSearchResults(c, items, page, size, limit, paginated)
return
}
if c.DefaultQuery("group_versions", "1") != "0" {
grouped := service.GroupMediaVersions(items)
if sortBy != "" {
sort.SliceStable(grouped, func(i, j int) bool {
return mediaSearchSortLess(grouped[i].Media, grouped[j].Media, sortBy)
})
}
writeMediaSearchResults(c, grouped, page, size, limit, paginated)
return
}
sortSearchMedia(items, sortBy)
writeMediaSearchResults(c, items, page, size, limit, paginated)
}
}
func writeMediaSearchResults[T any](c *gin.Context, items []T, page, size, limit int, paginated bool) {
if items == nil {
items = []T{}
}
if paginated {
c.JSON(http.StatusOK, gin.H{
"items": paginateSlice(items, page, size),
"total": len(items), "page": page, "page_size": size,
})
return
}
if len(items) > limit {
items = items[:limit]
}
c.JSON(http.StatusOK, gin.H{"items": items})
}
func remoteSearchMatchesFilter(item model.Media, filter service.MediaSearchFilter) bool {
if filter.LibraryID != "" && item.LibraryID != filter.LibraryID && item.DisplayLibraryID != filter.LibraryID {
return false
}
if filter.YearMin > 0 && item.Year < filter.YearMin || filter.YearMax > 0 && item.Year > filter.YearMax {
return false
}
if len(filter.Genres) == 0 {
return true
}
normalize := func(value string) string {
return strings.ToLower(strings.ReplaceAll(strings.TrimSpace(value), " ", ""))
}
for _, genre := range strings.Split(strings.ReplaceAll(item.Genres, ",", ","), ",") {
for _, wanted := range filter.Genres {
if normalize(genre) == normalize(wanted) {
return true
}
}
}
return false
}
func normalizeMediaSearchSort(value string) string {
value = strings.TrimSpace(value)
switch value {
case "title_asc", "year_desc", "rating_desc":
return value
default:
return ""
}
}
func sortSearchMedia(items []model.Media, sortBy string) {
if sortBy == "" {
return
}
sort.SliceStable(items, func(i, j int) bool {
return mediaSearchSortLess(items[i], items[j], sortBy)
})
}
func mediaSearchSortLess(a, b model.Media, sortBy string) bool {
switch sortBy {
case "title_asc":
left, right := strings.ToLower(strings.TrimSpace(a.Title)), strings.ToLower(strings.TrimSpace(b.Title))
if left != right {
return left < right
}
case "year_desc":
if a.Year != b.Year {
return a.Year > b.Year
}
case "rating_desc":
if a.Rating != b.Rating {
return a.Rating > b.Rating
}
}
if !a.CreatedAt.Equal(b.CreatedAt) {
return a.CreatedAt.After(b.CreatedAt)
}
return a.ID < b.ID
}
func boundedSearchYear(value string) int {
year, err := strconv.Atoi(strings.TrimSpace(value))
if err != nil || year < 1870 || year > 2200 {
return 0
}
return year
}
func streamHandler(svc *service.Container) gin.HandlerFunc {
return func(c *gin.Context) {
ctx := c.Request.Context()
+37
View File
@@ -0,0 +1,37 @@
package handler
import (
"net/http"
"strconv"
"github.com/gin-gonic/gin"
"github.com/truewhile/MeBox/internal/service"
)
func mediaQualityHandler(svc *service.Container) gin.HandlerFunc {
return func(c *gin.Context) {
page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
pageSize, _ := strconv.Atoi(c.DefaultQuery("page_size", "50"))
if page < 1 {
page = 1
} else if page > 1000000 {
page = 1000000
}
if pageSize < 1 {
pageSize = 50
} else if pageSize > 100 {
pageSize = 100
}
issue := c.DefaultQuery("issue", "unmatched")
if issue != "unmatched" && issue != "poster" && issue != "missing" {
c.JSON(http.StatusBadRequest, gin.H{"error": "未知检查分类"})
return
}
items, total, err := svc.Media.QualityIssues(c.Request.Context(), issue, page, pageSize)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
c.JSON(http.StatusOK, gin.H{"items": items, "total": total, "page": page, "page_size": pageSize})
}
}
@@ -0,0 +1,88 @@
package handler
import (
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"strconv"
"testing"
"github.com/gin-gonic/gin"
"github.com/glebarez/sqlite"
"go.uber.org/zap"
"gorm.io/gorm"
"github.com/truewhile/MeBox/internal/config"
"github.com/truewhile/MeBox/internal/middleware"
"github.com/truewhile/MeBox/internal/model"
"github.com/truewhile/MeBox/internal/repository"
"github.com/truewhile/MeBox/internal/service"
)
func TestSearchMediaSortLimitFiltersAndDeepPagination(t *testing.T) {
gin.SetMode(gin.TestMode)
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
if err != nil {
t.Fatal(err)
}
sqlDB, _ := db.DB()
defer sqlDB.Close()
if err := db.AutoMigrate(&model.Library{}, &model.Media{}, &model.User{}, &model.Setting{}, &model.PlayProfile{}); err != nil {
t.Fatal(err)
}
repos := repository.New(db)
lib := model.Library{Name: "Review", Path: "/review", Type: "movie", Enabled: true}
if err := repos.Library.Create(t.Context(), &lib); err != nil {
t.Fatal(err)
}
rows := make([]model.Media, 2001)
for i := range rows {
rows[i] = model.Media{
Base: model.Base{ID: fmt.Sprintf("review-%04d", i)}, LibraryID: lib.ID,
Title: fmt.Sprintf("Review %04d", i), Path: fmt.Sprintf("/review/%04d.mp4", i), Year: 2020, Genres: "Comedy",
}
}
rows[2000].Year = 2025
rows[2000].Genres = "Drama"
if err := db.CreateInBatches(rows, 100).Error; err != nil {
t.Fatal(err)
}
svc := &service.Container{Repo: repos, Media: service.NewMediaService(&config.Config{}, zap.NewNop(), repos)}
cases := []struct {
name, query, expectedID string
count, total, page, size int
}{
{name: "sorted output limit", query: "sort=year_desc&limit=1", expectedID: "review-2000", count: 1},
{name: "page past old 2000 cap", query: "sort=title_asc&page=2001&page_size=1", expectedID: "review-2000", count: 1, total: 2001, page: 2001, size: 1},
{name: "genre filtering", query: "genre=Drama&library_id=" + lib.ID, expectedID: "review-2000", count: 1},
{name: "normalized page", query: "page=0&page_size=0", count: 50, total: 2001, page: 1, size: 50},
{name: "bounded page size", query: "page=1&page_size=99999", count: 2000, total: 2001, page: 1, size: 2000},
{name: "huge page does not overflow", query: "page=" + strconv.Itoa(int(^uint(0)>>1)) + "&page_size=2000", count: 0, total: 2001, page: int(^uint(0) >> 1), size: 2000},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
w := httptest.NewRecorder()
ctx, _ := gin.CreateTestContext(w)
ctx.Set(middleware.CtxUserID, "admin")
ctx.Set(middleware.CtxUserRole, "admin")
ctx.Request = httptest.NewRequest(http.MethodGet, "/api/media?q=Review&group_versions=0&"+tc.query, nil)
searchMediaHandler(svc)(ctx)
var response struct {
Items []model.Media `json:"items"`
Total int `json:"total"`
Page int `json:"page"`
Size int `json:"page_size"`
}
if w.Code != http.StatusOK || json.Unmarshal(w.Body.Bytes(), &response) != nil {
t.Fatalf("search failed: status=%d body=%s", w.Code, w.Body.String())
}
if len(response.Items) != tc.count || response.Total != tc.total || response.Page != tc.page || response.Size != tc.size {
t.Fatalf("response count=%d total=%d page=%d size=%d, want %#v", len(response.Items), response.Total, response.Page, response.Size, tc)
}
if tc.expectedID != "" && response.Items[0].ID != tc.expectedID {
t.Fatalf("first ID=%q want %q", response.Items[0].ID, tc.expectedID)
}
})
}
}
+8 -1
View File
@@ -5,6 +5,7 @@ import (
"fmt"
"net/http"
"net/http/httptest"
"net/url"
"testing"
"github.com/gin-gonic/gin"
@@ -27,7 +28,8 @@ func TestSearchMediaSeriesKeepsRemoteLibrariesSeparate(t *testing.T) {
"TotalRecordCount": 1,
"Items": []map[string]any{{
"Id": "same-series", "Name": "龙珠改", "Type": "Series",
"ProviderIds": map[string]string{"Tmdb": "61709"},
"ProviderIds": map[string]string{"Tmdb": "61709"},
"ProductionYear": 2024, "Genres": []string{"Science Fiction"},
}},
})
}))
@@ -108,4 +110,9 @@ func TestSearchMediaSeriesKeepsRemoteLibrariesSeparate(t *testing.T) {
}
seen[item.DisplayLibraryID] = true
}
remoteLibraryID := service.EncodeEmbyRemoteID("mount-a", "view-a")
items, total := request("page=1&page_size=8&library_id=" + url.QueryEscape(remoteLibraryID) + "&year_min=2020&genre=Science+Fiction")
if total != 1 || len(items) != 1 || items[0].DisplayLibraryID != remoteLibraryID {
t.Fatalf("remote library/year/genre filters lost results: total=%d items=%#v", total, items)
}
}
+3 -3
View File
@@ -149,10 +149,10 @@ func readerDeleteSourceHandler(svc *service.Container) gin.HandlerFunc {
}
func readerDebugSourceHandler(svc *service.Container) gin.HandlerFunc {
var body struct {
Key string `json:"key" binding:"required"`
}
return func(c *gin.Context) {
var body struct {
Key string `json:"key" binding:"required"`
}
if err := c.ShouldBindJSON(&body); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
+3
View File
@@ -13,6 +13,8 @@ func registerAdminRoutes(api *gin.RouterGroup, cfg *config.Config, svc *service.
admin := api.Group("/admin")
admin.Use(middleware.AuthRequired(cfg.Secrets.JWTSecret), middleware.AdminRequired())
registerAdminUserRoutes(admin, svc)
admin.GET("/tasks", backgroundTasksHandler(svc))
admin.GET("/quality", mediaQualityHandler(svc))
registerAdminPermissionRoutes(admin, svc)
registerAdminSystemRoutes(admin, svc)
registerAdminBackupRoutes(admin, svc)
@@ -143,6 +145,7 @@ func registerAdminSystemRoutes(admin *gin.RouterGroup, svc *service.Container) {
func registerAdminBackupRoutes(admin *gin.RouterGroup, svc *service.Container) {
admin.GET("/backups", listBackupsHandler(svc))
admin.POST("/backups", createBackupHandler(svc))
admin.GET("/backups/download", downloadBackupHandler(svc))
admin.DELETE("/backups", deleteBackupHandler(svc))
admin.POST("/backups/restore", restoreBackupHandler(svc))
}
+23
View File
@@ -0,0 +1,23 @@
package model
import "time"
// BackgroundTaskRecord persists the operator-facing lifecycle of scan,
// organize, and other long-running application tasks.
type BackgroundTaskRecord struct {
ID string `gorm:"primaryKey;size:36" json:"id"`
Kind string `gorm:"index;size:32;not null" json:"kind"`
Name string `gorm:"size:255;not null" json:"name"`
Status string `gorm:"index;size:24;not null" json:"status"`
Stage string `gorm:"size:64" json:"stage"`
SourcePath string `gorm:"size:2048" json:"source_path"`
DestPath string `gorm:"size:2048" json:"dest_path"`
Message string `gorm:"type:text" json:"message"`
Error string `gorm:"type:text" json:"error"`
Details string `gorm:"type:text" json:"details"`
Metrics string `gorm:"type:text" json:"metrics"`
Items string `gorm:"type:text" json:"items"`
StartedAt time.Time `gorm:"index" json:"started_at"`
UpdatedAt time.Time `json:"updated_at"`
FinishedAt *time.Time `json:"finished_at,omitempty"`
}
+11 -6
View File
@@ -1,5 +1,7 @@
package model
import "time"
// Library 表示一个逻辑媒体库。Path 保留为兼容字段,指向第一条 LibraryRoot。
type Library struct {
Base
@@ -26,12 +28,15 @@ type LibraryRoot struct {
// Media 是单个可播放项。剧集链接到 SeriesID;电影 SeriesID == ""。
type Media struct {
Base
LibraryID string `gorm:"index;size:36;index:idx_media_library_release,priority:1" json:"library_id"`
LibraryRootID string `gorm:"index;size:36" json:"library_root_id,omitempty"`
SeriesID string `gorm:"index;size:128" json:"series_id,omitempty"`
Title string `gorm:"size:255;not null" json:"title"`
OriginalName string `gorm:"size:255" json:"original_name,omitempty"`
EpisodeTitle string `gorm:"size:255" json:"episode_title,omitempty"`
AvailabilityState string `gorm:"index;size:16;not null;default:available" json:"availability_state"`
MissingSince *time.Time `json:"missing_since,omitempty"`
LastSeenAt *time.Time `json:"last_seen_at,omitempty"`
LibraryID string `gorm:"index;size:36;index:idx_media_library_release,priority:1" json:"library_id"`
LibraryRootID string `gorm:"index;size:36" json:"library_root_id,omitempty"`
SeriesID string `gorm:"index;size:128" json:"series_id,omitempty"`
Title string `gorm:"size:255;not null" json:"title"`
OriginalName string `gorm:"size:255" json:"original_name,omitempty"`
EpisodeTitle string `gorm:"size:255" json:"episode_title,omitempty"`
// Path 的唯一性由 database 包按方言显式建索引,这里刻意不用 uniqueIndex:
// PostgreSQL 的 btree 单条索引项上限 2704 字节,而 varchar(1024) 在中文路径
// 下可达 3072 字节,直接对整列建唯一索引会让长路径的扫描插入直接失败
+1
View File
@@ -61,6 +61,7 @@ func AllModels() []interface{} {
&StrmUploadTask{},
&StrmDirCache{},
&ScrapeTask{},
&BackgroundTaskRecord{},
&EmbyMount{},
&ReaderBookSource{},
&ReaderSourceState{},
@@ -5,6 +5,7 @@ import (
"errors"
"path/filepath"
"strings"
"time"
"gorm.io/gorm"
"gorm.io/gorm/clause"
@@ -163,6 +164,12 @@ func (r *MediaRepository) upsertWithDB(ctx context.Context, db *gorm.DB, m *mode
existing.Path = m.Path
existing.DeletedAt = gorm.DeletedAt{}
}
if _, ok := updates["availability_state"]; ok {
existing.AvailabilityState = "available"
existing.MissingSince = nil
now := updates["last_seen_at"].(time.Time)
existing.LastSeenAt = &now
}
*m = existing
return existing.ID, nil
}
@@ -216,6 +223,10 @@ func (r *MediaRepository) findOrCreateMediaByPath(ctx context.Context, db *gorm.
if m.ScrapeStatus == "" {
m.ScrapeStatus = "pending"
}
now := time.Now()
m.AvailabilityState = "available"
m.MissingSince = nil
m.LastSeenAt = &now
// PostgreSQL 唯一键异常会中止整个事务,之后重查也会失败。
// DO NOTHING 保持事务可用,并兼容 path 的表达式唯一索引。
result := db.WithContext(ctx).Clauses(clause.OnConflict{DoNothing: true}).Create(m)
@@ -265,6 +276,11 @@ func (r *MediaRepository) findMediaByAlias(ctx context.Context, db *gorm.DB, cur
func mediaUpsertUpdates(existing, incoming model.Media, reclaimLibrary bool) map[string]any {
updates := map[string]any{}
if existing.AvailabilityState != "available" || existing.MissingSince != nil {
updates["availability_state"] = "available"
updates["missing_since"] = nil
updates["last_seen_at"] = time.Now()
}
addMediaFileScanUpdates(updates, existing, incoming)
addMediaTitleUpdates(updates, existing, incoming)
addMediaExternalIDUpdates(updates, existing, incoming)
+81 -37
View File
@@ -5,7 +5,7 @@
//
// 两种方言的备份形态不同:
// - SQLite:用 "VACUUM INTO"(3.27.0 起可用)产出一个一致的库文件,
// 扩展名 .db;恢复即「替换库文件 + 重启」。
// 扩展名 .db;恢复先暂存,在下次启动打开数据库前替换文件。
// - PostgreSQL:没有单文件语义,用逻辑备份(见 backup_postgres.go),
// 扩展名 .json.gz;恢复是在单个事务内重建全部表。
package service
@@ -24,6 +24,7 @@ import (
"gorm.io/gorm"
"github.com/truewhile/MeBox/internal/config"
"github.com/truewhile/MeBox/internal/database"
)
// BackupService manages database backups.
@@ -44,6 +45,7 @@ type BackupInfo struct {
Path string `json:"path"`
Size int64 `json:"size"`
CreatedAt time.Time `json:"created_at"`
Automatic bool `json:"automatic"`
}
func (b *BackupService) backupDir() string {
@@ -69,6 +71,23 @@ func (b *BackupService) backupFilePath(filename string) (string, error) {
return path, nil
}
// ResolvePath returns the absolute path of a listed backup after applying the
// same filename and directory checks used by restore and delete operations.
func (b *BackupService) ResolvePath(filename string) (string, error) {
path, err := b.backupFilePath(filename)
if err != nil {
return "", err
}
info, err := os.Lstat(path)
if err != nil {
return "", err
}
if !info.Mode().IsRegular() {
return "", errors.New("backup is not a regular file")
}
return path, nil
}
// isPostgres reports whether the service is backed by PostgreSQL.
//
// PostgreSQL 不支持 SQLite 的 "VACUUM INTO",按文件替换恢复也会写错目标,
@@ -80,11 +99,23 @@ func (b *BackupService) isPostgres() bool {
// Create produces a new hot backup. SQLite uses "VACUUM INTO"; PostgreSQL uses
// a logical dump.
func (b *BackupService) Create(ctx context.Context) (*BackupInfo, error) {
return b.create(ctx, false)
}
func (b *BackupService) CreateAutomatic(ctx context.Context) (*BackupInfo, error) {
return b.create(ctx, true)
}
func (b *BackupService) create(ctx context.Context, automatic bool) (*BackupInfo, error) {
dir := b.backupDir()
if err := os.MkdirAll(dir, 0o750); err != nil {
return nil, err
}
ts := time.Now().Format("20060102_150405")
now := time.Now()
ts := fmt.Sprintf("%s_%09d", now.Format("20060102_150405"), now.Nanosecond())
if automatic {
ts = "auto_" + ts
}
var (
dst string
@@ -108,15 +139,24 @@ func (b *BackupService) Create(ctx context.Context) (*BackupInfo, error) {
Path: dst,
Size: stat.Size(),
CreatedAt: stat.ModTime(),
Automatic: automatic,
}, nil
}
func (b *BackupService) createSQLiteBackup(ctx context.Context, stamp string) (string, error) {
dst := filepath.Join(b.backupDir(), fmt.Sprintf("mebox_%s.db", stamp))
// VACUUM INTO creates a clean, non-WAL copy atomically.
if err := b.db.WithContext(ctx).Exec("VACUUM INTO ?", dst).Error; err != nil {
tmp := dst + ".tmp"
defer os.Remove(tmp)
// Publish only completed backups, with credentials protected by file mode.
if err := b.db.WithContext(ctx).Exec("VACUUM INTO ?", tmp).Error; err != nil {
return "", fmt.Errorf("VACUUM INTO failed: %w", err)
}
if err := os.Chmod(tmp, 0o600); err != nil {
return "", err
}
if err := os.Rename(tmp, dst); err != nil {
return "", err
}
return dst, nil
}
@@ -132,7 +172,7 @@ func (b *BackupService) List() ([]BackupInfo, error) {
}
out := make([]BackupInfo, 0, len(entries))
for _, e := range entries {
if e.IsDir() || !isBackupFilename(e.Name()) {
if !e.Type().IsRegular() || !isBackupFilename(e.Name()) {
continue
}
info, err := e.Info()
@@ -144,6 +184,7 @@ func (b *BackupService) List() ([]BackupInfo, error) {
Path: filepath.Join(dir, e.Name()),
Size: info.Size(),
CreatedAt: info.ModTime(),
Automatic: strings.HasPrefix(e.Name(), "mebox_auto_"),
})
}
sort.Slice(out, func(i, j int) bool {
@@ -152,6 +193,36 @@ func (b *BackupService) List() ([]BackupInfo, error) {
return out, nil
}
// Prune keeps maxCount automatic backups for the current database dialect.
// Manual backups and backups from another dialect are never pruned.
// A value below one disables pruning so an invalid setting cannot erase all
// recoverable copies.
func (b *BackupService) Prune(maxCount int) error {
if maxCount < 1 {
return nil
}
items, err := b.List()
if err != nil {
return err
}
eligible := items[:0]
for _, item := range items {
if item.Automatic && strings.HasSuffix(item.Filename, postgresBackupSuffix) == b.isPostgres() {
eligible = append(eligible, item)
}
}
items = eligible
if maxCount >= len(items) {
return nil
}
for _, item := range items[maxCount:] {
if err := b.Delete(item.Filename); err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
}
return nil
}
// Delete removes a single backup file.
func (b *BackupService) Delete(filename string) error {
path, err := b.backupFilePath(filename)
@@ -161,12 +232,10 @@ func (b *BackupService) Delete(filename string) error {
return os.Remove(path)
}
// Restore replaces the live database with the given backup.
// WARNING: this is destructive. SQLite restores by swapping the database file
// (the caller should shut down the server afterwards); PostgreSQL restores by
// rebuilding every table inside a single transaction.
// Restore stages SQLite backups for the next startup. PostgreSQL restores
// rebuild every table inside a single transaction.
func (b *BackupService) Restore(ctx context.Context, filename string) error {
src, err := b.backupFilePath(filename)
src, err := b.ResolvePath(filename)
if err != nil {
return err
}
@@ -182,35 +251,10 @@ func (b *BackupService) Restore(ctx context.Context, filename string) error {
if !strings.HasSuffix(filename, ".db") {
return errors.New("该备份是 PostgreSQL 逻辑备份,当前运行在 SQLite 上,无法恢复")
}
return b.restoreSQLiteBackup(filename, src)
}
func (b *BackupService) restoreSQLiteBackup(filename, src string) error {
dbPath, err := filepath.Abs(b.cfg.Database.DBPath)
if err != nil || dbPath == "" {
return errors.New("invalid database path")
}
// Strategy: rename live → live.old, copy backup → live, delete old.
old := dbPath + ".before_restore"
if err := os.Rename(dbPath, old); err != nil && !errors.Is(err, os.ErrNotExist) {
if err := database.StageSQLiteRestore(ctx, b.cfg, src); err != nil {
return err
}
data, err := os.ReadFile(src) // #nosec G304 -- src is constrained by backupFilePath to the configured backups directory.
if err != nil {
// Revert
_ = os.Rename(old, dbPath)
return err
}
if err := os.WriteFile(dbPath, data, 0o600); err != nil { // #nosec G703 -- database path is trusted process config and normalized before restore.
_ = os.Rename(old, dbPath)
return err
}
_ = os.Remove(old)
// Also remove WAL/SHM so SQLite opens the fresh file cleanly.
_ = os.Remove(dbPath + "-wal")
_ = os.Remove(dbPath + "-shm")
b.log.Warn("database restored from backup — restart the server",
zap.String("backup", filename))
b.log.Warn("SQLite backup staged; restart to apply", zap.String("backup", filename))
return nil
}
+33 -12
View File
@@ -3,6 +3,7 @@ package service
import (
"compress/gzip"
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
@@ -87,7 +88,8 @@ func postgresBackupModelSchemas(db *gorm.DB) (map[string]*schema.Schema, map[str
// createPostgresBackup 把全库导出成一个 gzip 压缩的 JSON Lines 文件。
func (b *BackupService) createPostgresBackup(ctx context.Context, stamp string) (string, error) {
dst := filepath.Join(b.backupDir(), "mebox_"+stamp+postgresBackupSuffix)
file, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600)
tmp := dst + ".tmp"
file, err := os.OpenFile(tmp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
if err != nil {
return "", err
}
@@ -96,7 +98,7 @@ func (b *BackupService) createPostgresBackup(ctx context.Context, stamp string)
_ = file.Close()
if !complete {
// 半截备份文件比没有备份更危险:留着会让人以为有可用的还原点。
_ = os.Remove(dst)
_ = os.Remove(tmp)
}
}()
@@ -109,17 +111,30 @@ func (b *BackupService) createPostgresBackup(ctx context.Context, stamp string)
}); err != nil {
return "", fmt.Errorf("写入备份头失败: %w", err)
}
for _, m := range model.AllModels() {
table, sch, err := parseModelSchema(b.db, m)
if err != nil {
return "", err
}
if !b.db.Migrator().HasTable(table) {
continue
}
if err := b.dumpPostgresTable(ctx, m, sch, table, enc); err != nil {
return "", err
var options []*sql.TxOptions
if b.isPostgres() {
options = append(options, &sql.TxOptions{Isolation: sql.LevelRepeatableRead, ReadOnly: true})
}
// Every table must come from the same snapshot, including while background
// workers are inserting related rows during an automatic backup.
if err := b.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
snapshot := *b
snapshot.db = tx
for _, m := range model.AllModels() {
table, sch, err := parseModelSchema(tx, m)
if err != nil {
return err
}
if !tx.Migrator().HasTable(table) {
continue
}
if err := snapshot.dumpPostgresTable(ctx, m, sch, table, enc); err != nil {
return err
}
}
return nil
}, options...); err != nil {
return "", err
}
if err := zw.Close(); err != nil {
return "", fmt.Errorf("压缩备份失败: %w", err)
@@ -127,6 +142,12 @@ func (b *BackupService) createPostgresBackup(ctx context.Context, stamp string)
if err := file.Sync(); err != nil {
return "", fmt.Errorf("写入备份失败: %w", err)
}
if err := file.Close(); err != nil {
return "", err
}
if err := os.Rename(tmp, dst); err != nil {
return "", err
}
complete = true
return dst, nil
}
+65
View File
@@ -0,0 +1,65 @@
package service
import (
"os"
"path/filepath"
"runtime"
"testing"
"time"
)
func TestBackupRetentionPreservesManualAndOtherDialectBackups(t *testing.T) {
svc, _ := newBackupTestService(t)
names := []string{"mebox_auto_new.db", "mebox_auto_old.db", "mebox_manual.db", "mebox_auto_postgres.json.gz"}
for i, name := range names {
path := filepath.Join(svc.backupDir(), name)
if err := os.WriteFile(path, []byte("fixture"), 0o600); err != nil {
t.Fatal(err)
}
when := time.Now().Add(-time.Duration(i) * time.Hour)
if err := os.Chtimes(path, when, when); err != nil {
t.Fatal(err)
}
}
if err := svc.Prune(1); err != nil {
t.Fatal(err)
}
for _, name := range names {
_, err := os.Stat(filepath.Join(svc.backupDir(), name))
if name == "mebox_auto_old.db" {
if !os.IsNotExist(err) {
t.Fatalf("old automatic backup was retained: %v", err)
}
} else if err != nil {
t.Fatalf("backup %s should have been preserved: %v", name, err)
}
}
}
func TestBackupCreateUsesUniquePrivateCompletedFiles(t *testing.T) {
svc, _ := newBackupTestService(t)
manual, err := svc.Create(t.Context())
if err != nil {
t.Fatal(err)
}
automatic, err := svc.CreateAutomatic(t.Context())
if err != nil {
t.Fatal(err)
}
if manual.Filename == automatic.Filename || manual.Automatic || !automatic.Automatic {
t.Fatalf("invalid backup identities: manual=%#v automatic=%#v", manual, automatic)
}
items, err := svc.List()
if err != nil || len(items) != 2 {
t.Fatalf("completed backup list: items=%#v err=%v", items, err)
}
for _, item := range items {
info, err := os.Stat(item.Path)
if err != nil {
t.Fatal(err)
}
if runtime.GOOS != "windows" && info.Mode().Perm() != 0o600 {
t.Fatalf("backup permissions are not private: %s err=%v", item.Path, err)
}
}
}
+66
View File
@@ -2,17 +2,71 @@ package service
import (
"context"
"fmt"
"github.com/truewhile/MeBox/internal/model"
"github.com/truewhile/MeBox/internal/repository"
)
type MediaQualityItem struct {
Issue string `json:"issue"`
Media model.Media `json:"media"`
}
func (s *MediaService) QualityIssues(ctx context.Context, issue string, page, pageSize int) ([]MediaQualityItem, int64, error) {
if page < 1 {
page = 1
}
if pageSize < 1 {
pageSize = 50
}
if pageSize > 100 {
pageSize = 100
}
q := s.repo.DB.WithContext(ctx).Model(&model.Media{})
switch issue {
case "missing":
q = q.Where("availability_state = ?", "missing")
case "unmatched":
q = q.Where("scrape_status IN ? OR scrape_status = ''", []string{"pending", "no_match"})
case "poster":
q = q.Where("poster_url IS NULL OR TRIM(poster_url) = ''")
default:
return nil, 0, fmt.Errorf("unknown quality issue: %s", issue)
}
var total int64
if err := q.Count(&total).Error; err != nil {
return nil, 0, err
}
var rows []model.Media
if err := q.Order("updated_at DESC, id ASC").Offset((page - 1) * pageSize).Limit(pageSize).Find(&rows).Error; err != nil {
return nil, 0, err
}
s.attachLibraryMetadata(ctx, rows)
items := make([]MediaQualityItem, 0, len(rows))
for _, media := range rows {
items = append(items, MediaQualityItem{Issue: issue, Media: media})
}
return items, total, nil
}
type MediaSearchFilter struct {
LibraryID string
YearMin int
YearMax int
Genres []string
}
// SearchMedia performs a simple LIKE search across titles.
func (s *MediaService) SearchMedia(ctx context.Context, query string, limit int) ([]model.Media, error) {
return s.SearchMediaVisible(ctx, query, limit, MediaVisibility{IncludeNSFW: true})
}
func (s *MediaService) SearchMediaVisible(ctx context.Context, query string, limit int, visibility MediaVisibility) ([]model.Media, error) {
return s.SearchMediaVisibleFiltered(ctx, query, limit, visibility, MediaSearchFilter{})
}
func (s *MediaService) SearchMediaVisibleFiltered(ctx context.Context, query string, limit int, visibility MediaVisibility, filter MediaSearchFilter) ([]model.Media, error) {
if limit <= 0 {
limit = 50
} else if limit > maxMediaSearchLimit {
@@ -23,6 +77,10 @@ func (s *MediaService) SearchMediaVisible(ctx context.Context, query string, lim
IncludeNSFW: visibility.IncludeNSFW,
AllowedLibraryIDs: visibility.AllowedLibraryIDs,
HiddenLibraryIDs: visibility.HiddenLibraryIDs,
LibraryID: filter.LibraryID,
YearMin: filter.YearMin,
YearMax: filter.YearMax,
Genres: filter.Genres,
})
if err != nil {
return nil, err
@@ -45,6 +103,10 @@ func (s *MediaService) SearchMediaVisibleGrouped(ctx context.Context, query stri
}
func (s *MediaService) SearchMediaVisiblePage(ctx context.Context, query string, page, pageSize int, visibility MediaVisibility) ([]model.Media, int64, error) {
return s.SearchMediaVisiblePageFiltered(ctx, query, page, pageSize, visibility, MediaSearchFilter{})
}
func (s *MediaService) SearchMediaVisiblePageFiltered(ctx context.Context, query string, page, pageSize int, visibility MediaVisibility, filter MediaSearchFilter) ([]model.Media, int64, error) {
if pageSize <= 0 {
pageSize = 50
}
@@ -59,6 +121,10 @@ func (s *MediaService) SearchMediaVisiblePage(ctx context.Context, query string,
IncludeNSFW: visibility.IncludeNSFW,
AllowedLibraryIDs: visibility.AllowedLibraryIDs,
HiddenLibraryIDs: visibility.HiddenLibraryIDs,
LibraryID: filter.LibraryID,
YearMin: filter.YearMin,
YearMax: filter.YearMax,
Genres: filter.Genres,
})
if err != nil {
return nil, 0, err
+29 -26
View File
@@ -14,6 +14,7 @@ import (
"errors"
"strings"
"sync"
"time"
"go.uber.org/zap"
@@ -144,32 +145,34 @@ type localMediaProbeTask struct {
}
type existingLocalMedia struct {
LibraryRootID string
RelativePath string
Title string
OriginalName string
EpisodeTitle string
SizeBytes int64
DurationSec int
Width int
Height int
VideoCodec string
AudioCodec string
Container string
STRMURL string
FileID string
PosterURL string
BackdropURL string
Overview string
Year int
ReleaseDate string
Rating float32
TMDbID int
BangumiID int
DoubanID string
TheTVDBID string
SeasonNum int
EpisodeNum int
AvailabilityState string
MissingSince *time.Time
LibraryRootID string
RelativePath string
Title string
OriginalName string
EpisodeTitle string
SizeBytes int64
DurationSec int
Width int
Height int
VideoCodec string
AudioCodec string
Container string
STRMURL string
FileID string
PosterURL string
BackdropURL string
Overview string
Year int
ReleaseDate string
Rating float32
TMDbID int
BangumiID int
DoubanID string
TheTVDBID string
SeasonNum int
EpisodeNum int
// EpisodeFraction 是集号的小数部分(S01E11.5 → 0.5);变化同样要触发刷新,
// 否则已有行上的「半集」标记不会被写回。
EpisodeFraction float64
+3 -1
View File
@@ -33,7 +33,7 @@ func (s *ScannerService) existingLocalMediaSnapshotForRoot(ctx context.Context,
query = query.Where("(library_root_id = ? OR library_root_id = '' OR library_root_id IS NULL)", rootID)
}
rows, err := query.
Select("path", "library_root_id", "relative_path", "title", "original_name", "episode_title", "size_bytes", "duration_sec", "width", "height", "video_codec", "audio_codec", "container", "strm_url", "file_id", "poster_url", "backdrop_url", "overview", "year", "release_date", "rating", "tm_db_id", "bangumi_id", "douban_id", "thetvdb_id", "season_num", "episode_num", "episode_fraction", "genres", "countries", "languages", "nsfw", "scrape_status").
Select("path", "library_root_id", "relative_path", "title", "original_name", "episode_title", "size_bytes", "duration_sec", "width", "height", "video_codec", "audio_codec", "container", "strm_url", "file_id", "poster_url", "backdrop_url", "overview", "year", "release_date", "rating", "tm_db_id", "bangumi_id", "douban_id", "thetvdb_id", "season_num", "episode_num", "episode_fraction", "genres", "countries", "languages", "nsfw", "scrape_status", "availability_state", "missing_since").
Rows()
if err != nil {
return nil, err
@@ -80,6 +80,8 @@ func (s *ScannerService) existingLocalMediaSnapshotForRoot(ctx context.Context,
&row.Languages,
&row.NSFW,
&row.ScrapeStatus,
&row.AvailabilityState,
&row.MissingSince,
); err != nil {
return nil, err
}
+10 -3
View File
@@ -548,7 +548,7 @@ func TestScanLibraryMapsPersistedHostLibraryPath(t *testing.T) {
}
}
func TestRemovePathDeletesVanishedMedia(t *testing.T) {
func TestRemovePathRetainsVanishedMedia(t *testing.T) {
sc, repos := newScannerTestEnv(t)
cache := NewRuntimeCacheService(&config.Config{}, zap.NewNop())
sc.SetRuntimeCache(cache)
@@ -589,8 +589,15 @@ func TestRemovePathDeletesVanishedMedia(t *testing.T) {
if removed != 1 {
t.Fatalf("removed = %d, want 1", removed)
}
if countMedia(t, repos) != 0 {
t.Fatal("expected 0 media after removal")
if countMedia(t, repos) != 1 {
t.Fatal("missing media metadata must be retained")
}
var missing model.Media
if err := repos.DB.Where("path = ?", file).First(&missing).Error; err != nil {
t.Fatal(err)
}
if missing.AvailabilityState != "missing" || missing.MissingSince == nil {
t.Fatalf("missing state not recorded: %#v", missing)
}
if cache.GetJSON(t.Context(), "media:list:test", &cached) {
t.Fatal("vanished media removal should invalidate media cache")
+10 -2
View File
@@ -67,8 +67,16 @@ func TestScanLibraryScansMultipleRootsAndPrunesPerRoot(t *testing.T) {
if err := db.Find(&rows).Error; err != nil {
t.Fatal(err)
}
if len(rows) != 1 || rows[0].Path != fileB {
t.Fatalf("remaining rows = %#v, want offline root media preserved", rows)
if len(rows) != 2 {
t.Fatalf("remaining rows = %#v, want both media records preserved", rows)
}
for _, row := range rows {
if row.Path == fileA && row.AvailabilityState != "missing" {
t.Fatalf("vanished file was not marked missing: %#v", row)
}
if row.Path == fileB && row.AvailabilityState != "available" {
t.Fatalf("offline root status was changed: %#v", row)
}
}
}
+1 -1
View File
@@ -122,7 +122,7 @@ func (s *ScannerService) localMediaScanState(in localMediaScanStateInput) (bool,
}
existing, exists := in.existingMedia[in.cleanPath]
isNewMedia := !exists
if exists && in.ext != ".strm" && existing.SizeBytes == in.size && !localMetadataNeedsRefresh(existing, in.localMeta) && !localDerivedMetadataNeedsRefresh(existing, in.incoming) {
if exists && existing.AvailabilityState != "missing" && existing.MissingSince == nil && in.ext != ".strm" && existing.SizeBytes == in.size && !localMetadataNeedsRefresh(existing, in.localMeta) && !localDerivedMetadataNeedsRefresh(existing, in.incoming) {
// Unchanged rows are skipped, but a row that never received ffprobe data
// still needs a backfill probe so installing ffmpeg later repairs the library.
s.queueProbeBackfillIfNeeded(in.path, in.ext, existing)
@@ -372,7 +372,14 @@ func TestScanLibraryPrunesMissingMedia(t *testing.T) {
if err := db.Model(&model.Media{}).Where("library_id = ?", lib.ID).Count(&count).Error; err != nil {
t.Fatal(err)
}
if count != 1 {
t.Fatalf("media count = %d, want 1", count)
if count != 2 {
t.Fatalf("media count = %d, want current and missing records preserved", count)
}
var missing model.Media
if err := db.First(&missing, "id = ?", stale.ID).Error; err != nil {
t.Fatal(err)
}
if missing.AvailabilityState != "missing" {
t.Fatalf("stale media state = %q, want missing", missing.AvailabilityState)
}
}
+55
View File
@@ -0,0 +1,55 @@
package service
import (
"os"
"path/filepath"
"testing"
"github.com/truewhile/MeBox/internal/model"
)
func TestScanRestoresUnchangedMissingFileAndMetadata(t *testing.T) {
scanner, repos := newScannerTestEnv(t)
root := t.TempDir()
library := model.Library{Name: "Movies", Path: root, Type: "movie", Enabled: true}
if err := repos.Library.Create(t.Context(), &library); err != nil {
t.Fatal(err)
}
path := filepath.Join(root, "Example (2024).mkv")
writeTestFile(t, path, "unchanged content")
if _, err := scanner.ScanLibrary(t.Context(), library.ID); err != nil {
t.Fatal(err)
}
var original model.Media
if err := repos.DB.Where("path = ?", path).First(&original).Error; err != nil {
t.Fatal(err)
}
if err := repos.DB.Model(&original).Updates(map[string]any{
"title": "Manually matched title", "scrape_status": "matched", "tm_db_id": 42,
}).Error; err != nil {
t.Fatal(err)
}
if err := os.Remove(path); err != nil {
t.Fatal(err)
}
if changed, err := scanner.RemovePath(t.Context(), path); err != nil || changed != 1 {
t.Fatalf("mark missing: changed=%d err=%v", changed, err)
}
if changed, err := scanner.RemovePath(t.Context(), path); err != nil || changed != 0 {
t.Fatalf("repeated missing event: changed=%d err=%v", changed, err)
}
writeTestFile(t, path, "unchanged content")
if _, err := scanner.ScanLibrary(t.Context(), library.ID); err != nil {
t.Fatal(err)
}
var restored model.Media
if err := repos.DB.Where("path = ?", path).First(&restored).Error; err != nil {
t.Fatal(err)
}
if restored.ID != original.ID || restored.TMDbID != 42 || restored.Title != "Manually matched title" {
t.Fatalf("restored file lost identity or metadata: %#v", restored)
}
if restored.AvailabilityState != "available" || restored.MissingSince != nil || restored.LastSeenAt == nil {
t.Fatalf("unchanged restored file stayed missing: %#v", restored)
}
}
+44 -15
View File
@@ -2,6 +2,7 @@ package service
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
@@ -12,11 +13,15 @@ import (
"github.com/truewhile/MeBox/internal/model"
)
// RemovePath 移除磁盘上已不存在的媒体记录。普通的删除仍做物理删除;
// 可确认是 STRM 扩展名改名的路径则先保留软删除墓碑,供后续 ingest 继承元数据。
// RemovePath 将确认不存在的媒体标记为失联;只有可确认的 STRM 扩展名改名
// 才保留软删除墓碑,供后续 ingest 继承元数据。
func (s *ScannerService) RemovePath(ctx context.Context, path string) (int64, error) {
if _, err := os.Stat(path); err == nil {
return 0, nil // still exists; nothing to remove
} else if !errors.Is(err, os.ErrNotExist) {
// Permission, I/O, and mount errors are not evidence that a file was
// removed. Preserve records until the path can be inspected reliably.
return 0, nil
}
// 目录整体消失(删除/改名离开):连同其子树下的媒体行一并移除。
// 此前只删 path 精确匹配的行——目录本身通常没有 media 行,导致目录
@@ -33,7 +38,7 @@ func (s *ScannerService) RemovePath(ctx context.Context, path string) (int64, er
return 0, err
}
softIDs := make([]string, 0, len(rows))
hardIDs := make([]string, 0, len(rows))
missingIDs := make([]string, 0, len(rows))
for _, row := range rows {
// LIKE 里的 % _ 是通配符(候选集只会偏大),用 Go 前缀精确过滤,
// 避免对含 % / _ 的路径误删。
@@ -43,12 +48,12 @@ func (s *ScannerService) RemovePath(ctx context.Context, path string) (int64, er
// A vanished STRM with a live sibling (foo.strm <-> foo.mkv.strm)
// is a rename. Keep a soft-deleted tombstone so the later ingest can
// adopt its ID and scraped metadata even if fsnotify delivers the
// remove event before the create event. A true deletion stays hard.
// remove event before the create event.
if row.Path == path && liveSTRMPathAlias(path) {
softIDs = append(softIDs, row.ID)
continue
}
hardIDs = append(hardIDs, row.ID)
missingIDs = append(missingIDs, row.ID)
}
var removed int64
if len(softIDs) > 0 {
@@ -60,14 +65,12 @@ func (s *ScannerService) RemovePath(ctx context.Context, path string) (int64, er
}
removed += res.RowsAffected
}
if len(hardIDs) > 0 {
res := s.repo.DB.WithContext(ctx).Unscoped().
Where("id IN ?", hardIDs).
Delete(&model.Media{})
if res.Error != nil {
return removed, res.Error
if len(missingIDs) > 0 {
count, err := s.markMediaMissingByIDs(ctx, missingIDs)
if err != nil {
return removed, err
}
removed += res.RowsAffected
removed += count
}
if removed > 0 {
s.invalidateMediaCache(ctx)
@@ -82,7 +85,7 @@ const strmRenameTombstoneRetention = 7 * 24 * time.Hour
// purgeExpiredSTRMTombstones keeps the soft-delete grace period bounded.
// It is deliberately restricted to deleted media rows under a STRM alias
// workflow; normal deletions are still hard-deleted immediately.
// workflow; ordinary missing files retain their metadata for recovery.
func (s *ScannerService) purgeExpiredSTRMTombstones(ctx context.Context) {
cutoff := time.Now().Add(-strmRenameTombstoneRetention)
if err := s.repo.DB.WithContext(ctx).Unscoped().
@@ -121,7 +124,7 @@ func (s *ScannerService) pruneMissingMedia(ctx context.Context, libraryID string
}
stale = append(stale, row.ID)
}
return s.deleteMediaByIDs(ctx, stale, true)
return s.markMediaMissingByIDs(ctx, stale)
}
func (s *ScannerService) pruneMissingMediaForRoot(ctx context.Context, libraryID, rootID, rootPath string, seen map[string]struct{}) (int64, error) {
@@ -158,7 +161,33 @@ func (s *ScannerService) pruneMissingMediaForRoot(ctx context.Context, libraryID
}
stale = append(stale, row.ID)
}
return s.deleteMediaByIDs(ctx, stale, true)
return s.markMediaMissingByIDs(ctx, stale)
}
func (s *ScannerService) markMediaMissingByIDs(ctx context.Context, ids []string) (int64, error) {
if len(ids) == 0 {
return 0, nil
}
const batch = 500
var changed int64
now := time.Now()
for i := 0; i < len(ids); i += batch {
end := i + batch
if end > len(ids) {
end = len(ids)
}
result := s.repo.DB.WithContext(ctx).Model(&model.Media{}).
Where("id IN ? AND (availability_state IS NULL OR availability_state <> ?)", ids[i:end], "missing").
Updates(map[string]any{"availability_state": "missing", "missing_since": now})
if result.Error != nil {
return changed, result.Error
}
changed += result.RowsAffected
}
if changed > 0 {
s.invalidateMediaCache(ctx)
}
return changed, nil
}
func pathBelongsToRoot(pathValue, rootPath string) bool {
+1 -1
View File
@@ -202,7 +202,7 @@ func (s *ScannerService) scanLocalLibraryFiles(ctx context.Context, lib *model.L
func existingLocalMediaFileIDs(existingMedia map[string]existingLocalMedia) map[string]string {
seenInodes := make(map[string]string)
for path, existing := range existingMedia {
if existing.FileID != "" {
if existing.FileID != "" && existing.AvailabilityState != "missing" {
seenInodes[existing.FileID] = path
}
}
+10 -3
View File
@@ -175,7 +175,7 @@ func TestIngestPathAdoptsSoftDeletedSTRMRenameTombstone(t *testing.T) {
}
}
func TestRemovePathStillHardDeletesTrueDeletionWithoutAlias(t *testing.T) {
func TestRemovePathRetainsTrueDeletionWithoutAlias(t *testing.T) {
sc, repos := newScannerTestEnv(t)
root := t.TempDir()
lib := model.Library{Name: "Anime", Path: root, Type: "anime", Enabled: true}
@@ -197,8 +197,15 @@ func TestRemovePathStillHardDeletesTrueDeletionWithoutAlias(t *testing.T) {
if err := repos.DB.Unscoped().Model(&model.Media{}).Where("path = ?", path).Count(&count).Error; err != nil {
t.Fatal(err)
}
if count != 0 {
t.Fatalf("true deletion left %d tombstone row(s)", count)
if count != 1 {
t.Fatalf("missing media count = %d, want metadata retained", count)
}
var media model.Media
if err := repos.DB.Where("path = ?", path).First(&media).Error; err != nil {
t.Fatal(err)
}
if media.AvailabilityState != "missing" || media.MissingSince == nil {
t.Fatalf("true deletion did not record missing state: %#v", media)
}
}
+12
View File
@@ -50,6 +50,7 @@ type SchedulerService struct {
readerFilesTTLHours int
segments *MediaSegmentService
backup *BackupService
mu sync.Mutex
stopCh chan struct{}
@@ -65,6 +66,10 @@ func (s *SchedulerService) SetTaskTracker(tasks *TaskTrackerService) {
s.tasks = tasks
}
func (s *SchedulerService) SetBackupService(backup *BackupService) {
s.backup = backup
}
func (s *SchedulerService) SetOrganizePipeline(pipeline *OrganizePipelineService) {
s.organizePipeline = pipeline
}
@@ -196,6 +201,13 @@ func (s *SchedulerService) Start(ctx context.Context) {
run: s.jobTelegramExpiryWarning,
})
}
if s.backup != nil {
s.jobs = append(s.jobs, &scheduledJob{
name: "database_backup",
interval: 5 * time.Minute,
run: s.jobDatabaseBackup,
})
}
for _, j := range s.jobs {
initialDelay := 15 * time.Second
if j.name == "library_scan" || j.name == "organize_source" || j.name == "segment_prewarm" {
+47
View File
@@ -11,6 +11,53 @@ import (
"go.uber.org/zap"
)
const lastAutomaticBackupDateKey = "backup.last_automatic_date"
func (s *SchedulerService) jobDatabaseBackup(ctx context.Context) error {
if s == nil || s.backup == nil || s.repo == nil || s.repo.Setting == nil {
return nil
}
enabled, _ := s.repo.Setting.Get(ctx, "backup.auto_enabled")
if !parseBoolSetting(enabled, false) {
return nil
}
clock, _ := s.repo.Setting.Get(ctx, "backup.auto_time")
if clock == "" {
clock = "03:00"
}
parsed, err := time.Parse("15:04", clock)
if err != nil {
return nil
}
now := s.currentTime().In(time.Local)
dueAt := time.Date(now.Year(), now.Month(), now.Day(), parsed.Hour(), parsed.Minute(), 0, 0, now.Location())
if now.Before(dueAt) {
return nil
}
date := now.Format("2006-01-02")
last, _ := s.repo.Setting.Get(ctx, lastAutomaticBackupDateKey)
if last == date {
return nil
}
if _, err := s.backup.CreateAutomatic(ctx); err != nil {
return err
}
// Record the successful backup before pruning. If retention cleanup fails,
// the next scheduler tick must not create another copy for the same day.
if err := s.repo.Setting.Set(ctx, lastAutomaticBackupDateKey, date); err != nil {
return err
}
retentionValue, _ := s.repo.Setting.Get(ctx, "backup.retention")
retention, err := strconv.Atoi(strings.TrimSpace(retentionValue))
if err != nil || retention < 1 {
retention = 7
}
if err := s.backup.Prune(retention); err != nil {
return err
}
return nil
}
// jobScanLibraries re-walks every enabled library.
//
// 默认关闭:文件变更由 WatcherService 增量入库,无需周期性全量重扫。
+4
View File
@@ -63,6 +63,9 @@ func (b *serviceContainerBuilder) startRealtimeServices() {
b.c.WSHub = NewHub(b.log)
helper.Go(b.log, "ws.hub", b.c.WSHub.Run)
b.c.Tasks = NewTaskTrackerService(b.log, b.c.WSHub)
if b.repos != nil {
b.c.Tasks.SetPersistence(b.repos.DB)
}
b.c.SystemUpdate = NewSystemUpdateService(b.cfg, b.log, b.repos, b.c.Tasks, b.version)
b.c.SSEHub = NewSSEHub(b.log)
@@ -206,6 +209,7 @@ func (b *serviceContainerBuilder) initAccessAndStorageServices() {
b.c.Organizer, b.c.WSHub, b.cfg.Cache.CacheDir,
)
b.c.Scheduler.SetTaskTracker(b.c.Tasks)
b.c.Scheduler.SetBackupService(b.c.Backup)
b.c.Scheduler.SetOrganizePipeline(b.c.OrganizePipeline)
b.c.Scheduler.SetSegments(b.c.Segments)
b.c.Scheduler.SetImageCachePolicyProvider(func() ImageCachePolicy {
+82 -4
View File
@@ -2,6 +2,7 @@ package service
import (
"context"
"encoding/json"
"html"
"strings"
"sync"
@@ -9,12 +10,16 @@ import (
"github.com/google/uuid"
"go.uber.org/zap"
"gorm.io/gorm"
"github.com/truewhile/MeBox/internal/model"
)
const (
TaskStatusRunning = "running"
TaskStatusCompleted = "completed"
TaskStatusFailed = "failed"
TaskStatusRunning = "running"
TaskStatusCompleted = "completed"
TaskStatusFailed = "failed"
TaskStatusInterrupted = "interrupted"
TaskKindOrganize = "organize"
TaskKindScan = "scan"
@@ -70,6 +75,26 @@ type TaskTrackerService struct {
recent []BackgroundTask
maxRecent int
now func() time.Time
db *gorm.DB
}
// SetPersistence enables durable task history and marks tasks left running by
// a previous process as interrupted. Actual task execution is never resumed
// automatically; this record is an operator-visible recovery point.
func (t *TaskTrackerService) SetPersistence(db *gorm.DB) {
if t == nil || db == nil {
return
}
t.mu.Lock()
t.db = db
now := t.currentTime()
t.mu.Unlock()
err := db.Model(&model.BackgroundTaskRecord{}).
Where("status = ?", TaskStatusRunning).
Updates(map[string]any{"status": TaskStatusInterrupted, "error": "服务在任务完成前重启;请检查文件状态后重新执行", "updated_at": now, "finished_at": now}).Error
if err != nil && t.log != nil {
t.log.Warn("recover interrupted background tasks failed", zap.Error(err))
}
}
type TaskHandle struct {
@@ -105,9 +130,11 @@ func (t *TaskTrackerService) Start(kind, name string, update TaskUpdate) *TaskHa
StartedAt: now,
UpdatedAt: now,
}
applyTaskUpdate(task, update)
t.mu.Lock()
t.active[task.ID] = task
snapshot := cloneBackgroundTask(*task)
t.persist(snapshot)
t.mu.Unlock()
t.publish(snapshot)
return &TaskHandle{tracker: t, id: task.ID}
@@ -132,18 +159,47 @@ func (t *TaskTrackerService) Snapshot() TaskSnapshot {
return TaskSnapshot{}
}
t.mu.Lock()
defer t.mu.Unlock()
active := make([]BackgroundTask, 0, len(t.active))
for _, task := range t.active {
active = append(active, cloneBackgroundTask(*task))
}
recent := make([]BackgroundTask, 0, len(t.recent))
db := t.db
limit := t.maxRecent
if limit < 1 {
limit = 30
}
for _, task := range t.recent {
recent = append(recent, cloneBackgroundTask(task))
}
t.mu.Unlock()
if db != nil {
var rows []model.BackgroundTaskRecord
if err := db.Where("status <> ?", TaskStatusRunning).Order("started_at DESC").Limit(limit).Find(&rows).Error; err == nil {
recent = make([]BackgroundTask, 0, len(rows))
for _, row := range rows {
recent = append(recent, backgroundTaskFromRecord(row))
}
} else if t.log != nil {
t.log.Warn("load background task history failed; using memory history", zap.Error(err))
}
}
return TaskSnapshot{Active: active, Recent: recent}
}
func backgroundTaskFromRecord(row model.BackgroundTaskRecord) BackgroundTask {
task := BackgroundTask{
ID: row.ID, Kind: row.Kind, Name: row.Name, Status: row.Status,
Stage: row.Stage, SourcePath: row.SourcePath, DestPath: row.DestPath,
Message: row.Message, Error: row.Error, StartedAt: row.StartedAt,
UpdatedAt: row.UpdatedAt, FinishedAt: row.FinishedAt,
}
_ = json.Unmarshal([]byte(row.Details), &task.Details)
_ = json.Unmarshal([]byte(row.Metrics), &task.Metrics)
_ = json.Unmarshal([]byte(row.Items), &task.Items)
return task
}
func (t *TaskTrackerService) update(id string, update TaskUpdate) {
now := t.currentTime()
t.mu.Lock()
@@ -155,6 +211,7 @@ func (t *TaskTrackerService) update(id string, update TaskUpdate) {
applyTaskUpdate(task, update)
task.UpdatedAt = now
snapshot := cloneBackgroundTask(*task)
t.persist(snapshot)
t.mu.Unlock()
t.publish(snapshot)
}
@@ -186,6 +243,7 @@ func (t *TaskTrackerService) finish(id string, err error, update TaskUpdate) {
t.recent = t.recent[:t.maxRecent]
}
notifier := t.failureNotifier
t.persist(snapshot)
t.mu.Unlock()
t.publish(snapshot)
@@ -195,6 +253,26 @@ func (t *TaskTrackerService) finish(id string, err error, update TaskUpdate) {
}
}
func (t *TaskTrackerService) persist(task BackgroundTask) {
if t == nil || t.db == nil {
return
}
encode := func(value any) string {
data, _ := json.Marshal(value)
return string(data)
}
row := model.BackgroundTaskRecord{
ID: task.ID, Kind: task.Kind, Name: task.Name, Status: task.Status,
Stage: task.Stage, SourcePath: task.SourcePath, DestPath: task.DestPath,
Message: task.Message, Error: task.Error, Details: encode(task.Details),
Metrics: encode(task.Metrics), Items: encode(task.Items), StartedAt: task.StartedAt,
UpdatedAt: task.UpdatedAt, FinishedAt: task.FinishedAt,
}
if err := t.db.Save(&row).Error; err != nil && t.log != nil {
t.log.Warn("persist background task failed", zap.String("task_id", task.ID), zap.Error(err))
}
}
// SetFailureNotifier 注入任务失败时的管理员通知回调。
func (t *TaskTrackerService) SetFailureNotifier(fn func(ctx context.Context, text string)) {
if t == nil {
@@ -0,0 +1,78 @@
package service
import (
"fmt"
"sync"
"testing"
"go.uber.org/zap"
"github.com/truewhile/MeBox/internal/model"
)
func TestTaskTrackerPersistsHistoryAcrossRestart(t *testing.T) {
db := newServiceTestDB(t)
tracker := NewTaskTrackerService(zap.NewNop(), nil)
tracker.SetPersistence(db)
task := tracker.Start(TaskKindScan, "scan", TaskUpdate{Details: []string{"initial detail"}})
task.Update(TaskUpdate{Metrics: map[string]int64{"visited": 42}})
task.Finish(nil, TaskUpdate{Message: "done"})
restarted := NewTaskTrackerService(zap.NewNop(), nil)
restarted.SetPersistence(db)
snapshot := restarted.Snapshot()
if len(snapshot.Active) != 0 || len(snapshot.Recent) != 1 {
t.Fatalf("unexpected history: %#v", snapshot)
}
recent := snapshot.Recent[0]
if recent.Status != TaskStatusCompleted || recent.Metrics["visited"] != 42 || len(recent.Details) != 1 || recent.FinishedAt == nil {
t.Fatalf("task history lost fields: %#v", recent)
}
}
func TestTaskTrackerConcurrentUpdatesDoNotRegressFinishedTasks(t *testing.T) {
db := newServiceTestDB(t)
sqlDB, err := db.DB()
if err != nil {
t.Fatal(err)
}
sqlDB.SetMaxOpenConns(1)
tracker := NewTaskTrackerService(zap.NewNop(), nil)
tracker.SetPersistence(db)
var workers sync.WaitGroup
for i := 0; i < 12; i++ {
workers.Add(1)
go func(index int) {
defer workers.Done()
task := tracker.Start(TaskKindScan, fmt.Sprintf("scan-%d", index), TaskUpdate{})
task.Update(TaskUpdate{Metrics: map[string]int64{"visited": 1}})
task.Finish(nil, TaskUpdate{})
task.Update(TaskUpdate{Message: "late update"})
_ = tracker.Snapshot()
}(i)
}
workers.Wait()
var completed int64
if err := db.Model(&model.BackgroundTaskRecord{}).Where("status = ?", TaskStatusCompleted).Count(&completed).Error; err != nil || completed != 12 {
t.Fatalf("finished task status regressed: completed=%d err=%v", completed, err)
}
if snapshot := tracker.Snapshot(); len(snapshot.Active) != 0 || len(snapshot.Recent) != 12 {
t.Fatalf("inconsistent task snapshot: %#v", snapshot)
}
}
func TestTaskTrackerMarksAbandonedTasksInterrupted(t *testing.T) {
db := newServiceTestDB(t)
tracker := NewTaskTrackerService(zap.NewNop(), nil)
tracker.SetPersistence(db)
tracker.Start(TaskKindOrganize, "unfinished", TaskUpdate{})
restarted := NewTaskTrackerService(zap.NewNop(), nil)
restarted.SetPersistence(db)
snapshot := restarted.Snapshot()
if len(snapshot.Recent) != 1 || snapshot.Recent[0].Status != TaskStatusInterrupted || snapshot.Recent[0].FinishedAt == nil {
t.Fatalf("abandoned task not marked interrupted: %#v", snapshot)
}
var running int64
if err := db.Model(&model.BackgroundTaskRecord{}).Where("status = ?", TaskStatusRunning).Count(&running).Error; err != nil || running != 0 {
t.Fatalf("stale running tasks: %d err=%v", running, err)
}
}