mirror of
https://github.com/truewhile/MeBox.git
synced 2026-11-15 20:34:23 +08:00
Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 6186c8b848 | |||
| 0db8698fce |
@@ -54,6 +54,7 @@ func setDefaults(v *viper.Viper) {
|
||||
v.SetDefault("cache.emby_latest_ttl_seconds", 300)
|
||||
v.SetDefault("cache.reader_content_max_size_mb", 1024)
|
||||
v.SetDefault("cache.reader_content_ttl_hours", 168)
|
||||
v.SetDefault("cache.reader_files_ttl_hours", 720)
|
||||
|
||||
v.SetDefault("search.backend", "")
|
||||
v.SetDefault("search.opensearch_url", "")
|
||||
|
||||
@@ -68,6 +68,9 @@ func (c *Config) normalize() error {
|
||||
if c.Cache.ReaderContentTTLHours < 0 {
|
||||
c.Cache.ReaderContentTTLHours = 0
|
||||
}
|
||||
if c.Cache.ReaderFilesTTLHours < 0 {
|
||||
c.Cache.ReaderFilesTTLHours = 0
|
||||
}
|
||||
c.Search.Backend = strings.ToLower(strings.TrimSpace(c.Search.Backend))
|
||||
if c.Search.Index == "" {
|
||||
c.Search.Index = "mebox_media"
|
||||
|
||||
@@ -137,6 +137,9 @@ type CacheConfig struct {
|
||||
ReaderContentMaxSizeMB int `mapstructure:"reader_content_max_size_mb"`
|
||||
// ReaderContentTTLHours 阅读正文缓存的保留时长(小时),0 表示不过期。
|
||||
ReaderContentTTLHours int `mapstructure:"reader_content_ttl_hours"`
|
||||
// ReaderFilesTTLHours 书源文件缓存(java.cacheFile / downloadFile)的保留时长(小时)。
|
||||
// 0 表示使用内置兜底值(30 天)。
|
||||
ReaderFilesTTLHours int `mapstructure:"reader_files_ttl_hours"`
|
||||
}
|
||||
|
||||
type SearchConfig struct {
|
||||
|
||||
@@ -115,7 +115,12 @@ func (r *MediaRepository) upsertWithDB(ctx context.Context, db *gorm.DB, m *mode
|
||||
return m.ID, nil
|
||||
}
|
||||
|
||||
updates := mediaUpsertUpdates(existing, *m)
|
||||
// 孤儿行回收:原库已被删除时(DeleteLibrary 硬删库行;扫描与删除并发时扫描也会
|
||||
// 把行写回已删库),这条行会钉死在不存在的 library_id 上。media.path 是全局唯一
|
||||
// 索引,重扫既插不进同路径新行,又因 LibraryID 非空而拒绝迁移归属,于是新库/重建库
|
||||
// 永远查不到这些条目(表现为"扫描明明有文件,库却是空的")。这里显式识别并允许迁移。
|
||||
reclaimLibrary := r.mediaRowLibraryIsOrphaned(ctx, db, existing.LibraryID, m.LibraryID, m.Path)
|
||||
updates := mediaUpsertUpdates(existing, *m, reclaimLibrary)
|
||||
if adopted {
|
||||
updates["path"] = m.Path
|
||||
if existing.DeletedAt.Valid {
|
||||
@@ -139,6 +144,42 @@ func (r *MediaRepository) upsertWithDB(ctx context.Context, db *gorm.DB, m *mode
|
||||
return existing.ID, nil
|
||||
}
|
||||
|
||||
// mediaRowLibraryIsOrphaned reports whether a media row is still pinned to a
|
||||
// library that no longer exists.
|
||||
//
|
||||
// 删除媒体库会硬删库行(见 MediaService.DeleteLibrary),但扫描与删除并发时,
|
||||
// 已在途的扫描仍可能把这批文件写回已删库的 library_id;历史数据里也可能留下
|
||||
// 指向早已消失的库的行。因为 media.path 是全局唯一索引,这些行一旦存在:
|
||||
// 1. 重扫无法插入同路径的新行(唯一约束冲突);
|
||||
// 2. addMediaPlacementUpdates 又因为 existing.LibraryID 非空而拒绝迁移归属。
|
||||
//
|
||||
// 结果就是新库 / 重建的同名库永远查不到这些条目,扫描报告有文件而库显示为空。
|
||||
// 这里把"原库已不存在"这种情况单独判定出来,交给扫描把归属迁移到当前库。
|
||||
func (r *MediaRepository) mediaRowLibraryIsOrphaned(ctx context.Context, db *gorm.DB, existingLibraryID, incomingLibraryID, incomingPath string) bool {
|
||||
existingLibraryID = strings.TrimSpace(existingLibraryID)
|
||||
incomingLibraryID = strings.TrimSpace(incomingLibraryID)
|
||||
// 归属未变或原行为空(后者走既有的回填分支)时无需查库。
|
||||
if existingLibraryID == "" || existingLibraryID == incomingLibraryID {
|
||||
return false
|
||||
}
|
||||
// 云盘媒体的归属迁移由 addMediaPlacementUpdates 无条件处理,无需再查库。
|
||||
if strings.HasPrefix(strings.ToLower(strings.TrimSpace(incomingPath)), "cloud://") {
|
||||
return false
|
||||
}
|
||||
if db == nil || !db.Migrator().HasTable(&model.Library{}) {
|
||||
return false
|
||||
}
|
||||
var count int64
|
||||
// 刻意不加 Unscoped:软删除的库同样不再出现在库里,挂在它下面的媒体行
|
||||
// 也属于"查不到"的孤儿,一并允许回收。查询失败时保守处理:宁可保留旧
|
||||
// 归属,也不要把仍有效的库误判成孤儿。
|
||||
if err := db.WithContext(ctx).Model(&model.Library{}).
|
||||
Where("id = ?", existingLibraryID).Count(&count).Error; err != nil {
|
||||
return false
|
||||
}
|
||||
return count == 0
|
||||
}
|
||||
|
||||
func (r *MediaRepository) findOrCreateMediaByPath(ctx context.Context, db *gorm.DB, m *model.Media, aliasPaths []string) (model.Media, bool, bool, error) {
|
||||
var existing model.Media
|
||||
err := db.WithContext(ctx).Unscoped().Where("path = ?", m.Path).First(&existing).Error
|
||||
@@ -194,14 +235,14 @@ func (r *MediaRepository) findMediaByAlias(ctx context.Context, db *gorm.DB, cur
|
||||
return existing, err
|
||||
}
|
||||
|
||||
func mediaUpsertUpdates(existing, incoming model.Media) map[string]any {
|
||||
func mediaUpsertUpdates(existing, incoming model.Media, reclaimLibrary bool) map[string]any {
|
||||
updates := map[string]any{}
|
||||
addMediaFileScanUpdates(updates, existing, incoming)
|
||||
addMediaTitleUpdates(updates, existing, incoming)
|
||||
addMediaExternalIDUpdates(updates, existing, incoming)
|
||||
addMatchedMediaMetadataUpdates(updates, existing, incoming)
|
||||
addMediaArtworkUpdates(updates, existing, incoming)
|
||||
addMediaPlacementUpdates(updates, existing, incoming)
|
||||
addMediaPlacementUpdates(updates, existing, incoming, reclaimLibrary)
|
||||
addMediaSTRMUpdate(updates, existing, incoming)
|
||||
return updates
|
||||
}
|
||||
@@ -308,13 +349,15 @@ func addMediaArtworkUpdates(updates map[string]any, existing, incoming model.Med
|
||||
}
|
||||
}
|
||||
|
||||
func addMediaPlacementUpdates(updates map[string]any, existing, incoming model.Media) {
|
||||
func addMediaPlacementUpdates(updates map[string]any, existing, incoming model.Media, reclaimLibrary bool) {
|
||||
// 云盘媒体:同一 cloud:// 文件可能先被父目录库扫描入库,之后用户按二级
|
||||
// 分类重新挂载/扫描到更精确的分类库。此时让 library_id 迁移到当前扫描库,
|
||||
// 否则媒体被钉死在旧库、新分类库里看不到(表现为"媒体部分消失")。
|
||||
// 本地媒体物理位置固定:仅在原 library_id 为空时回填,不迁移。
|
||||
// 本地媒体物理位置固定:仅在原 library_id 为空、或原库已不存在(孤儿行,
|
||||
// reclaimLibrary)时才回填/迁移,既避免把仍属于其他库的本地媒体抢走,
|
||||
// 又能让扫描回收被删除库遗留的行。
|
||||
if isCloudMediaPath := strings.HasPrefix(strings.ToLower(strings.TrimSpace(incoming.Path)), "cloud://"); incoming.LibraryID != "" && incoming.LibraryID != existing.LibraryID {
|
||||
if isCloudMediaPath || existing.LibraryID == "" {
|
||||
if isCloudMediaPath || existing.LibraryID == "" || reclaimLibrary {
|
||||
updates["library_id"] = incoming.LibraryID
|
||||
}
|
||||
}
|
||||
|
||||
@@ -39,7 +39,9 @@ const (
|
||||
// audioTranscodeDirName 缓存目录名,位于 cache.cache_dir 之下。
|
||||
audioTranscodeDirName = "reader-audio"
|
||||
// maxAudioTranscodeCacheBytes 转码缓存上限,超出按修改时间淘汰最旧的。
|
||||
maxAudioTranscodeCacheBytes = 4 << 30
|
||||
// 1GB 是权衡:单章语音 96kbps 约 1MB/分钟,一本 10 小时的有声书约 600MB,
|
||||
// 也就是说同时能保住的「最近听过的书」在个位数;超出后旧书再听会重新转码。
|
||||
maxAudioTranscodeCacheBytes = 1 << 30
|
||||
// audioTranscodeKeepRatio 触发淘汰后回落到上限的比例,避免每次写入都淘汰。
|
||||
audioTranscodeKeepRatio = 0.9
|
||||
// maxAudioTranscodeDuration 单章转码超时。
|
||||
|
||||
@@ -2,6 +2,7 @@ package reader
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -255,3 +256,50 @@ func indexOf(items []string, want string) int {
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// 转码缓存上限是 1GB:单章语音约 1MB/分钟(96kbps),
|
||||
// 也就是一本 10 小时的有声书约 600MB,同时能保住「最近听过的书」在个位数。
|
||||
// 这里把常量与淘汰后的落点一起钉住,防止以后被无意改回大值。
|
||||
func TestAudioTranscodeCacheLimitIsOneGiB(t *testing.T) {
|
||||
if maxAudioTranscodeCacheBytes != int64(1)<<30 {
|
||||
t.Fatalf("转码缓存上限应为 1GB,实际 %d 字节", maxAudioTranscodeCacheBytes)
|
||||
}
|
||||
// 淘汰目标是上限的 90%:模拟缓存略超上限时,删掉最旧的若干章回到目标以下。
|
||||
dir := t.TempDir()
|
||||
limit := int64(1000)
|
||||
perFile := int64(120)
|
||||
for i := 0; i < 9; i++ { // 1080 > 1000,超限
|
||||
name := filepath.Join(dir, fmt.Sprintf("ch%02d.mp3", i))
|
||||
if err := os.WriteFile(name, make([]byte, perFile), 0o640); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ts := time.Now().Add(-time.Duration(9-i) * time.Minute)
|
||||
if err := os.Chtimes(name, ts, ts); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
pruneAudioTranscodeCache(dir, limit)
|
||||
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var remain int64
|
||||
for _, e := range entries {
|
||||
info, err := e.Info()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
remain += info.Size()
|
||||
}
|
||||
if remain > limit {
|
||||
t.Fatalf("淘汰后应落回上限内: remain=%d limit=%d", remain, limit)
|
||||
}
|
||||
// 最新的那章必须还在(LRU 先删最旧的)。
|
||||
if _, err := os.Stat(filepath.Join(dir, "ch08.mp3")); err != nil {
|
||||
t.Fatalf("最新一章应保留: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "ch00.mp3")); !os.IsNotExist(err) {
|
||||
t.Fatalf("最旧一章应被删除,err=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
)
|
||||
@@ -402,3 +403,58 @@ func TestContentCacheWithoutCacheDir(t *testing.T) {
|
||||
t.Fatalf("正文异常: %q", out.Content)
|
||||
}
|
||||
}
|
||||
|
||||
// 容量与过期淘汰:总量超配额时按 LRU 删到 90%,过期条目优先清理。
|
||||
// 这是「长期运行不会撑满磁盘」的保障,覆盖 pruneReaderContentCache 两条分支。
|
||||
func TestContentCachePruneByTTLAndQuota(t *testing.T) {
|
||||
svc, srv, book := prepareCacheTestBook(t)
|
||||
ctx := t.Context()
|
||||
src := mustSource(t, svc, srv.URL)
|
||||
|
||||
// 造 4 条缓存,其中 1 条已过期。
|
||||
for i := 1; i <= 4; i++ {
|
||||
ch := model.ReaderChapter{
|
||||
Index: i - 1, Title: fmt.Sprintf("第 %d 章", i),
|
||||
URL: fmt.Sprintf("%s/book/1/c%d.html", srv.URL, i),
|
||||
}
|
||||
svc.saveCachedContent(ctx, src, book, ch, &ChapterContent{
|
||||
Type: "text", Content: strings.Repeat("正", 200),
|
||||
})
|
||||
}
|
||||
rows, err := svc.repo.ListContentCacheByBook(ctx, contentBookKey(src.ID, book.BookURL))
|
||||
if err != nil || len(rows) != 4 {
|
||||
t.Fatalf("应有 4 条缓存: n=%d err=%v", len(rows), err)
|
||||
}
|
||||
// 把第 1 条标记为已过期。
|
||||
expired := rows[0]
|
||||
expired.ExpiresAt = time.Now().Add(-time.Hour).Unix()
|
||||
if err := svc.repo.UpsertContentCache(ctx, &expired); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
svc.PruneContentCache(ctx)
|
||||
rows, err = svc.repo.ListContentCacheByBook(ctx, contentBookKey(src.ID, book.BookURL))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, row := range rows {
|
||||
if row.ChapterKey == expired.ChapterKey {
|
||||
t.Fatal("过期条目应被清理")
|
||||
}
|
||||
}
|
||||
|
||||
// 容量配额:把上限压到远小于现有体积,触发 LRU 淘汰。
|
||||
var total int64
|
||||
for _, row := range rows {
|
||||
total += row.SizeBytes
|
||||
}
|
||||
if total <= 0 {
|
||||
t.Fatal("缓存体积应大于 0")
|
||||
}
|
||||
svc.cfg.Cache.ReaderContentMaxSizeMB = 0 // 0 = 不限,先确认不误删
|
||||
svc.PruneContentCache(ctx)
|
||||
after, _ := svc.repo.ListContentCacheByBook(ctx, contentBookKey(src.ID, book.BookURL))
|
||||
if len(after) != len(rows) {
|
||||
t.Fatalf("未超配额不应淘汰: before=%d after=%d", len(rows), len(after))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// 长期运行时各类缓存必须有上限或清理,否则会慢慢吃满磁盘。
|
||||
// 这里覆盖新增的「书源文件缓存 + 孤儿临时文件」清理任务。
|
||||
|
||||
func writeAgedFile(t *testing.T, path string, age time.Duration) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte("x"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stamp := time.Now().Add(-age)
|
||||
if err := os.Chtimes(path, stamp, stamp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobCleanReaderFilesPrunesByRetention(t *testing.T) {
|
||||
cacheDir := t.TempDir()
|
||||
s := &SchedulerService{cacheDir: cacheDir, readerFilesTTLHours: 24}
|
||||
|
||||
// 超龄的书源文件(字体/静态 JS)应被删掉;新文件保留。
|
||||
oldFile := filepath.Join(cacheDir, "reader", "files", "abc.woff")
|
||||
newFile := filepath.Join(cacheDir, "reader", "files", "nested", "fresh.js")
|
||||
writeAgedFile(t, oldFile, 48*time.Hour)
|
||||
writeAgedFile(t, newFile, time.Hour)
|
||||
|
||||
if err := s.jobCleanReaderFiles(context.Background()); err != nil {
|
||||
t.Fatalf("清理失败: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(oldFile); !os.IsNotExist(err) {
|
||||
t.Fatalf("超龄书源文件应被删除,err=%v", err)
|
||||
}
|
||||
if _, err := os.Stat(newFile); err != nil {
|
||||
t.Fatalf("未超龄文件不应被删除: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobCleanReaderFilesRemovesOrphanTempFiles(t *testing.T) {
|
||||
cacheDir := t.TempDir()
|
||||
s := &SchedulerService{cacheDir: cacheDir, readerFilesTTLHours: 24}
|
||||
ctx := context.Background()
|
||||
|
||||
// 中断的转码产物 .mp3.part 与正文缓存的 .content-* 临时文件。
|
||||
oldPart := filepath.Join(cacheDir, "reader-audio", "deadbeef.mp3.part")
|
||||
freshPart := filepath.Join(cacheDir, "reader-audio", "running.mp3.part")
|
||||
oldContentTmp := filepath.Join(cacheDir, "reader-content", "aa", "bb", ".content-123")
|
||||
// 成品与正常缓存文件不能被误删。
|
||||
finished := filepath.Join(cacheDir, "reader-audio", "done.mp3")
|
||||
contentJSON := filepath.Join(cacheDir, "reader-content", "aa", "bb", "chapter.json")
|
||||
|
||||
writeAgedFile(t, oldPart, 48*time.Hour)
|
||||
writeAgedFile(t, freshPart, time.Minute)
|
||||
writeAgedFile(t, oldContentTmp, 48*time.Hour)
|
||||
writeAgedFile(t, finished, 48*time.Hour)
|
||||
writeAgedFile(t, contentJSON, 48*time.Hour)
|
||||
|
||||
if err := s.jobCleanReaderFiles(ctx); err != nil {
|
||||
t.Fatalf("清理失败: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(oldPart); !os.IsNotExist(err) {
|
||||
t.Fatalf("孤儿 .part 应被删除,err=%v", err)
|
||||
}
|
||||
if _, err := os.Stat(oldContentTmp); !os.IsNotExist(err) {
|
||||
t.Fatalf("孤儿 .content-* 应被删除,err=%v", err)
|
||||
}
|
||||
if _, err := os.Stat(freshPart); err != nil {
|
||||
t.Fatalf("正在写入的 .part 不应被删除: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(finished); err != nil {
|
||||
t.Fatalf("转码成品不应被删除: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(contentJSON); err != nil {
|
||||
t.Fatalf("正文缓存文件不应被删除: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 未配置 TTL 时用兜底保留时长,而不是「不清理」。
|
||||
func TestJobCleanReaderFilesFallsBackToDefaultRetention(t *testing.T) {
|
||||
cacheDir := t.TempDir()
|
||||
s := &SchedulerService{cacheDir: cacheDir} // readerFilesTTLHours = 0
|
||||
|
||||
// 40 天前的文件应被兜底规则(30 天)清掉。
|
||||
veryOld := filepath.Join(cacheDir, "reader", "files", "ancient.js")
|
||||
writeAgedFile(t, veryOld, 40*24*time.Hour)
|
||||
|
||||
if err := s.jobCleanReaderFiles(context.Background()); err != nil {
|
||||
t.Fatalf("清理失败: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(veryOld); !os.IsNotExist(err) {
|
||||
t.Fatalf("未配置 TTL 也应有兜底清理,err=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 目录不存在时不应报错(缓存未启用或尚未产生文件)。
|
||||
func TestJobCleanReaderFilesMissingDirs(t *testing.T) {
|
||||
s := &SchedulerService{cacheDir: filepath.Join(t.TempDir(), "nope")}
|
||||
if err := s.jobCleanReaderFiles(context.Background()); err != nil {
|
||||
t.Fatalf("目录不存在应静默通过: %v", err)
|
||||
}
|
||||
// cacheDir 为空(未配置)同样直接返回。
|
||||
empty := &SchedulerService{}
|
||||
if err := empty.jobCleanReaderFiles(context.Background()); err != nil {
|
||||
t.Fatalf("未配置缓存目录应静默通过: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -282,6 +282,106 @@ func TestScanLibraryUpdatesExistingPathFromOverlappingLibrary(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestScanLibraryReclaimsMediaOrphanedByDeletedLibrary covers the "scan reports
|
||||
// files but the library stays empty" failure: a row left behind by a deleted
|
||||
// library keeps its stale library_id, and because media.path carries a global
|
||||
// unique index the rescan can neither insert a fresh row nor rewrite the
|
||||
// ownership, so a rebuilt library stays empty forever. The scan must reclaim
|
||||
// such orphaned rows for the library being scanned.
|
||||
func TestScanLibraryReclaimsMediaOrphanedByDeletedLibrary(t *testing.T) {
|
||||
sc, repos := newScannerTestEnv(t)
|
||||
root := t.TempDir()
|
||||
file := filepath.Join(root, "Orphaned Episode 1.mkv")
|
||||
if err := os.WriteFile(file, []byte("payload"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
oldLib := model.Library{Name: "Old", Path: root, Type: "movie", Enabled: true}
|
||||
if err := repos.Library.Create(t.Context(), &oldLib); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := sc.ScanLibrary(t.Context(), oldLib.ID); err != nil {
|
||||
t.Fatalf("first scan: %v", err)
|
||||
}
|
||||
|
||||
// Reproduce the gap the scan cannot see: the library row is gone while its
|
||||
// media rows survive (hard-deleted library, or an in-flight scan writing
|
||||
// after DeleteLibrary).
|
||||
if err := repos.DB.Unscoped().Delete(&model.Library{}, "id = ?", oldLib.ID).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := countMediaInLibrary(t, repos, oldLib.ID); got != 1 {
|
||||
t.Fatalf("precondition: rows still pinned to deleted library = %d, want 1", got)
|
||||
}
|
||||
|
||||
newLib := model.Library{Name: "New", Path: root, Type: "movie", Enabled: true}
|
||||
if err := repos.Library.Create(t.Context(), &newLib); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
res, err := sc.ScanLibrary(t.Context(), newLib.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("rescan: %v", err)
|
||||
}
|
||||
if res.ErrorCount != 0 {
|
||||
t.Fatalf("rescan errors = %#v, want none", res.Errors)
|
||||
}
|
||||
if got := countMedia(t, repos); got != 1 {
|
||||
t.Fatalf("media count = %d, want 1 (orphan reclaimed, not duplicated)", got)
|
||||
}
|
||||
if got := countMediaInLibrary(t, repos, newLib.ID); got != 1 {
|
||||
t.Fatalf("rebuilt library media count = %d, want 1", got)
|
||||
}
|
||||
var media model.Media
|
||||
if err := repos.DB.First(&media, "path = ?", file).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if media.LibraryID != newLib.ID {
|
||||
t.Fatalf("media library_id = %q, want reclaimed to %q", media.LibraryID, newLib.ID)
|
||||
}
|
||||
}
|
||||
|
||||
// TestScanLibraryKeepsOwnershipWhenOwningLibraryStillExists pins the other half
|
||||
// of the rule: local media stays with its existing library as long as that
|
||||
// library is real, so overlapping libraries do not steal each other's rows.
|
||||
func TestScanLibraryKeepsOwnershipWhenOwningLibraryStillExists(t *testing.T) {
|
||||
sc, repos := newScannerTestEnv(t)
|
||||
root := t.TempDir()
|
||||
file := filepath.Join(root, "Shared (2026).mkv")
|
||||
if err := os.WriteFile(file, []byte("payload"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ownerLib := model.Library{Name: "Owner", Path: root, Type: "movie", Enabled: true}
|
||||
otherLib := model.Library{Name: "Other", Path: root, Type: "movie", Enabled: true}
|
||||
if err := repos.Library.Create(t.Context(), &ownerLib); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := repos.Library.Create(t.Context(), &otherLib); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := sc.ScanLibrary(t.Context(), ownerLib.ID); err != nil {
|
||||
t.Fatalf("owner scan: %v", err)
|
||||
}
|
||||
if _, err := sc.ScanLibrary(t.Context(), otherLib.ID); err != nil {
|
||||
t.Fatalf("other scan: %v", err)
|
||||
}
|
||||
var media model.Media
|
||||
if err := repos.DB.First(&media, "path = ?", file).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if media.LibraryID != ownerLib.ID {
|
||||
t.Fatalf("media library_id = %q, want unchanged owner %q", media.LibraryID, ownerLib.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func countMediaInLibrary(t *testing.T, repos *repository.Container, libraryID string) int64 {
|
||||
t.Helper()
|
||||
var n int64
|
||||
if err := repos.DB.Model(&model.Media{}).Where("library_id = ?", libraryID).Count(&n).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func TestScanLibrarySkipsUnchangedLocalMetadata(t *testing.T) {
|
||||
sc, repos := newScannerTestEnv(t)
|
||||
root := t.TempDir()
|
||||
|
||||
@@ -45,6 +45,9 @@ type SchedulerService struct {
|
||||
|
||||
// readerContentCleaner 阅读正文缓存的清理钩子(由阅读模块注入)。
|
||||
readerContentCleaner func(context.Context)
|
||||
// readerFilesTTLHours 书源文件缓存(java.cacheFile / downloadFile)的保留小时数。
|
||||
// 0 表示用内置兜底值(见 readerFilesRetentionMax)。
|
||||
readerFilesTTLHours int
|
||||
|
||||
segments *MediaSegmentService
|
||||
|
||||
@@ -96,6 +99,11 @@ func (s *SchedulerService) SetReaderContentCleaner(fn func(context.Context)) {
|
||||
s.readerContentCleaner = fn
|
||||
}
|
||||
|
||||
// SetReaderFilesTTL 配置书源文件缓存的保留小时数(0 表示用兜底值)。
|
||||
func (s *SchedulerService) SetReaderFilesTTL(hours int) {
|
||||
s.readerFilesTTLHours = hours
|
||||
}
|
||||
|
||||
// scheduledJob is one recurring task.
|
||||
type scheduledJob struct {
|
||||
name string
|
||||
@@ -165,6 +173,12 @@ func (s *SchedulerService) Start(ctx context.Context) {
|
||||
interval: 1 * time.Hour,
|
||||
run: s.jobCleanReaderContentCache,
|
||||
})
|
||||
// 书源文件缓存与孤儿临时文件的清理和正文缓存同源,一起注册。
|
||||
s.jobs = append(s.jobs, &scheduledJob{
|
||||
name: "reader_files_cleanup",
|
||||
interval: 24 * time.Hour,
|
||||
run: s.jobCleanReaderFiles,
|
||||
})
|
||||
}
|
||||
// 片头预热只在注入了 Segments 时注册,避免测试跑无转外网任务。
|
||||
if s.segments != nil {
|
||||
|
||||
@@ -2,6 +2,7 @@ package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -186,6 +187,62 @@ func (s *SchedulerService) jobCleanReaderContentCache(ctx context.Context) error
|
||||
return nil
|
||||
}
|
||||
|
||||
// readerFilesRetentionMax 书源文件缓存(java.cacheFile / downloadFile)的保底保留时长。
|
||||
// 未配置 TTL 时用它兜底:字体、静态 JS 这类文件会被书源长期复用,不能删太早。
|
||||
const readerFilesRetentionMax = 30 * 24 * time.Hour
|
||||
|
||||
// readerTempOrphanAge 临时产物的清理阈值:正常写入是「临时文件 + 立刻 rename」,
|
||||
// 存活超过这个时长的只可能是进程被杀/断电留下的孤儿(正在写的文件 mtime 始终在刷新)。
|
||||
const readerTempOrphanAge = 24 * time.Hour
|
||||
|
||||
// jobCleanReaderFiles 清理阅读相关缓存里没有其它机制管的部分:
|
||||
// - cache/reader/files:书源 java.cacheFile / java.downloadFile 落盘的文件
|
||||
// (字体、静态 JS 等),按保留时长淘汰——此前完全没有清理,会无限增长;
|
||||
// - 孤儿临时文件:转码的 *.mp3.part 与正文缓存的 .content-*,写入中断后会残留。
|
||||
func (s *SchedulerService) jobCleanReaderFiles(ctx context.Context) error {
|
||||
if s.cacheDir == "" {
|
||||
return nil
|
||||
}
|
||||
retention := s.readerFilesTTLHours
|
||||
if retention <= 0 {
|
||||
retention = int(readerFilesRetentionMax / time.Hour)
|
||||
}
|
||||
if err := walkAndPrune(filepath.Join(s.cacheDir, "reader", "files"),
|
||||
time.Now().Add(-time.Duration(retention)*time.Hour)); err != nil {
|
||||
return err
|
||||
}
|
||||
// 转码缓存的 tmp 是 <目标>.mp3.part,与成品同目录。
|
||||
if err := pruneOrphanTempFiles(filepath.Join(s.cacheDir, "reader-audio")); err != nil {
|
||||
return err
|
||||
}
|
||||
// 正文缓存的临时文件有固定前缀。
|
||||
return pruneOrphanTempFiles(filepath.Join(s.cacheDir, "reader-content"))
|
||||
}
|
||||
|
||||
// pruneOrphanTempFiles 删除缓存目录里超龄的临时文件(*.part / .content-*)。
|
||||
func pruneOrphanTempFiles(root string) error {
|
||||
if root == "" {
|
||||
return nil
|
||||
}
|
||||
if _, err := os.Stat(root); err != nil {
|
||||
return nil
|
||||
}
|
||||
cutoff := time.Now().Add(-readerTempOrphanAge)
|
||||
return filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil || info.IsDir() {
|
||||
return nil
|
||||
}
|
||||
name := info.Name()
|
||||
if !strings.HasSuffix(name, ".part") && !strings.HasPrefix(name, ".content-") {
|
||||
return nil
|
||||
}
|
||||
if info.ModTime().Before(cutoff) {
|
||||
_ = os.Remove(path) // #nosec G122 -- 缓存目录内的孤儿临时文件,尽力清理
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// jobCleanTranscodeCache deletes HLS artefacts older than 24h.
|
||||
func (s *SchedulerService) jobCleanTranscodeCache(ctx context.Context) error {
|
||||
if s.cacheDir == "" {
|
||||
|
||||
@@ -214,6 +214,9 @@ func (b *serviceContainerBuilder) initAccessAndStorageServices() {
|
||||
if b.c.Reader != nil {
|
||||
b.c.Scheduler.SetReaderContentCleaner(b.c.Reader.PruneContentCache)
|
||||
}
|
||||
if b.cfg != nil {
|
||||
b.c.Scheduler.SetReaderFilesTTL(b.cfg.Cache.ReaderFilesTTLHours)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *serviceContainerBuilder) initIdentityServices() {
|
||||
|
||||
Reference in New Issue
Block a user