fix cloud library scale and playback compatibility

Root cause: display-only cloud library filtering was reused by scan jobs, merged cloud mounts could be skipped, and OpenList listing relied on WebDAV/first-page behavior that could cap huge directories around 100 items. Cloud scans also let new items consume probe budget before existing rows with missing track/artwork metadata.\n\nChanges:\n- split display filtering from scannable cloud filtering so merged cloud mounts still scan\n- add OpenList API pagination with WebDAV fallback\n- prioritize existing cloud media missing metadata before new imports\n- restrict automatic cloud sync to one successful 19:00-21:00 daily window and keep manual scan immediate\n- disable startup cloud full-scan by default, with an explicit opt-in setting\n- improve Emby/cloud playback compatibility and cache/search/test coverage from the continued work
This commit is contained in:
ShukeBta
2026-06-11 18:31:01 +08:00
parent c22d8271ae
commit 23d1fbfddb
35 changed files with 2797 additions and 139 deletions
+179 -2
View File
@@ -1,6 +1,8 @@
package service
import (
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"strings"
@@ -96,8 +98,8 @@ func TestScanCloudLibraryImportsRecursivePlayableMedia(t *testing.T) {
if err != nil {
t.Fatalf("rescan same cloud: %v", err)
}
if res.Added != 0 || res.Updated != 2 {
t.Fatalf("same cloud rescan should update existing rows only, got %#v", res)
if res.Added != 0 || res.Updated != 0 || res.Skipped != 2 {
t.Fatalf("same cloud rescan should skip unchanged rows, got %#v", res)
}
empty = true
@@ -493,6 +495,181 @@ func TestScanCloudLibraryReadsRemoteNFOAndArtwork(t *testing.T) {
}
}
func TestScanOpenListCloudLibraryUsesAPIPaginationBeyondFirstPage(t *testing.T) {
const totalFiles = 125
requestedPages := map[int]bool{}
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/api/fs/list" {
t.Fatalf("unexpected path %s", r.URL.Path)
}
if r.Method != http.MethodPost {
t.Fatalf("unexpected method %s", r.Method)
}
if r.Header.Get("Authorization") != "openlist-token" {
t.Fatalf("missing openlist token: %q", r.Header.Get("Authorization"))
}
var in struct {
Path string `json:"path"`
Page int `json:"page"`
PerPage int `json:"per_page"`
}
if err := json.NewDecoder(r.Body).Decode(&in); err != nil {
t.Fatalf("decode request: %v", err)
}
if in.Path != "/Movies" {
t.Fatalf("path = %q, want /Movies", in.Path)
}
if in.PerPage <= 100 {
t.Fatalf("per_page = %d, want API pagination larger than legacy 100", in.PerPage)
}
requestedPages[in.Page] = true
effectivePageSize := in.PerPage
if effectivePageSize > 100 {
effectivePageSize = 100
}
start := (in.Page - 1) * effectivePageSize
content := []map[string]any{}
for idx := start; idx < totalFiles && idx < start+effectivePageSize; idx++ {
content = append(content, map[string]any{
"name": fmt.Sprintf("Movie.%03d.mkv", idx+1),
"size": int64(1024 + idx),
"is_dir": false,
})
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{
"code": 200,
"message": "success",
"data": map[string]any{
"content": content,
"total": totalFiles,
},
})
}))
defer upstream.Close()
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
if err != nil {
t.Fatal(err)
}
if err := db.AutoMigrate(&model.Library{}, &model.Media{}, &model.Setting{}, &model.StorageConfig{}); err != nil {
t.Fatal(err)
}
repos := repository.New(db)
log := zap.NewNop()
storage := NewStorageConfigService(log, repos, NewCryptoService("", log))
if _, err := storage.Save(t.Context(), StorageInput{
Type: "openlist",
Config: map[string]any{
"server": upstream.URL,
"token": "openlist-token",
},
}); err != nil {
t.Fatal(err)
}
lib := model.Library{Name: "OpenList · Movies", Path: BuildCloudLibraryPath("openlist", "/Movies", "/Movies"), Type: "movie", Enabled: true}
if err := repos.Library.Create(t.Context(), &lib); err != nil {
t.Fatal(err)
}
scanner := NewScannerService(&config.Config{}, log, repos, NewHub(log), nil, nil)
scanner.SetStorageConfig(storage)
res, err := scanner.ScanLibrary(t.Context(), lib.ID)
if err != nil {
t.Fatalf("scan openlist: %v", err)
}
if res.Added != totalFiles {
t.Fatalf("scan result = %#v, want added=%d", res, totalFiles)
}
if got := countMedia(t, repos); got != totalFiles {
t.Fatalf("media count = %d, want %d", got, totalFiles)
}
if !requestedPages[1] || !requestedPages[2] {
t.Fatalf("expected pagination beyond the first 100 entries, got pages %#v", requestedPages)
}
}
func TestScanCloudLibraryQueuesMissingExistingTrackMetadataBeforeNewFiles(t *testing.T) {
const newFiles = maxCloudMediaProbeQueuePerScan + 5
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/file/sort" || r.URL.Query().Get("pdir_fid") != "0" {
t.Fatalf("unexpected cloud list request %s?%s", r.URL.Path, r.URL.RawQuery)
}
var b strings.Builder
b.WriteString(`{"status":200,"code":0,"data":{"list":[`)
for i := 0; i < newFiles; i++ {
if i > 0 {
b.WriteByte(',')
}
_, _ = fmt.Fprintf(&b, `{"fid":"new-%02d","file_name":"New.Movie.%02d.mkv","dir":false,"size":%d}`, i, i, 1000+i)
}
_, _ = fmt.Fprintf(&b, `,{"fid":"existing","file_name":"Existing.Show.S01E01.mkv","dir":false,"size":2048}]}}`)
_, _ = w.Write([]byte(b.String()))
}))
defer upstream.Close()
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
if err != nil {
t.Fatal(err)
}
if err := db.AutoMigrate(&model.Library{}, &model.Media{}, &model.Setting{}, &model.StorageConfig{}); err != nil {
t.Fatal(err)
}
repos := repository.New(db)
log := zap.NewNop()
storage := NewStorageConfigService(log, repos, NewCryptoService("", log))
if _, err := storage.Save(t.Context(), StorageInput{
Type: "quark",
Config: map[string]any{
"cookie": "kps=test",
"base": upstream.URL,
},
}); err != nil {
t.Fatal(err)
}
lib := model.Library{Name: "夸克网盘", Path: "cloud://quark/0", Type: "tv", Enabled: true}
if err := repos.Library.Create(t.Context(), &lib); err != nil {
t.Fatal(err)
}
existingPath := "cloud://quark/Existing.Show.S01E01.mkv"
if err := repos.DB.Create(&model.Media{
LibraryID: lib.ID,
Title: "Existing Show",
Path: existingPath,
SizeBytes: 2048,
Container: "mkv",
STRMURL: "/api/cloud/play/quark?ref=existing",
SeasonNum: 1,
EpisodeNum: 1,
}).Error; err != nil {
t.Fatal(err)
}
scanner := NewScannerService(&config.Config{}, log, repos, NewHub(log), NewFFprobeService(&config.Config{}, log), nil)
scanner.storage = storage
res, err := scanner.ScanLibrary(t.Context(), lib.ID)
if err != nil {
t.Fatalf("scan cloud: %v", err)
}
if res.Added != newFiles || res.Skipped != 1 {
t.Fatalf("scan result = %#v, want new files added and existing skipped", res)
}
foundExistingProbe := false
for {
select {
case task := <-scanner.cloudMediaProbeQueue:
if task.path == existingPath {
foundExistingProbe = true
}
default:
if !foundExistingProbe {
t.Fatal("existing media missing track metadata did not receive probe budget before new files")
}
return
}
}
}
func TestParseCloudImagePlaybackURL(t *testing.T) {
typ, ref, ok := parseCloudImagePlaybackURL("http://nas.local/api/cloud/play/openlist?ref=%2FAnime%2FJianLai%2Fposter.jpg")
if !ok || typ != "openlist" || ref != "/Anime/JianLai/poster.jpg" {