tcjlj/MeBox
1
0
Fork 0
mirror of https://github.com/truewhile/MeBox.git synced 2026-09-29 19:36:36 +08:00
Code Issues Packages Projects Releases Wiki Activity

Improve large library storage backends and scans

Browse Source
This commit is contained in:
ShukeBta
2026-06-15 18:16:34 +08:00
parent c9e6adf041
commit a5bf4bfdd4
36 changed files with 2340 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/handler/media.go
+63 -4
View File
@@ -6,6 +6,7 @@ import (
"errors"
"net/http"
"strconv"
"strings"
"github.com/gin-gonic/gin"
@@ -130,7 +131,7 @@ func scanLibraryHandler(svc *service.Container) gin.HandlerFunc {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
finishHTTPTask(task, nil, "completed", "手动扫描入库结束", scanTaskMetrics(res), nil)
finishHTTPTask(task, nil, "completed", "手动扫描入库结束", scanTaskMetrics(res), scanTaskDetails(res, 20))
c.JSON(http.StatusOK, res)
}
}
@@ -161,15 +162,49 @@ func scanTaskMetrics(res *service.ScanResult) map[string]int64 {
"probed": int64(res.Probed),
"local_metadata": int64(res.LocalMetadata),
"removed": res.Removed,
"errors": int64(res.ErrorCount),
}
}
func scanTaskDetails(res *service.ScanResult, limit int) []string {
if res == nil || limit <= 0 {
return nil
}
out := make([]string, 0, limit)
for _, line := range res.Errors {
line = strings.TrimSpace(line)
if line == "" {
continue
}
out = append(out, "错误: "+line)
if len(out) >= limit {
return out
}
}
return out
}
func listMediaHandler(svc *service.Container) gin.HandlerFunc {
return func(c *gin.Context) {
id := c.Param("id")
page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
size, _ := strconv.Atoi(c.DefaultQuery("page_size", "50"))
items, total, err := svc.Media.ListMediaVisible(c.Request.Context(), id, page, size, mediaVisibilityForRequest(c, svc))
groupVersions := c.DefaultQuery("group_versions", "1") != "0"
if !groupVersions {
items, total, err := svc.Media.ListMediaVisible(c.Request.Context(), id, page, size, mediaVisibilityForRequest(c, svc))
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
c.JSON(http.StatusOK, gin.H{
"items": items,
"total": total,
"page": page,
"page_size": size,
})
return
}
items, total, err := svc.Media.ListMediaVisibleGrouped(c.Request.Context(), id, page, size, mediaVisibilityForRequest(c, svc))
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
@@ -205,10 +240,25 @@ func getMediaHandler(svc *service.Container) gin.HandlerFunc {
func searchMediaHandler(svc *service.Container) gin.HandlerFunc {
return func(c *gin.Context) {
q := c.Query("q")
groupVersions := c.DefaultQuery("group_versions", "1") != "0"
if c.Query("page") != "" || c.Query("page_size") != "" {
page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
size, _ := strconv.Atoi(c.DefaultQuery("page_size", "50"))
items, total, err := svc.Media.SearchMediaVisiblePage(c.Request.Context(), q, page, size, mediaVisibilityForRequest(c, svc))
if !groupVersions {
items, total, err := svc.Media.SearchMediaVisiblePage(c.Request.Context(), q, page, size, mediaVisibilityForRequest(c, svc))
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
c.JSON(http.StatusOK, gin.H{
"items": items,
"total": total,
"page": page,
"page_size": size,
})
return
}
items, total, err := svc.Media.SearchMediaVisiblePageGrouped(c.Request.Context(), q, page, size, mediaVisibilityForRequest(c, svc))
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
@@ -222,7 +272,16 @@ func searchMediaHandler(svc *service.Container) gin.HandlerFunc {
return
}
limit, _ := strconv.Atoi(c.DefaultQuery("limit", "50"))
items, err := svc.Media.SearchMediaVisible(c.Request.Context(), q, limit, mediaVisibilityForRequest(c, svc))
if !groupVersions {
items, err := svc.Media.SearchMediaVisible(c.Request.Context(), q, limit, mediaVisibilityForRequest(c, svc))
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
c.JSON(http.StatusOK, gin.H{"items": items})
return
}
items, err := svc.Media.SearchMediaVisibleGrouped(c.Request.Context(), q, limit, mediaVisibilityForRequest(c, svc))
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
internal/handler/media_test.go
+85
View File
@@ -5,6 +5,7 @@ import (
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/gin-gonic/gin"
"github.com/glebarez/sqlite"
@@ -90,6 +91,66 @@ func TestListLibrariesIncludeHiddenNormalizesCloudDisplayNames(t *testing.T) {
}
}
func TestListMediaGroupsMultipleVersionsByDefault(t *testing.T) {
gin.SetMode(gin.TestMode)
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
if err != nil {
t.Fatal(err)
}
if err := db.AutoMigrate(&model.User{}, &model.Library{}, &model.Media{}, &model.Setting{}, &model.PlayProfile{}); err != nil {
t.Fatal(err)
}
repos := repository.New(db)
lib := model.Library{Name: "Movies", Path: "/media/movies", Type: "movie", Enabled: true}
if err := repos.Library.Create(t.Context(), &lib); err != nil {
t.Fatal(err)
}
if err := repos.DB.Create(&[]model.Media{
{
Base: model.Base{ID: "movie-1080", CreatedAt: time.Now().Add(-time.Minute)},
LibraryID: lib.ID,
Title: "流浪地球",
Path: "/media/movies/The.Wandering.Earth.2019.1080p.mkv",
Year: 2019,
Width: 1920,
Height: 1080,
SizeBytes: 100,
},
{
Base: model.Base{ID: "movie-2160", CreatedAt: time.Now()},
LibraryID: lib.ID,
Title: "流浪地球",
Path: "cloud://openlist/Movies/The.Wandering.Earth.2019.2160p.mkv",
Year: 2019,
Width: 3840,
Height: 2160,
SizeBytes: 200,
},
}).Error; err != nil {
t.Fatal(err)
}
svc := &service.Container{
Repo: repos,
Media: service.NewMediaService(&config.Config{}, zap.NewNop(), repos),
}
grouped := requestMediaList(t, svc, "/api/libraries/"+lib.ID+"/media", lib.ID)
if grouped.Total != 1 || len(grouped.Items) != 1 {
t.Fatalf("grouped response total=%d len=%d body=%#v", grouped.Total, len(grouped.Items), grouped)
}
if grouped.Items[0].ID != "movie-2160" {
t.Fatalf("primary id = %q, want highest quality version", grouped.Items[0].ID)
}
if len(grouped.Items[0].Versions) != 2 {
t.Fatalf("versions = %#v, want both versions", grouped.Items[0].Versions)
}
raw := requestMediaList(t, svc, "/api/libraries/"+lib.ID+"/media?group_versions=0", lib.ID)
if raw.Total != 2 || len(raw.Items) != 2 {
t.Fatalf("raw response total=%d len=%d body=%#v", raw.Total, len(raw.Items), raw)
}
}
func requestLibraries(t *testing.T, svc *service.Container, userID, role, path string) []model.Library {
t.Helper()
w := httptest.NewRecorder()
@@ -107,3 +168,27 @@ func requestLibraries(t *testing.T, svc *service.Container, userID, role, path s
}
return libs
}
type mediaListResponse struct {
Items []service.MediaItem `json:"items"`
Total int64 `json:"total"`
}
func requestMediaList(t *testing.T, svc *service.Container, path, libraryID string) mediaListResponse {
t.Helper()
w := httptest.NewRecorder()
c, _ := gin.CreateTestContext(w)
c.Set(middleware.CtxUserID, "user-1")
c.Set(middleware.CtxUserRole, "user")
c.Params = gin.Params{{Key: "id", Value: libraryID}}
c.Request = httptest.NewRequest(http.MethodGet, path, nil)
listMediaHandler(svc)(c)
if w.Code != http.StatusOK {
t.Fatalf("GET %s status = %d body=%s", path, w.Code, w.Body.String())
}
var payload mediaListResponse
if err := json.Unmarshal(w.Body.Bytes(), &payload); err != nil {
t.Fatalf("decode media list: %v", err)
}
return payload
}
internal/handler/stats.go
+58
View File
@@ -2,7 +2,12 @@
package handler
import (
"crypto/sha1"
"encoding/hex"
"net/http"
"sort"
"strings"
"time"
"github.com/gin-gonic/gin"
"gorm.io/gorm"
@@ -45,6 +50,18 @@ func applyStatsVisibility(c *gin.Context, svc *service.Container, snap *service.
}
}
snap.Libraries = visibleLibraries
cacheKey := visibleStatsCacheKey(visibility, activeLibraryIDs)
if svc.Cache != nil {
var cached visibleStatsCacheValue
if svc.Cache.GetJSON(c.Request.Context(), cacheKey, &cached) {
snap.Libraries = cached.Libraries
snap.MediaCount = cached.MediaCount
snap.TotalSizeBytes = cached.TotalSizeBytes
snap.TotalSeconds = cached.TotalSeconds
snap.RecentlyAdded = cached.RecentlyAdded
return nil
}
}
q := applyMediaVisibilityQuery(svc.Repo.DB.WithContext(c.Request.Context()).Model(&model.Media{}), visibility)
q = applyActiveLibraryQuery(q, activeLibraryIDs)
@@ -72,9 +89,50 @@ func applyStatsVisibility(c *gin.Context, svc *service.Container, snap *service.
return err
}
snap.RecentlyAdded = recent
if svc.Cache != nil {
svc.Cache.SetJSON(c.Request.Context(), cacheKey, visibleStatsCacheValue{
Libraries: snap.Libraries,
MediaCount: snap.MediaCount,
TotalSizeBytes: snap.TotalSizeBytes,
TotalSeconds: snap.TotalSeconds,
RecentlyAdded: snap.RecentlyAdded,
}, 10*time.Second)
}
return nil
}
type visibleStatsCacheValue struct {
Libraries int64 `json:"libraries"`
MediaCount int64 `json:"media_count"`
TotalSizeBytes int64 `json:"total_size_bytes"`
TotalSeconds int64 `json:"total_seconds"`
RecentlyAdded []model.Media `json:"recently_added"`
}
func visibleStatsCacheKey(visibility service.MediaVisibility, activeLibraryIDs []string) string {
allowed := append([]string(nil), visibility.AllowedLibraryIDs...)
hidden := append([]string(nil), visibility.HiddenLibraryIDs...)
active := append([]string(nil), activeLibraryIDs...)
sort.Strings(allowed)
sort.Strings(hidden)
sort.Strings(active)
sum := sha1.Sum([]byte(strings.Join([]string{
"visible",
strings.Join(active, ","),
strings.Join(allowed, ","),
strings.Join(hidden, ","),
boolString(visibility.IncludeNSFW),
}, "|")))
return "stats:visible:" + hex.EncodeToString(sum[:])
}
func boolString(value bool) string {
if value {
return "1"
}
return "0"
}
func applyMediaVisibilityQuery(q *gorm.DB, visibility service.MediaVisibility) *gorm.DB {
if !visibility.IncludeNSFW {
q = q.Where("nsfw = ?", false)
internal/handler/stats_extra.go
+6 -2
View File
@@ -144,13 +144,17 @@ func statsLibrariesHandler(svc *service.Container) gin.HandlerFunc {
if !service.LibraryVisibleForUser(c.Request.Context(), svc.Repo, l, visibility) {
continue
}
libraryIDs, err := service.MergedLibraryIDsForLibrary(c.Request.Context(), svc.Repo, l.ID)
if err != nil || len(libraryIDs) == 0 {
libraryIDs = []string{l.ID}
}
var count int64
var size int64
_ = applyMediaVisibilityQuery(svc.Repo.DB.Model(&model.Media{}), visibility).
Where("library_id = ?", l.ID).
Where("library_id IN ?", libraryIDs).
Count(&count).Error
_ = applyMediaVisibilityQuery(svc.Repo.DB.Model(&model.Media{}), visibility).
Where("library_id = ?", l.ID).
Where("library_id IN ?", libraryIDs).
Select("COALESCE(SUM(size_bytes),0)").Row().Scan(&size)
out = append(out, gin.H{
"library": l,
internal/handler/stats_test.go
+51
View File
@@ -1,6 +1,8 @@
package handler
import (
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
@@ -62,3 +64,52 @@ func TestStatsSnapshotHidesAdultRecentlyAddedForUser(t *testing.T) {
t.Fatalf("recently added should hide adult library, got %#v", snap.RecentlyAdded)
}
}
func TestStatsLibrariesCountsMergedCloudLibraryItems(t *testing.T) {
gin.SetMode(gin.TestMode)
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
if err != nil {
t.Fatal(err)
}
if err := db.AutoMigrate(&model.User{}, &model.Library{}, &model.Media{}, &model.Setting{}, &model.PlayProfile{}); err != nil {
t.Fatal(err)
}
repos := repository.New(db)
local := model.Library{Name: "国产电影", Path: "/media/国产电影", Type: "movie", Enabled: true}
cloud := model.Library{Name: "OpenList · 国产电影", Path: service.BuildCloudLibraryPath("openlist", "/国产电影", "/国产电影"), Type: "movie", Enabled: true}
for _, lib := range []*model.Library{&local, &cloud} {
if err := repos.Library.Create(t.Context(), lib); err != nil {
t.Fatal(err)
}
}
if err := db.Create(&[]model.Media{
{LibraryID: local.ID, Title: "本地版本", Path: "/media/国产电影/local.mkv", SizeBytes: 100},
{LibraryID: cloud.ID, Title: "云盘版本", Path: "cloud://openlist/国产电影/cloud.mkv", SizeBytes: 200},
}).Error; err != nil {
t.Fatal(err)
}
svc := &service.Container{Repo: repos}
w := httptest.NewRecorder()
c, _ := gin.CreateTestContext(w)
c.Request = httptest.NewRequest("GET", "/api/stats/libraries", nil)
statsLibrariesHandler(svc)(c)
if w.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", w.Code, w.Body.String())
}
var payload struct {
Libraries []struct {
ItemCount int64 `json:"item_count"`
TotalSize int64 `json:"total_size"`
} `json:"libraries"`
}
if err := json.Unmarshal(w.Body.Bytes(), &payload); err != nil {
t.Fatal(err)
}
if len(payload.Libraries) != 1 {
t.Fatalf("libraries = %#v, want one merged display library", payload.Libraries)
}
if payload.Libraries[0].ItemCount != 2 || payload.Libraries[0].TotalSize != 300 {
t.Fatalf("merged stats = %#v, want count=2 size=300", payload.Libraries[0])
}
}