Compare commits

...

20 Commits

Author SHA1 Message Date
github-actions[bot] 9b956b928b chore: bump version to 0.0.39 [skip ci] 2026-08-26 13:41:25 +00:00
truewhile c3187f6e3f 优化上传逻辑
优化上传逻辑
2026-08-26 21:41:06 +08:00
github-actions[bot] 60c815a8b3 chore: bump version to 0.0.38 [skip ci] 2026-08-26 09:06:04 +00:00
truewhile 41b155ea31 Merge branch 'main' of https://github.com/truewhile/MMTL 2026-08-26 17:05:45 +08:00
truewhile 2888ae8bf7 8 2026-08-26 17:05:41 +08:00
github-actions[bot] 7363064d89 chore: bump version to 0.0.37 [skip ci] 2026-08-26 08:16:56 +00:00
truewhile 1d53bf2ae1 7 2026-08-26 16:16:39 +08:00
github-actions[bot] 618165ec31 chore: bump version to 0.0.36 [skip ci] 2026-08-26 07:51:14 +00:00
truewhile 87c66a9b8c 6 2026-08-26 15:50:59 +08:00
github-actions[bot] 1ea4724261 chore: bump version to 0.0.35 [skip ci] 2026-08-26 06:50:29 +00:00
truewhile 3d372f039e Merge branch 'main' of https://github.com/truewhile/MMTL 2026-08-26 14:50:10 +08:00
truewhile 0384017e98 6 2026-08-26 14:50:06 +08:00
github-actions[bot] 0332579d5f chore: bump version to 0.0.34 [skip ci] 2026-08-26 04:52:28 +00:00
truewhile 6aefe18caa 5 2026-08-26 12:52:10 +08:00
github-actions[bot] ef72fc8d83 chore: bump version to 0.0.33 [skip ci] 2026-08-26 04:12:13 +00:00
truewhile 4764c09572 4 2026-08-26 12:11:56 +08:00
github-actions[bot] 98ca766a37 chore: bump version to 0.0.32 [skip ci] 2026-08-26 03:38:14 +00:00
truewhile 13c9035b76 3 2026-08-26 11:37:58 +08:00
github-actions[bot] ad6d0ba21d chore: bump version to 0.0.31 [skip ci] 2026-08-26 03:19:09 +00:00
truewhile 431f7f088b 2 2026-08-26 11:18:53 +08:00
44 changed files with 1742 additions and 281 deletions
+1 -1
View File
@@ -1 +1 @@
0.0.30
0.0.39
+2 -1
View File
@@ -3,6 +3,7 @@ module github.com/ShukeBta/MMTL
go 1.25.0
require (
github.com/aliyun/alibabacloud-oss-go-sdk-v2 v1.5.2
github.com/fsnotify/fsnotify v1.7.0
github.com/gin-gonic/gin v1.9.1
github.com/glebarez/sqlite v1.11.0
@@ -16,6 +17,7 @@ require (
go.uber.org/zap v1.27.0
golang.org/x/crypto v0.21.0
golang.org/x/sys v0.20.0
golang.org/x/time v0.15.0
gorm.io/driver/postgres v1.5.7
gorm.io/gorm v1.30.0
)
@@ -72,7 +74,6 @@ require (
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
golang.org/x/net v0.21.0 // indirect
golang.org/x/text v0.20.0 // indirect
golang.org/x/time v0.15.0 // indirect
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
+2
View File
@@ -1,3 +1,5 @@
github.com/aliyun/alibabacloud-oss-go-sdk-v2 v1.5.2 h1:40yUSXwdkWN851BHCq6uiDhleh7A4+0yIBS+IUAqZVY=
github.com/aliyun/alibabacloud-oss-go-sdk-v2 v1.5.2/go.mod h1:FTzydeQVmR24FI0D6XWUOMKckjXehM/jgMn1xC+DA9M=
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
+38 -8
View File
@@ -16,12 +16,13 @@ import (
)
type createLibraryReq struct {
Name string `json:"name" binding:"required"`
Path string `json:"path"`
Paths []string `json:"paths"`
Roots []service.LibraryRootInput `json:"roots"`
Type string `json:"type"`
CoverURL string `json:"cover_url"`
Name string `json:"name"`
Path string `json:"path"`
Paths []string `json:"paths"`
Roots []service.LibraryRootInput `json:"roots"`
Type string `json:"type"`
CoverURL string `json:"cover_url"`
CreatePerSubfolder bool `json:"create_per_subfolder"`
}
func listLibrariesHandler(svc *service.Container) gin.HandlerFunc {
@@ -88,9 +89,38 @@ func createLibraryHandler(svc *service.Container) gin.HandlerFunc {
}
}
if len(roots) == 0 && strings.TrimSpace(req.Path) != "" {
roots = append(roots, service.LibraryRootInput{Path: req.Path})
roots = append(roots, service.LibraryRootInput{Path: req.Path})
}
var l *model.Library
if req.CreatePerSubfolder {
parent := ""
if len(roots) > 0 {
parent = roots[0].Path
} else if strings.TrimSpace(req.Path) != "" {
parent = req.Path
}
l, err := svc.Media.CreateLibraryWithRootsAndCover(c.Request.Context(), req.Name, req.Type, req.CoverURL, roots)
created, err := svc.Media.CreateLibrariesPerSubfolder(c.Request.Context(), parent, req.Type, req.CoverURL)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
uid, _ := c.Get("ctx_user_id")
for i := range created {
lib := &created[i]
svc.Audit.Record(c.Request.Context(), toString(uid), "library.create", lib.ID, c.ClientIP(), lib.Path)
if svc.Watcher != nil {
go func() { _ = svc.Watcher.Refresh(context.Background()) }()
}
for _, root := range lib.Roots {
if root.Enabled {
queueLibraryRootScan(svc, lib.ID, root.ID)
}
}
}
c.JSON(http.StatusCreated, gin.H{"libraries": created})
return
}
l, err := svc.Media.CreateLibraryWithRootsAndCover(c.Request.Context(), req.Name, req.Type, req.CoverURL, roots)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
+2
View File
@@ -50,12 +50,14 @@ func registerAdminStrmRoutes(admin *gin.RouterGroup, svc *service.Container) {
admin.POST("/strm/downloads/:id/retry", retryStrmDownloadHandler(svc))
admin.POST("/strm/downloads/clear-done", clearDoneDownloadsHandler(svc))
admin.POST("/strm/downloads/clear-finished", clearFinishedDownloadsHandler(svc))
admin.POST("/strm/downloads/clear-canceled", clearCanceledDownloadsHandler(svc))
admin.POST("/strm/downloads/retry-failed", retryAllFailedDownloadsHandler(svc))
admin.POST("/strm/downloads/cancel-pending", cancelPendingDownloadsHandler(svc))
admin.GET("/strm/uploads", uploadQueueHandler(svc))
admin.POST("/strm/uploads/:id/cancel", cancelStrmUploadHandler(svc))
admin.POST("/strm/uploads/:id/retry", retryStrmUploadHandler(svc))
admin.POST("/strm/uploads/cancel-pending", cancelPendingUploadsHandler(svc))
admin.POST("/strm/uploads/clear-canceled", clearCanceledUploadsHandler(svc))
}
func registerAdminUserRoutes(admin *gin.RouterGroup, svc *service.Container) {
+22
View File
@@ -392,6 +392,28 @@ func clearFinishedDownloadsHandler(svc *service.Container) gin.HandlerFunc {
}
}
func clearCanceledDownloadsHandler(svc *service.Container) gin.HandlerFunc {
return func(c *gin.Context) {
n, err := svc.Strm.ClearCanceledDownloadTasks(c.Request.Context())
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
c.JSON(http.StatusOK, gin.H{"deleted": n})
}
}
func clearCanceledUploadsHandler(svc *service.Container) gin.HandlerFunc {
return func(c *gin.Context) {
n, err := svc.Strm.ClearCanceledUploadTasks(c.Request.Context())
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
c.JSON(http.StatusOK, gin.H{"deleted": n})
}
}
func retryAllFailedDownloadsHandler(svc *service.Container) gin.HandlerFunc {
return func(c *gin.Context) {
n, err := svc.Strm.RetryAllFailedDownloadTasks(c.Request.Context())
+10 -4
View File
@@ -50,10 +50,16 @@ func TestStrmAdminRoutesAreRegistered(t *testing.T) {
"GET /api/admin/strm/downloads",
"POST /api/admin/strm/downloads/:id/cancel",
"POST /api/admin/strm/downloads/:id/retry",
"GET /api/admin/strm/uploads",
"POST /api/admin/strm/uploads/:id/cancel",
"POST /api/admin/strm/uploads/:id/retry",
"GET /api/strm/play/:provider/:file",
"POST /api/admin/strm/downloads/clear-finished",
"POST /api/admin/strm/downloads/clear-canceled",
"POST /api/admin/strm/downloads/retry-failed",
"POST /api/admin/strm/downloads/cancel-pending",
"GET /api/admin/strm/uploads",
"POST /api/admin/strm/uploads/:id/cancel",
"POST /api/admin/strm/uploads/:id/retry",
"POST /api/admin/strm/uploads/cancel-pending",
"POST /api/admin/strm/uploads/clear-canceled",
"GET /api/strm/play/:provider/:file",
} {
if !routes[want] {
t.Fatalf("%s route is not registered", want)
+1 -1
View File
@@ -163,7 +163,7 @@ func historyDeleteHandler(svc *service.Container) gin.HandlerFunc {
c.JSON(http.StatusBadRequest, gin.H{"error": "status must be completed or incomplete"})
return
}
res := q.Delete(&model.PlaybackHistory{})
res := q.Unscoped().Delete(&model.PlaybackHistory{})
if err := res.Error; err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
+2 -2
View File
@@ -62,9 +62,9 @@ func (r *ApiConfigRepository) Update(ctx context.Context, c *model.ApiConfig) er
}).Error
}
// Delete removes an API config.
// Delete 物理删除 API 配置。
func (r *ApiConfigRepository) Delete(ctx context.Context, provider string) error {
return r.db.WithContext(ctx).Where("provider = ?", provider).Delete(&model.ApiConfig{}).Error
return r.db.WithContext(ctx).Unscoped().Where("provider = ?", provider).Delete(&model.ApiConfig{}).Error
}
// UpdateTestResult 更新测试结果。
+1 -1
View File
@@ -23,7 +23,7 @@ func (r *FavoriteRepository) Toggle(ctx context.Context, userID, mediaID string)
if err != nil {
return false, err
}
return false, r.db.WithContext(ctx).Delete(&f).Error
return false, r.db.WithContext(ctx).Unscoped().Delete(&f).Error
}
// ListByUser returns all favourite media IDs for a user.
+3 -4
View File
@@ -79,10 +79,9 @@ func (r *LibraryRepository) FindByID(ctx context.Context, id string) (*model.Lib
return &l, nil
}
// Delete removes a library and (soft) cascades to its media via repository
// callers; we do not run CASCADE here to keep this method narrow.
// Delete 物理删除媒体库。
func (r *LibraryRepository) Delete(ctx context.Context, id string) error {
return r.db.WithContext(ctx).Delete(&model.Library{}, "id = ?", id).Error
return r.db.WithContext(ctx).Unscoped().Delete(&model.Library{}, "id = ?", id).Error
}
func (r *LibraryRepository) ListRoots(ctx context.Context, libraryID string) ([]model.LibraryRoot, error) {
@@ -149,7 +148,7 @@ func (r *LibraryRepository) DeleteRoot(ctx context.Context, libraryID, rootID st
if !r.hasLibraryRootsTable() {
return nil
}
return r.db.WithContext(ctx).Where("library_id = ?", libraryID).Delete(&model.LibraryRoot{}, "id = ?", rootID).Error
return r.db.WithContext(ctx).Unscoped().Where("library_id = ?", libraryID).Delete(&model.LibraryRoot{}, "id = ?", rootID).Error
}
func (r *LibraryRepository) hasLibraryRootsTable() bool {
+3 -3
View File
@@ -116,12 +116,12 @@ func (r *MediaRepository) ListByLibrariesFiltered(ctx context.Context, libraryID
// DeleteByLibrary purges all media tied to a library.
func (r *MediaRepository) DeleteByLibrary(ctx context.Context, libraryID string) error {
// FTS 行由 media 表上的触发器同步清理(软删/硬删都覆盖)。
return r.db.WithContext(ctx).Where("library_id = ?", libraryID).Delete(&model.Media{}).Error
// FTS 行由 media 表上的触发器同步清理(物理删除触发 FTS 清理)。
return r.db.WithContext(ctx).Unscoped().Where("library_id = ?", libraryID).Delete(&model.Media{}).Error
}
func (r *MediaRepository) DeleteByLibraryRoot(ctx context.Context, libraryID, rootID string) error {
return r.db.WithContext(ctx).
return r.db.WithContext(ctx).Unscoped().
Where("library_id = ? AND library_root_id = ?", libraryID, rootID).
Delete(&model.Media{}).Error
}
+2 -2
View File
@@ -51,9 +51,9 @@ func (r *PermissionRepository) Upsert(ctx context.Context, p *model.UserPermissi
})
}
// Delete removes a permission record.
// Delete 物理删除权限记录。
func (r *PermissionRepository) Delete(ctx context.Context, userID string) error {
return withSQLiteBusyRetry(ctx, func() error {
return r.db.WithContext(ctx).Where("user_id = ?", userID).Delete(&model.UserPermission{}).Error
return r.db.WithContext(ctx).Unscoped().Where("user_id = ?", userID).Delete(&model.UserPermission{}).Error
})
}
+2 -2
View File
@@ -59,9 +59,9 @@ func (r *PlayProfileRepository) Update(ctx context.Context, id string, patch map
Where("id = ?", id).Updates(patch).Error
}
// Delete soft-deletes a profile.
// Delete 物理删除播放档案。
func (r *PlayProfileRepository) Delete(ctx context.Context, id string) error {
return r.db.WithContext(ctx).Delete(&model.PlayProfile{}, "id = ?", id).Error
return r.db.WithContext(ctx).Unscoped().Delete(&model.PlayProfile{}, "id = ?", id).Error
}
// ClearDefaultsFor resets is_default for all of a user's profiles.
@@ -72,10 +72,10 @@ func (r *RefreshTokenRepository) RevokeOldestActiveByUserID(ctx context.Context,
})
}
// DeleteExpired removes all expired refresh tokens.
// DeleteExpired 物理清理所有过期的 refresh tokens。
func (r *RefreshTokenRepository) DeleteExpired(ctx context.Context) error {
return withSQLiteBusyRetry(ctx, func() error {
return r.db.WithContext(ctx).Where("expires_at < ?", time.Now()).Delete(&model.RefreshToken{}).Error
return r.db.WithContext(ctx).Unscoped().Where("expires_at < ?", time.Now()).Delete(&model.RefreshToken{}).Error
})
}
+2 -2
View File
@@ -29,9 +29,9 @@ func (r *SettingRepository) Set(ctx context.Context, key, value string) error {
return r.db.WithContext(ctx).Save(&s).Error
}
// Delete removes a setting key.
// Delete 物理删除设置键。
func (r *SettingRepository) Delete(ctx context.Context, key string) error {
return r.db.WithContext(ctx).Where("key = ?", key).Delete(&model.Setting{}).Error
return r.db.WithContext(ctx).Unscoped().Where("key = ?", key).Delete(&model.Setting{}).Error
}
// All returns every key/value pair (used by the admin UI).
+2 -2
View File
@@ -66,9 +66,9 @@ func (r *StorageConfigRepository) Upsert(ctx context.Context, c *model.StorageCo
}).Error
}
// Delete removes a storage config by ID.
// Delete 物理删除存储配置。
func (r *StorageConfigRepository) Delete(ctx context.Context, id string) error {
return r.db.WithContext(ctx).Where("id = ?", id).Delete(&model.StorageConfig{}).Error
return r.db.WithContext(ctx).Unscoped().Where("id = ?", id).Delete(&model.StorageConfig{}).Error
}
// FindByID returns a storage config by ID.
+31 -9
View File
@@ -59,7 +59,7 @@ func (r *StrmAccountRepository) Update(ctx context.Context, a *model.StrmAccount
func (r *StrmAccountRepository) Delete(ctx context.Context, id string) error {
return withSQLiteBusyRetry(ctx, func() error {
return r.db.WithContext(ctx).Where("id = ?", id).Delete(&model.StrmAccount{}).Error
return r.db.WithContext(ctx).Unscoped().Where("id = ?", id).Delete(&model.StrmAccount{}).Error
})
}
@@ -123,7 +123,7 @@ func (r *StrmSyncPathRepository) Update(ctx context.Context, p *model.StrmSyncPa
func (r *StrmSyncPathRepository) Delete(ctx context.Context, id string) error {
return withSQLiteBusyRetry(ctx, func() error {
return r.db.WithContext(ctx).Where("id = ?", id).Delete(&model.StrmSyncPath{}).Error
return r.db.WithContext(ctx).Unscoped().Where("id = ?", id).Delete(&model.StrmSyncPath{}).Error
})
}
@@ -286,7 +286,7 @@ func (r *StrmDownloadTaskRepository) Update(ctx context.Context, t *model.StrmDo
func (r *StrmDownloadTaskRepository) Delete(ctx context.Context, id string) error {
return withSQLiteBusyRetry(ctx, func() error {
return r.db.WithContext(ctx).Where("id = ?", id).Delete(&model.StrmDownloadTask{}).Error
return r.db.WithContext(ctx).Unscoped().Where("id = ?", id).Delete(&model.StrmDownloadTask{}).Error
})
}
@@ -294,7 +294,7 @@ func (r *StrmDownloadTaskRepository) Delete(ctx context.Context, id string) erro
func (r *StrmDownloadTaskRepository) ClearDone(ctx context.Context) (int64, error) {
var count int64
err := withSQLiteBusyRetry(ctx, func() error {
res := r.db.WithContext(ctx).Where("status = ?", model.StrmTaskDone).Delete(&model.StrmDownloadTask{})
res := r.db.WithContext(ctx).Unscoped().Where("status = ?", model.StrmTaskDone).Delete(&model.StrmDownloadTask{})
count = res.RowsAffected
return res.Error
})
@@ -305,7 +305,7 @@ func (r *StrmDownloadTaskRepository) ClearDone(ctx context.Context) (int64, erro
func (r *StrmDownloadTaskRepository) ClearFinished(ctx context.Context) (int64, error) {
var count int64
err := withSQLiteBusyRetry(ctx, func() error {
res := r.db.WithContext(ctx).Where("status IN ?", []string{model.StrmTaskDone, model.StrmTaskFailed}).
res := r.db.WithContext(ctx).Unscoped().Where("status IN ?", []string{model.StrmTaskDone, model.StrmTaskFailed, model.StrmTaskCanceled}).
Delete(&model.StrmDownloadTask{})
count = res.RowsAffected
return res.Error
@@ -313,6 +313,17 @@ func (r *StrmDownloadTaskRepository) ClearFinished(ctx context.Context) (int64,
return count, err
}
// ClearCanceled 清空全部已取消下载任务。
func (r *StrmDownloadTaskRepository) ClearCanceled(ctx context.Context) (int64, error) {
var count int64
err := withSQLiteBusyRetry(ctx, func() error {
res := r.db.WithContext(ctx).Unscoped().Where("status = ?", model.StrmTaskCanceled).Delete(&model.StrmDownloadTask{})
count = res.RowsAffected
return res.Error
})
return count, err
}
// RetryAllFailed 把所有失败任务重置回待处理,清空错误与重试计数。
func (r *StrmDownloadTaskRepository) RetryAllFailed(ctx context.Context) (int64, error) {
var count int64
@@ -381,7 +392,7 @@ func (r *StrmDownloadTaskRepository) GetActiveLocalPathMap(ctx context.Context,
func (r *StrmDownloadTaskRepository) DeleteFinishedOlderThan(ctx context.Context, before time.Time) error {
return withSQLiteBusyRetry(ctx, func() error {
return r.db.WithContext(ctx).Where("status IN ? AND finished_at < ?",
return r.db.WithContext(ctx).Unscoped().Where("status IN ? AND finished_at < ?",
[]string{model.StrmTaskDone, model.StrmTaskFailed, model.StrmTaskCanceled}, before).
Delete(&model.StrmDownloadTask{}).Error
})
@@ -522,10 +533,21 @@ func (r *StrmUploadTaskRepository) Update(ctx context.Context, t *model.StrmUplo
func (r *StrmUploadTaskRepository) Delete(ctx context.Context, id string) error {
return withSQLiteBusyRetry(ctx, func() error {
return r.db.WithContext(ctx).Where("id = ?", id).Delete(&model.StrmUploadTask{}).Error
return r.db.WithContext(ctx).Unscoped().Where("id = ?", id).Delete(&model.StrmUploadTask{}).Error
})
}
// ClearCanceled 清空全部已取消上传任务。
func (r *StrmUploadTaskRepository) ClearCanceled(ctx context.Context) (int64, error) {
var count int64
err := withSQLiteBusyRetry(ctx, func() error {
res := r.db.WithContext(ctx).Unscoped().Where("status = ?", model.StrmTaskCanceled).Delete(&model.StrmUploadTask{})
count = res.RowsAffected
return res.Error
})
return count, err
}
// CancelPending 批量取消所有排队中和进行中的任务。
func (r *StrmUploadTaskRepository) CancelPending(ctx context.Context) (int64, error) {
now := time.Now()
@@ -573,7 +595,7 @@ func (r *StrmUploadTaskRepository) GetActiveLocalPathMap(ctx context.Context, sy
func (r *StrmUploadTaskRepository) DeleteFinishedOlderThan(ctx context.Context, before time.Time) error {
return withSQLiteBusyRetry(ctx, func() error {
return r.db.WithContext(ctx).Where("status IN ? AND finished_at < ?",
return r.db.WithContext(ctx).Unscoped().Where("status IN ? AND finished_at < ?",
[]string{model.StrmTaskDone, model.StrmTaskFailed, model.StrmTaskCanceled}, before).
Delete(&model.StrmUploadTask{}).Error
})
@@ -614,7 +636,7 @@ func (r *StrmDirCacheRepository) Set(ctx context.Context, syncPathID, dirID, pat
func (r *StrmDirCacheRepository) DeleteBySyncPathID(ctx context.Context, syncPathID string) error {
return withSQLiteBusyRetry(ctx, func() error {
return r.db.WithContext(ctx).Where("sync_path_id = ?", syncPathID).Delete(&model.StrmDirCache{}).Error
return r.db.WithContext(ctx).Unscoped().Where("sync_path_id = ?", syncPathID).Delete(&model.StrmDirCache{}).Error
})
}
+11 -20
View File
@@ -133,26 +133,17 @@ func (r *UserRepository) TouchLogin(ctx context.Context, id string) error {
})
}
// Delete removes a user (soft-delete via gorm.DeletedAt), releases the unique
// username, and drops Telegram bindings so future re-created users bind cleanly.
// Delete 物理删除用户并级联清理其关联记录。
func (r *UserRepository) Delete(ctx context.Context, id string) error {
return r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
var user model.User
if err := tx.Where("id = ?", id).First(&user).Error; err != nil {
return err
}
released := user.Username + "__deleted__" + time.Now().Format("20060102150405.000000000")
if len(released) > 64 {
sum := sha256.Sum256([]byte(user.ID + user.Username))
base := user.Username
if len(base) > 43 {
base = base[:43]
}
released = base + "__deleted__" + hex.EncodeToString(sum[:])[:10]
}
if err := tx.Model(&model.User{}).Where("id = ?", id).Update("username", released).Error; err != nil {
return err
}
return tx.Delete(&model.User{}, "id = ?", id).Error
return withSQLiteBusyRetry(ctx, func() error {
return r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
_ = tx.Unscoped().Where("user_id = ?", id).Delete(&model.RefreshToken{})
_ = tx.Unscoped().Where("user_id = ?", id).Delete(&model.UserPermission{})
_ = tx.Unscoped().Where("user_id = ?", id).Delete(&model.PlayProfile{})
_ = tx.Unscoped().Where("user_id = ?", id).Delete(&model.PlaybackHistory{})
_ = tx.Unscoped().Where("user_id = ?", id).Delete(&model.Favorite{})
_ = tx.Unscoped().Where("user_id = ?", id).Delete(&model.UserDevice{})
return tx.Unscoped().Delete(&model.User{}, "id = ?", id).Error
})
})
}
+46 -10
View File
@@ -17,6 +17,9 @@ package cloud
import (
"context"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"github.com/ShukeBta/MMTL/internal/service/cloud115"
@@ -61,16 +64,16 @@ func (p *openAPI115Provider) List(ctx context.Context, dirID string) ([]FileEntr
if err != nil {
return nil, err
}
for _, f := range files {
out = append(out, FileEntry{
ID: f.FileId,
Name: f.FileName,
IsDir: f.Category == cloud115.TypeDir,
Size: f.FileSize,
MTime: f.Utime,
PickCode: f.PickCode,
})
}
for _, f := range files {
out = append(out, FileEntry{
ID: f.FileId,
Name: f.FileName,
IsDir: f.Category == cloud115.TypeDir,
Size: f.FileSize,
MTime: f.Utime,
PickCode: f.PickCode,
})
}
if len(files) < pageSize {
break
}
@@ -105,6 +108,39 @@ func (p *openAPI115Provider) ResolveWithUA(ctx context.Context, fileRef, ua stri
// OpenClient 暴露底层客户端(token 刷新用)。
func (p *openAPI115Provider) OpenClient() *cloud115.OpenClient { return p.c }
// PutFileNamed 把本地元数据上传到 115 指定父目录(parentCID 为父目录 cid)。
// io.Reader 无法携带文件名,因此走独立的 named 上传接口。将内容落为临时文件后
// 重命名为目标文件名,再交给 115 上传(/open/upload/init 的 file_name 取真实文件名)。
func (p *openAPI115Provider) PutFileNamed(ctx context.Context, parentCID, fileName string, r io.Reader) error {
tmp, err := os.CreateTemp("", "mmtl-upload-*")
if err != nil {
return fmt.Errorf("115: 创建临时文件失败:%w", err)
}
tmpPath := tmp.Name()
defer func() {
_ = tmp.Close()
_ = os.Remove(tmpPath)
}()
if _, err := io.Copy(tmp, r); err != nil {
return fmt.Errorf("115: 写入临时文件失败:%w", err)
}
if err := tmp.Close(); err != nil {
return fmt.Errorf("115: 关闭临时文件失败:%w", err)
}
// 重命名为目标文件名,保证上传到 115 后保留原始文件名
if fileName != "" && fileName != filepath.Base(tmpPath) {
namedPath := filepath.Join(filepath.Dir(tmpPath), fileName)
if err := os.Rename(tmpPath, namedPath); err == nil {
tmpPath = namedPath
}
}
_, err = p.c.Upload(ctx, tmpPath, parentCID, "", "")
if err != nil {
return err
}
return nil
}
// RefreshToken 刷新访问令牌并返回新令牌;refresh_token 失效时返回
// cloud115.IsRefreshTokenDead(err) 为 true 的错误。
func (p *openAPI115Provider) RefreshToken(refreshToken string) (*cloud115.TokenData, error) {
@@ -375,3 +375,57 @@ func TestThrottleCodeHandling(t *testing.T) {
t.Fatal("code 770004 should trigger throttle status")
}
}
func TestRemoteFileDetailRelativePath(t *testing.T) {
rootCID := "3238787832374488117" // 影视库
// 场景 1:115 目录 paths 中只有祖先目录链,不包含自身
d1 := &RemoteFileDetail{
FileId: "3251154147730910635",
FileName: "出包王女",
Paths: []struct {
FileId string `json:"file_id"`
Name string `json:"file_name"`
}{
{FileId: "0", Name: "根目录"},
{FileId: "3238787832374488117", Name: "影视库"},
{FileId: "3238787913223892116", Name: "动漫"},
},
}
if got := d1.RelativePath(rootCID); got != "动漫/出包王女" {
t.Errorf("d1.RelativePath = %q, want %q", got, "动漫/出包王女")
}
// 场景 2:祖先中间目录,自身在 paths 末尾
d2 := &RemoteFileDetail{
FileId: "3238787913223892116",
FileName: "动漫",
Paths: []struct {
FileId string `json:"file_id"`
Name string `json:"file_name"`
}{
{FileId: "0", Name: "根目录"},
{FileId: "3238787832374488117", Name: "影视库"},
{FileId: "3238787913223892116", Name: "动漫"},
},
}
if got := d2.RelativePath(rootCID); got != "动漫" {
t.Errorf("d2.RelativePath = %q, want %q", got, "动漫")
}
// 场景 3:根同步目录自身
d3 := &RemoteFileDetail{
FileId: rootCID,
FileName: "影视库",
Paths: []struct {
FileId string `json:"file_id"`
Name string `json:"file_name"`
}{
{FileId: "0", Name: "根目录"},
{FileId: rootCID, Name: "影视库"},
},
}
if got := d3.RelativePath(rootCID); got != "" {
t.Errorf("d3.RelativePath = %q, want %q", got, "")
}
}
+12 -2
View File
@@ -145,12 +145,15 @@ type RemoteFileDetail struct {
// RelativePath 计算该目录相对于根同步目录(rootCID)的相对路径。
func (d *RemoteFileDetail) RelativePath(rootCID string) string {
if d == nil || len(d.Paths) == 0 {
if d == nil {
return ""
}
if rootCID == "" {
rootCID = "0"
}
if d.FileId == rootCID {
return ""
}
rootIdx := -1
for i, p := range d.Paths {
if p.FileId == rootCID {
@@ -165,16 +168,23 @@ func (d *RemoteFileDetail) RelativePath(rootCID string) string {
} else if len(d.Paths) > 0 && (d.Paths[0].FileId == "0" || d.Paths[0].FileId == "") {
start = 1
}
hasSelf := false
for i := start; i < len(d.Paths); i++ {
if d.Paths[i].FileId == d.FileId {
hasSelf = true
}
name := strings.TrimSpace(d.Paths[i].Name)
if name != "" {
segments = append(segments, name)
}
}
// 若 115 返回的 paths 祖先链未包含当前目录自身,则将其自身目录名 FileName 补在末尾
if !hasSelf && strings.TrimSpace(d.FileName) != "" && d.FileId != rootCID {
segments = append(segments, strings.TrimSpace(d.FileName))
}
return strings.Join(segments, "/")
}
// ─── 下载直链 ──────────────────────────────────────────────────────────────────
type downloadURLData struct {
+347
View File
@@ -0,0 +1,347 @@
// 阿里云 OSS multipart 分片上传(用于 115 元数据上传直传)。
// 使用 115 下发的临时 STS 凭证,将本地文件分片上传到 OSS,并经 complete 回调
// 通知 115 完成落盘。参考 QMediaSync 的 OSSMultipartUploader 实现。
package cloud115
import (
"context"
"errors"
"fmt"
"io"
"os"
"sort"
"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss"
"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss/credentials"
)
const (
defaultMultipartPartSize int64 = 32 * 1024 * 1024
multipartPartAlign int64 = 1024 * 1024
maxMultipartParts int64 = 9999
maxMultipartPartSize int64 = 5 * 1024 * 1024 * 1024
)
type ossMultipartClient interface {
InitiateMultipartUpload(context.Context, *oss.InitiateMultipartUploadRequest, ...func(*oss.Options)) (*oss.InitiateMultipartUploadResult, error)
UploadPart(context.Context, *oss.UploadPartRequest, ...func(*oss.Options)) (*oss.UploadPartResult, error)
ListParts(context.Context, *oss.ListPartsRequest, ...func(*oss.Options)) (*oss.ListPartsResult, error)
CompleteMultipartUpload(context.Context, *oss.CompleteMultipartUploadRequest, ...func(*oss.Options)) (*oss.CompleteMultipartUploadResult, error)
AbortMultipartUpload(context.Context, *oss.AbortMultipartUploadRequest, ...func(*oss.Options)) (*oss.AbortMultipartUploadResult, error)
}
// OSSMultipartUploader 封装 OSS multipart 上传。
type OSSMultipartUploader struct {
client ossMultipartClient
}
// OSSMultipartUploadInput 是 multipart 上传输入。
type OSSMultipartUploadInput struct {
Bucket string
Object string
Callback string
CallbackVar string
FilePath string
FileSize int64
UploadId string
PartSize int64
PartRetryMax int
refreshClient func(context.Context) (ossMultipartClient, error)
}
// OSSMultipartUploadResult 是 multipart 上传后的结果。
type OSSMultipartUploadResult struct {
CallbackResult map[string]any
UploadId string
PartSize int64
TotalParts int
UploadedBytes int64
UploadedParts int
}
// CalculateMultipartPartSize 计算 OSS multipart 分片大小与分片数量。
func CalculateMultipartPartSize(fileSize int64) (int64, int, error) {
if fileSize < 0 {
return 0, 0, fmt.Errorf("文件大小不能为负数:%d", fileSize)
}
partSize := defaultMultipartPartSize
minPartSize := ceilDiv(fileSize, maxMultipartParts)
if minPartSize > partSize {
partSize = roundUp(minPartSize, multipartPartAlign)
}
if partSize > maxMultipartPartSize {
return 0, 0, fmt.Errorf("文件过大,所需分片大小 %d 超过 OSS 上限 %d", partSize, maxMultipartPartSize)
}
totalParts := int(ceilDiv(fileSize, partSize))
if totalParts == 0 {
totalParts = 1
}
if int64(totalParts) > maxMultipartParts {
return 0, 0, fmt.Errorf("分片数量 %d 超过上限 %d", totalParts, maxMultipartParts)
}
return partSize, totalParts, nil
}
// NewOSSMultipartUploader 创建 OSS multipart 上传器。
func NewOSSMultipartUploader(endpoint, accessKeyId, accessKeySecret, securityToken string) *OSSMultipartUploader {
return &OSSMultipartUploader{client: newOSSMultipartClient(endpoint, accessKeyId, accessKeySecret, securityToken)}
}
func newOSSMultipartClient(endpoint, accessKeyId, accessKeySecret, securityToken string) ossMultipartClient {
cfg := oss.LoadDefaultConfig().
WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKeyId, accessKeySecret, securityToken)).
WithRegion("cn-shenzhen").
WithEndpoint(endpoint)
return oss.NewClient(cfg)
}
// UploadFile 上传文件并完成 OSS multipart,返回 complete callback 结果。
func (u *OSSMultipartUploader) UploadFile(ctx context.Context, input OSSMultipartUploadInput) (map[string]any, error) {
result, err := u.UploadFileWithResult(ctx, input)
if err != nil {
return nil, err
}
return result.CallbackResult, nil
}
// UploadFileWithResult 上传文件并返回 multipart 结果。
func (u *OSSMultipartUploader) UploadFileWithResult(ctx context.Context, input OSSMultipartUploadInput) (OSSMultipartUploadResult, error) {
if input.PartRetryMax <= 0 {
input.PartRetryMax = 3
}
partSize := input.PartSize
totalParts := 0
var err error
if partSize <= 0 {
partSize, totalParts, err = CalculateMultipartPartSize(input.FileSize)
if err != nil {
return OSSMultipartUploadResult{}, err
}
} else {
totalParts = int(ceilDiv(input.FileSize, partSize))
}
uploadId := input.UploadId
if uploadId == "" {
initResult, err := u.client.InitiateMultipartUpload(ctx, &oss.InitiateMultipartUploadRequest{
Bucket: oss.Ptr(input.Bucket),
Key: oss.Ptr(input.Object),
RequestCommon: oss.RequestCommon{
Parameters: map[string]string{"sequential": "1"},
},
})
if err != nil {
return OSSMultipartUploadResult{}, fmt.Errorf("初始化 OSS multipart 失败:%w", err)
}
if initResult.UploadId == nil || *initResult.UploadId == "" {
return OSSMultipartUploadResult{}, fmt.Errorf("初始化 OSS multipart 返回空 upload_id")
}
uploadId = *initResult.UploadId
}
existingPartMap := make(map[int32]int64)
existingParts, err := u.ListUploadedParts(ctx, input.Bucket, input.Object, uploadId)
if err == nil {
for _, part := range existingParts {
existingPartMap[part.PartNumber] = part.Size
}
}
file, err := os.Open(input.FilePath)
if err != nil {
return OSSMultipartUploadResult{}, fmt.Errorf("打开待上传文件失败:%w", err)
}
defer file.Close()
var uploadedBytes int64
uploadedParts := 0
completeParts := make([]oss.UploadPart, 0, totalParts)
for partNumber := 1; partNumber <= totalParts; partNumber++ {
offset := int64(partNumber-1) * partSize
length := minInt64(partSize, input.FileSize-offset)
if length < 0 {
length = 0
}
if existingSize, ok := existingPartMap[int32(partNumber)]; ok && existingSize == length {
uploadedBytes += length
uploadedParts++
}
etag, err := u.uploadPartWithRetry(ctx, input, uploadId, int32(partNumber), file, offset, length)
if err != nil {
return OSSMultipartUploadResult{}, err
}
uploadedBytes += length
uploadedParts++
completeParts = append(completeParts, oss.UploadPart{
PartNumber: int32(partNumber),
ETag: oss.Ptr(etag),
})
}
sort.Slice(completeParts, func(i, j int) bool {
return completeParts[i].PartNumber < completeParts[j].PartNumber
})
completeResult, err := u.client.CompleteMultipartUpload(ctx, &oss.CompleteMultipartUploadRequest{
Bucket: oss.Ptr(input.Bucket),
Key: oss.Ptr(input.Object),
UploadId: oss.Ptr(uploadId),
CompleteMultipartUpload: &oss.CompleteMultipartUpload{
Parts: completeParts,
},
Callback: oss.Ptr(input.Callback),
CallbackVar: oss.Ptr(input.CallbackVar),
})
if err != nil {
return OSSMultipartUploadResult{}, fmt.Errorf("完成 OSS multipart 失败:%w", err)
}
return OSSMultipartUploadResult{
CallbackResult: completeResult.CallbackResult,
UploadId: uploadId,
PartSize: partSize,
TotalParts: totalParts,
UploadedBytes: uploadedBytes,
UploadedParts: uploadedParts,
}, nil
}
// ListUploadedParts 查询 OSS 已上传分片。
func (u *OSSMultipartUploader) ListUploadedParts(ctx context.Context, bucket, object, uploadId string) ([]struct {
PartNumber int32
Size int64
}, error) {
parts := []struct {
PartNumber int32
Size int64
}{}
result, err := u.client.ListParts(ctx, &oss.ListPartsRequest{
Bucket: oss.Ptr(bucket),
Key: oss.Ptr(object),
UploadId: oss.Ptr(uploadId),
MaxParts: 1000,
})
if err != nil {
return nil, fmt.Errorf("查询 OSS 已上传分片失败:%w", err)
}
for _, part := range result.Parts {
parts = append(parts, struct {
PartNumber int32
Size int64
}{PartNumber: part.PartNumber, Size: part.Size})
}
return parts, nil
}
func (u *OSSMultipartUploader) uploadPartWithRetry(
ctx context.Context,
input OSSMultipartUploadInput,
uploadId string,
partNumber int32,
file *os.File,
offset, length int64,
) (string, error) {
var lastErr error
for attempt := 0; attempt < input.PartRetryMax; attempt++ {
reader := io.NewSectionReader(file, offset, length)
result, err := u.client.UploadPart(ctx, &oss.UploadPartRequest{
Bucket: oss.Ptr(input.Bucket),
Key: oss.Ptr(input.Object),
PartNumber: partNumber,
UploadId: oss.Ptr(uploadId),
Body: reader,
ContentLength: oss.Ptr(length),
})
if err == nil {
if result.ETag == nil || *result.ETag == "" {
return "", fmt.Errorf("OSS part %d 返回空 ETag", partNumber)
}
return *result.ETag, nil
}
lastErr = err
if attempt < input.PartRetryMax-1 && input.refreshClient != nil {
refreshed, refreshErr := input.refreshClient(ctx)
if refreshErr != nil {
lastErr = refreshErr
continue
}
u.client = refreshed
}
}
return "", fmt.Errorf("上传 OSS part %d 失败:%w", partNumber, lastErr)
}
// ParseCompleteCallbackResult 校验并解析 OSS complete 后的 115 callback 结果。
func ParseCompleteCallbackResult(result map[string]any) (UploadCompleteResult, error) {
if result == nil {
return UploadCompleteResult{}, errors.New("OSS complete callback 结果为空")
}
if state, ok := result["state"].(bool); ok && !state {
return UploadCompleteResult{}, fmt.Errorf("115 callback 返回失败:%s", anyToString(result["message"]))
}
if message := anyToString(result["message"]); message != "" {
return UploadCompleteResult{}, fmt.Errorf("115 callback 返回错误:%s", message)
}
data, ok := result["data"].(map[string]any)
if !ok {
return UploadCompleteResult{}, errors.New("115 callback 缺少 data")
}
complete := UploadCompleteResult{
FileId: anyToString(data["file_id"]),
PickCode: anyToString(data["pick_code"]),
ParentId: anyToString(data["parent_id"]),
Sha1: anyToString(data["sha1"]),
Size: anyToInt64(data["size"]),
Mtime: anyToInt64(data["mtime"]),
}
if complete.FileId == "" || complete.PickCode == "" {
return UploadCompleteResult{}, errors.New("115 callback 缺少 file_id/pick_code")
}
return complete, nil
}
func ceilDiv(n, d int64) int64 {
if d <= 0 {
return 0
}
if n <= 0 {
return 0
}
return (n + d - 1) / d
}
func roundUp(n, align int64) int64 {
if align <= 0 {
return n
}
return ceilDiv(n, align) * align
}
func minInt64(a, b int64) int64 {
if a < b {
return a
}
return b
}
func anyToInt64(v any) int64 {
switch t := v.(type) {
case string:
var n int64
fmt.Sscanf(t, "%d", &n)
return n
case float64:
return int64(t)
case int64:
return t
case int:
return int64(t)
default:
return 0
}
}
func anyToString(v any) string {
if s, ok := v.(string); ok {
return s
}
return fmt.Sprintf("%v", v)
}
+7 -2
View File
@@ -26,10 +26,15 @@ var (
executorOnce sync.Once
)
// GetGlobalExecutor 获取全局队列执行器单例(默认 QPS=2, QPM=120, QPH=6000,保障 115 API 调用安全不超频)。
// GetGlobalExecutor 获取全局队列执行器单例(默认 QPS=3, QPM=200, QPH=12000,保障 115 API 调用安全不超频)。
//
// 历史教训:QPS 提到 8 后,下载换直链接口(/open/ufile/downurl,WAF 重点盯防对象)
// 瞬时突发撞上 115 风控,返回阿里云 405 阻断页(HTTP 405),导致全量同步失败。
// 因此回调到 3——这是经过实测的安全上限:宁慢勿触发风控,一旦 405 冷却 180 秒,
// 整体吞吐反而更低。下载实际走 CDN 不受此限速影响,瓶颈仅在换链环节。
func GetGlobalExecutor() *QueueExecutor {
executorOnce.Do(func() {
globalExecutor = NewQueueExecutor(2, 120, 6000)
globalExecutor = NewQueueExecutor(3, 200, 12000)
})
return globalExecutor
}
+47
View File
@@ -0,0 +1,47 @@
package cloud115
import (
"crypto/sha1"
"encoding/hex"
"io"
"os"
)
// FileSHA1 计算文件完整 SHA1(小写 hex)。
func FileSHA1(path string) (string, error) {
f, err := os.Open(path)
if err != nil {
return "", err
}
defer f.Close()
h := sha1.New()
if _, err := io.Copy(h, f); err != nil {
return "", err
}
return hex.EncodeToString(h.Sum(nil)), nil
}
// FileSHA1Partial 计算文件 [start,end](含)字节区间的 SHA1(小写 hex)。
// 用于 115 上传二次签名按 sign_check 指定的区间重算哈希。
func FileSHA1Partial(path string, start, end int64) (string, error) {
if start < 0 {
start = 0
}
if end < start {
end = start
}
f, err := os.Open(path)
if err != nil {
return "", err
}
defer f.Close()
if _, err := f.Seek(start, io.SeekStart); err != nil {
return "", err
}
length := end - start + 1
h := sha1.New()
if _, err := io.CopyN(h, f, length); err != nil {
return "", err
}
return hex.EncodeToString(h.Sum(nil)), nil
}
+355
View File
@@ -0,0 +1,355 @@
// 115 网盘元数据上传能力:115 开放平台调度 + 阿里云 OSS 直传。
// 参考 QMediaSync 的上传流程实现:
//
// POST /open/upload/init 上传初始化/秒传调度(含二次签名)
// GET /open/upload/get_token 获取 OSS 临时上传凭证(STS)
// OSS multipart 分片直传 + callback 完成
//
// 上传目标父目录为 115 目录 ID(cid),而非路径字符串。
package cloud115
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"strconv"
"strings"
)
// 115 上传状态码。
const (
UploadInitStatusNeedUpload = 1 // 需要真实上传
UploadInitStatusRapidUploaded = 2 // 秒传成功
UploadInitStatusSignFailed = 6 // 签名验证失败
UploadInitStatusNeedSign = 7 // 需要二次签名
UploadInitStatusSignRejected = 8 // 签名认证失败
)
// UploadInitRequest 是 /open/upload/init 的结构化请求。
type UploadInitRequest struct {
FileName string
FileSize int64
ParentFileId string
FileSha1 string
Preid string
PickCode string
TopUpload string
SignKey string
SignVal string
}
// UploadInitResult 是 /open/upload/init 的调度结果。
type UploadInitResult struct {
PickCode string
Status int
FileId string
Target string
Bucket string
Object string
SignKey string
SignCheck string
Callback UploadResultCallBack
}
type uploadScheduleAPIResult struct {
PickCode string `json:"pick_code"`
Status int `json:"status"`
FileId string `json:"file_id"`
Target string `json:"target"`
Version string `json:"version"`
Bucket string `json:"bucket"`
Object string `json:"object"`
SignKey string `json:"sign_key"`
SignCheck string `json:"sign_check"`
Callback json.RawMessage `json:"callback"`
}
// UploadResultCallBack 是 init 返回给 OSS complete 使用的 callback 内容。
type UploadResultCallBack struct {
Callback string `json:"callback"`
CallbackVar string `json:"callback_var"`
}
// UploadToken 是 /open/upload/get_token 返回的 OSS STS 临时凭证。
type UploadToken struct {
Endpoint string `json:"endpoint"`
AccessKeySecret string `json:"AccessKeySecret"`
AccessKeySecrett string `json:"AccessKeySecrett"`
SecurityToken string `json:"SecurityToken"`
Expiration string `json:"Expiration"`
AccessKeyId string `json:"AccessKeyId"`
}
func (token *UploadToken) normalize() {
if token == nil {
return
}
if token.AccessKeySecret == "" {
token.AccessKeySecret = token.AccessKeySecrett
}
}
// UploadCompleteResult 是 OSS complete callback 成功后的远端文件定位结果。
type UploadCompleteResult struct {
FileId string
PickCode string
ParentId string
Sha1 string
Size int64
Mtime int64
}
// SignCheckRange 是 115 二次认证要求的闭区间 [start,end]。
type SignCheckRange struct {
Start int64
End int64
}
// UploadInit 调用 115 上传初始化/秒传调度接口。
func (c *OpenClient) UploadInit(ctx context.Context, input UploadInitRequest) (*UploadInitResult, error) {
params := buildUploadInitForm(input)
resp, err := c.doAuthJSON(ctx, "POST", ProAPIBase+"/open/upload/init", params, 2)
if err != nil {
return nil, err
}
var raw uploadScheduleAPIResult
if err := json.Unmarshal(resp.Data, &raw); err != nil {
return nil, fmt.Errorf("115: 解析 upload/init 结果失败:%w", err)
}
callback, err := decodeUploadCallback(raw.Callback)
if err != nil {
return nil, err
}
return &UploadInitResult{
PickCode: raw.PickCode,
Status: raw.Status,
FileId: raw.FileId,
Target: raw.Target,
Bucket: raw.Bucket,
Object: raw.Object,
SignKey: raw.SignKey,
SignCheck: raw.SignCheck,
Callback: callback,
}, nil
}
func buildUploadInitForm(input UploadInitRequest) map[string]string {
topUpload := input.TopUpload
if topUpload == "" {
topUpload = "0"
}
params := map[string]string{
"file_name": input.FileName,
"file_size": strconv.FormatInt(input.FileSize, 10),
"target": fmt.Sprintf("U_1_%s", input.ParentFileId),
"fileid": input.FileSha1,
"preid": input.Preid,
"topupload": topUpload,
}
if input.PickCode != "" {
params["pick_code"] = input.PickCode
}
if input.SignKey != "" && input.SignVal != "" {
params["sign_key"] = input.SignKey
params["sign_val"] = input.SignVal
}
return params
}
func decodeUploadCallback(raw json.RawMessage) (UploadResultCallBack, error) {
if len(raw) == 0 || string(raw) == "null" {
return UploadResultCallBack{}, nil
}
if raw[0] == '[' {
var callbacks []UploadResultCallBack
if err := json.Unmarshal(raw, &callbacks); err != nil {
return UploadResultCallBack{}, err
}
if len(callbacks) == 0 {
return UploadResultCallBack{}, nil
}
return callbacks[0], nil
}
var callback UploadResultCallBack
if err := json.Unmarshal(raw, &callback); err != nil {
return UploadResultCallBack{}, err
}
return callback, nil
}
func parseSignCheckRange(value string) (SignCheckRange, error) {
parts := strings.Split(value, "-")
if len(parts) != 2 {
return SignCheckRange{}, fmt.Errorf("sign_check 格式错误:%s", value)
}
start, err := strconv.ParseInt(strings.TrimSpace(parts[0]), 10, 64)
if err != nil {
return SignCheckRange{}, err
}
end, err := strconv.ParseInt(strings.TrimSpace(parts[1]), 10, 64)
if err != nil {
return SignCheckRange{}, err
}
if start < 0 || end < start {
return SignCheckRange{}, fmt.Errorf("sign_check 范围非法:%s", value)
}
return SignCheckRange{Start: start, End: end}, nil
}
// GetUploadToken 获取 115 下发的 OSS 临时上传凭证。
func (c *OpenClient) GetUploadToken(ctx context.Context) (*UploadToken, error) {
resp, err := c.doAuthJSON(ctx, "GET", ProAPIBase+"/open/upload/get_token", nil, 2)
if err != nil {
return nil, err
}
var token UploadToken
if err := json.Unmarshal(resp.Data, &token); err != nil {
return nil, fmt.Errorf("115: 解析 get_token 结果失败:%w", err)
}
token.normalize()
return &token, nil
}
// Upload 上传单个本地文件到 115 指定父目录(cid),返回成功后的远端文件信息。
// filePath 必须是落到磁盘的真实文件路径(调用方负责把 io.Reader 落盘为临时文件)。
func (c *OpenClient) Upload(ctx context.Context, filePath, parentCID, signKey, signVal string) (*UploadCompleteResult, error) {
fileSize := fileSizeOf(filePath)
if fileSize < 0 {
return nil, fmt.Errorf("115: 无法获取文件大小:%s", filePath)
}
fileSha1, err := FileSHA1(filePath)
if err != nil {
return nil, fmt.Errorf("115: 计算文件 SHA1 失败:%w", err)
}
preSha1, err := FileSHA1Partial(filePath, 0, 128*1024-1)
if err != nil {
return nil, fmt.Errorf("115: 计算文件前 128 KiB SHA1 失败:%w", err)
}
request := UploadInitRequest{
FileName: baseNameOf(filePath),
FileSize: fileSize,
ParentFileId: parentCID,
FileSha1: fileSha1,
Preid: preSha1,
TopUpload: "0",
SignKey: signKey,
SignVal: signVal,
}
initResult, err := c.UploadInit(ctx, request)
if err != nil {
return nil, fmt.Errorf("115: 上传初始化失败:%w", err)
}
status := initResult.Status
if status == UploadInitStatusNeedSign {
// 二次签名:按 sign_check 指定区间重算 sha1
rng, err := parseSignCheckRange(initResult.SignCheck)
if err != nil {
return nil, err
}
signValue, err := FileSHA1Partial(filePath, rng.Start, rng.End)
if err != nil {
return nil, err
}
request.SignKey = initResult.SignKey
request.SignVal = signValue
initResult, err = c.UploadInit(ctx, request)
if err != nil {
return nil, fmt.Errorf("115: 上传二次签名失败:%w", err)
}
status = initResult.Status
}
switch status {
case UploadInitStatusRapidUploaded:
// 秒传成功
return &UploadCompleteResult{FileId: initResult.FileId, PickCode: initResult.PickCode}, nil
case UploadInitStatusSignFailed:
return nil, fmt.Errorf("115: 签名验证后失败")
case UploadInitStatusSignRejected:
return nil, fmt.Errorf("115: 签名认证失败")
case UploadInitStatusNeedUpload:
// 真实上传:OSS multipart
default:
return &UploadCompleteResult{FileId: initResult.FileId, PickCode: initResult.PickCode}, nil
}
if initResult.Bucket == "" || initResult.Object == "" {
return nil, fmt.Errorf("115: upload/init 缺少 bucket/object 信息")
}
token, err := c.GetUploadToken(ctx)
if err != nil {
return nil, fmt.Errorf("115: 获取上传凭证失败:%w", err)
}
if token == nil || token.Endpoint == "" || token.AccessKeyId == "" || token.AccessKeySecret == "" {
return nil, fmt.Errorf("115: 上传凭证不完整")
}
uploader := NewOSSMultipartUploader(token.Endpoint, token.AccessKeyId, token.AccessKeySecret, token.SecurityToken)
result, err := uploader.UploadFile(ctx, OSSMultipartUploadInput{
Bucket: initResult.Bucket,
Object: initResult.Object,
Callback: initResult.Callback.Callback,
CallbackVar: initResult.Callback.CallbackVar,
FilePath: filePath,
FileSize: fileSize,
refreshClient: func(ctx context.Context) (ossMultipartClient, error) {
refreshed, rerr := c.GetUploadToken(ctx)
if rerr != nil || refreshed == nil {
return nil, rerr
}
return newOSSMultipartClient(refreshed.Endpoint, refreshed.AccessKeyId, refreshed.AccessKeySecret, refreshed.SecurityToken), nil
},
})
if err != nil {
return nil, fmt.Errorf("115: OSS 上传失败:%w", err)
}
complete, err := ParseCompleteCallbackResult(result)
if err != nil {
return nil, err
}
return &complete, nil
}
// MkDir 在 115 的 parentCid 下创建目录,返回新目录 cid。
func (c *OpenClient) MkDir(ctx context.Context, parentCID, name string) (string, error) {
params := map[string]string{
"cname": name,
"pid": parentCID,
}
resp, err := c.doAuthJSON(ctx, "POST", ProAPIBase+"/open/folder/add", params, 2)
if err != nil {
return "", err
}
// /open/folder/add 结构:{ aid, cid, fid, name, pid, ... },单一对象
var r struct {
Cid string `json:"cid"`
}
if err := json.Unmarshal(resp.Data, &r); err != nil {
return "", fmt.Errorf("115: 解析 folder/add 结果失败:%w", err)
}
if r.Cid == "" {
return "", errors.New("115: folder/add 未返回 cid")
}
return r.Cid, nil
}
func fileSizeOf(path string) int64 {
info, err := os.Stat(path)
if err != nil {
return -1
}
if info.IsDir() {
return -1
}
return info.Size()
}
func baseNameOf(path string) string {
s := path
for i := len(s) - 1; i >= 0; i-- {
if s[i] == '/' || s[i] == '\\' {
return s[i+1:]
}
}
return s
}
+94
View File
@@ -0,0 +1,94 @@
package cloud115
import (
"os"
"path/filepath"
"testing"
)
func TestFileSHA1(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "a.txt")
if err := os.WriteFile(path, []byte("hello"), 0o644); err != nil {
t.Fatal(err)
}
sum, err := FileSHA1(path)
if err != nil {
t.Fatal(err)
}
// sha1("hello") = aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d
if sum != "aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d" {
t.Errorf("unexpected sha1: %s", sum)
}
}
func TestFileSHA1Partial(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "b.txt")
// 10 bytes: "0123456789"
if err := os.WriteFile(path, []byte("0123456789"), 0o644); err != nil {
t.Fatal(err)
}
// bytes [2,4] = "234"
sum, err := FileSHA1Partial(path, 2, 4)
if err != nil {
t.Fatal(err)
}
if sum != "0ec09ef9836da03f1add21e3ef607627e687e790" {
t.Errorf("unexpected partial sha1: %s", sum)
}
}
func TestParseSignCheckRange(t *testing.T) {
rng, err := parseSignCheckRange("0-131071")
if err != nil {
t.Fatal(err)
}
if rng.Start != 0 || rng.End != 131071 {
t.Errorf("unexpected range: %+v", rng)
}
if _, err := parseSignCheckRange("bad"); err == nil {
t.Error("expected error for bad range")
}
if _, err := parseSignCheckRange("100-50"); err == nil {
t.Error("expected error for end<start")
}
}
func TestCalculateMultipartPartSize(t *testing.T) {
// small file: 1 MiB -> partSize 32MiB, 1 part
ps, parts, err := CalculateMultipartPartSize(1 << 20)
if err != nil {
t.Fatal(err)
}
if ps != defaultMultipartPartSize {
t.Errorf("partSize=%d, want %d", ps, defaultMultipartPartSize)
}
if parts != 1 {
t.Errorf("parts=%d, want 1", parts)
}
// zero-size -> 1 part
_, parts, err = CalculateMultipartPartSize(0)
if err != nil {
t.Fatal(err)
}
if parts != 1 {
t.Errorf("zero-size parts=%d, want 1", parts)
}
// negative -> error
if _, _, err := CalculateMultipartPartSize(-1); err == nil {
t.Error("expected error for negative size")
}
}
func TestBaseNameOf(t *testing.T) {
if got := baseNameOf("/a/b/file.nfo"); got != "file.nfo" {
t.Errorf("got %s", got)
}
if got := baseNameOf("a\\b\\c.jpg"); got != "c.jpg" {
t.Errorf("got %s", got)
}
if got := baseNameOf("top.txt"); got != "top.txt" {
t.Errorf("got %s", got)
}
}
+4 -4
View File
@@ -43,10 +43,10 @@ func (s *MediaService) DeleteLibrary(ctx context.Context, id string) error {
if err := tx.Unscoped().Where("library_id = ?", id).Delete(&model.Media{}).Error; err != nil {
return err
}
if err := hardDeleteLibraryRoots(ctx, tx, id); err != nil {
return err
}
return tx.Delete(&model.Library{}, "id = ?", id).Error
if err := hardDeleteLibraryRoots(ctx, tx, id); err != nil {
return err
}
return tx.Unscoped().Delete(&model.Library{}, "id = ?", id).Error
})
if err == nil {
s.invalidateMediaCache(ctx)
+41
View File
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
@@ -59,6 +60,46 @@ func (s *MediaService) CreateLibraryWithRootsAndCover(ctx context.Context, name,
return lib, nil
}
// CreateLibrariesPerSubfolder 为 parent 目录下的每个直接子目录各建一个媒体库,
// 媒体库名取子目录名,路径指向该子目录。kind 为空时按子目录名推断类型。
func (s *MediaService) CreateLibrariesPerSubfolder(ctx context.Context, parent, kind, coverURL string) ([]model.Library, error) {
parent = strings.TrimSpace(parent)
if parent == "" {
return nil, errors.New("parent path required")
}
dir, err := resolveAccessibleLibraryPath(parent)
if err != nil {
return nil, err
}
entries, err := os.ReadDir(dir)
if err != nil {
return nil, fmt.Errorf("read directory failed: %w", err)
}
subdirs := make([]string, 0, len(entries))
for _, entry := range entries {
if !entry.IsDir() {
continue
}
if strings.HasPrefix(entry.Name(), ".") {
continue
}
subdirs = append(subdirs, filepath.Join(dir, entry.Name()))
}
if len(subdirs) == 0 {
return nil, errors.New("no subfolders found")
}
created := make([]model.Library, 0, len(subdirs))
for _, subdir := range subdirs {
name := filepath.Base(subdir)
lib, err := s.CreateLibraryWithRootsAndCover(ctx, name, kind, coverURL, []LibraryRootInput{{Path: subdir}})
if err != nil {
return nil, fmt.Errorf("create library for %s: %w", subdir, err)
}
created = append(created, *lib)
}
return created, nil
}
func (s *MediaService) UpdateLibraryCover(ctx context.Context, libraryID, coverURL string) error {
return s.repo.DB.WithContext(ctx).Model(&model.Library{}).Where("id = ?", libraryID).
Update("cover_url", strings.TrimSpace(coverURL)).Error
+2 -17
View File
@@ -10,25 +10,10 @@ import (
const maxRecycleBinRecords = 200
// SoftDelete moves a media row to the recycle bin (gorm soft delete).
// The on-disk file is kept; admins can purge it later.
// SoftDelete 物理删除媒体记录(统一硬删除以降低 SQLite 存储与索引压力)。
func (s *MediaService) SoftDelete(ctx context.Context, id string) error {
media, err := s.repo.Media.FindByID(ctx, id)
if err != nil {
return err
}
if media != nil && isCloudMediaPath(media.Path) {
err := s.repo.DB.WithContext(ctx).Unscoped().Where("id = ?", id).Delete(&model.Media{}).Error
if err == nil {
s.invalidateMediaCache(ctx)
}
return err
}
err = s.repo.DB.WithContext(ctx).Where("id = ?", id).Delete(&model.Media{}).Error
err := s.repo.DB.WithContext(ctx).Unscoped().Where("id = ?", id).Delete(&model.Media{}).Error
if err == nil {
if pruneErr := pruneRecycleBinRows(ctx, s.repo.DB, maxRecycleBinRecords); pruneErr != nil {
return pruneErr
}
s.invalidateMediaCache(ctx)
}
return err
@@ -229,9 +229,9 @@ func (o *OrganizerService) replaceVersions(ctx context.Context, src string, exis
o.log.Warn("organize replace remove existing failed",
zap.String("path", e), zap.Error(err))
}
if o.repo != nil && o.repo.DB != nil {
_ = o.repo.DB.WithContext(ctx).Where("path = ?", e).Delete(&model.Media{}).Error
}
if o.repo != nil && o.repo.DB != nil {
_ = o.repo.DB.WithContext(ctx).Unscoped().Where("path = ?", e).Delete(&model.Media{}).Error
}
}
// Move staged file + sidecars into the final path.
if err := os.Rename(stage, dst); err != nil {
@@ -97,7 +97,7 @@ func (o *OrganizerService) deleteMediaRowForPath(ctx context.Context, path strin
if o == nil || o.repo == nil || o.repo.DB == nil {
return
}
_ = o.repo.DB.WithContext(ctx).Where("path = ?", path).Delete(&model.Media{}).Error
_ = o.repo.DB.WithContext(ctx).Unscoped().Where("path = ?", path).Delete(&model.Media{}).Error
}
func (o *OrganizerService) mediaPathExists(ctx context.Context, path string) bool {
+5 -5
View File
@@ -196,18 +196,18 @@ func (p *PlaybackService) AddToPlaylist(ctx context.Context, playlistID, mediaID
return p.repo.DB.Create(item).Error
}
// RemoveFromPlaylist removes a media item from a playlist (idempotent).
// RemoveFromPlaylist 物理删除播放列表项(幂等)。
func (p *PlaybackService) RemoveFromPlaylist(ctx context.Context, playlistID, mediaID string) error {
return p.repo.DB.
return p.repo.DB.WithContext(ctx).Unscoped().
Where("playlist_id = ? AND media_id = ?", playlistID, mediaID).
Delete(&model.PlaylistItem{}).Error
}
// DeletePlaylist removes a playlist and all of its items.
// DeletePlaylist 物理删除播放列表及其全部条目。
func (p *PlaybackService) DeletePlaylist(ctx context.Context, playlistID string) error {
if err := p.repo.DB.Where("playlist_id = ?", playlistID).
if err := p.repo.DB.WithContext(ctx).Unscoped().Where("playlist_id = ?", playlistID).
Delete(&model.PlaylistItem{}).Error; err != nil {
return err
}
return p.repo.DB.Where("id = ?", playlistID).Delete(&model.Playlist{}).Error
return p.repo.DB.WithContext(ctx).Unscoped().Where("id = ?", playlistID).Delete(&model.Playlist{}).Error
}
+4 -5
View File
@@ -11,13 +11,12 @@ import (
"github.com/ShukeBta/MMTL/internal/model"
)
// RemovePath deletes the media row for a path that has disappeared from disk
// (incremental delete used by the watcher on Remove/Rename events).
// RemovePath 物理删除磁盘上已不存在的媒体记录。
func (s *ScannerService) RemovePath(ctx context.Context, path string) (int64, error) {
if _, err := os.Stat(path); err == nil {
return 0, nil // still exists; nothing to remove
}
res := s.repo.DB.WithContext(ctx).
res := s.repo.DB.WithContext(ctx).Unscoped().
Where("path = ?", path).
Delete(&model.Media{})
if res.Error == nil && res.RowsAffected > 0 {
@@ -55,7 +54,7 @@ func (s *ScannerService) pruneMissingMedia(ctx context.Context, libraryID string
}
stale = append(stale, row.ID)
}
return s.deleteMediaByIDs(ctx, stale, false)
return s.deleteMediaByIDs(ctx, stale, true)
}
func (s *ScannerService) pruneMissingMediaForRoot(ctx context.Context, libraryID, rootID, rootPath string, seen map[string]struct{}) (int64, error) {
@@ -92,7 +91,7 @@ func (s *ScannerService) pruneMissingMediaForRoot(ctx context.Context, libraryID
}
stale = append(stale, row.ID)
}
return s.deleteMediaByIDs(ctx, stale, false)
return s.deleteMediaByIDs(ctx, stale, true)
}
func pathBelongsToRoot(pathValue, rootPath string) bool {
+73 -3
View File
@@ -13,6 +13,7 @@ import (
"os"
"path/filepath"
"strings"
"sync"
"time"
"go.uber.org/zap"
@@ -27,7 +28,12 @@ const (
)
// downloadWorker 下载队列 worker:认领 → 解析直链 → 下载 → 落盘。
//
// 采用「批量认领 + 全局并发限流」:一次认领数个任务,用 StrmService 上的全局信号量
// 限制整个进程「同时换直链+下载」的并发数(与 115 换链风控匹配,见 strmDownloadSemCap),
// 同时让下载充分并行。换链走全局令牌桶(QPS=3)兜底,下载走 CDN 不限速。
func (s *StrmService) downloadWorker(ctx context.Context) {
const claimBatch = 12 // 每次批量认领的任务数
for {
select {
case <-ctx.Done():
@@ -42,7 +48,7 @@ func (s *StrmService) downloadWorker(ctx context.Context) {
sleepContext(ctx, left)
continue
}
tasks, err := s.repo.StrmDownload.ClaimPendingDownload(ctx, 1)
tasks, err := s.repo.StrmDownload.ClaimPendingDownload(ctx, claimBatch)
if err != nil {
s.log.Warn("claim strm download task failed", zap.Error(err))
sleepContext(ctx, 3*time.Second)
@@ -52,9 +58,21 @@ func (s *StrmService) downloadWorker(ctx context.Context) {
sleepContext(ctx, 2*time.Second)
continue
}
// 并发处理本批任务:每个任务先获取全局下载槽位,槽位内部执行换链+下载。
// 信号量与令牌桶双重限速,确保任意时刻并发换链请求不超过安全阈值。
var wg sync.WaitGroup
for i := range tasks {
s.processDownloadTask(ctx, &tasks[i])
wg.Add(1)
go func(i int) {
defer wg.Done()
if !s.acquireDownloadSlot(ctx) {
return
}
defer s.releaseDownloadSlot()
s.processDownloadTask(ctx, &tasks[i])
}(i)
}
wg.Wait()
}
}
@@ -139,7 +157,7 @@ func (s *StrmService) processUploadTask(ctx context.Context, task *model.StrmUpl
}
}
if task.Provider == model.StrmProvider115 {
finish(model.StrmTaskFailed, "115 网盘暂不支持元数据上传")
s.processUpload115(ctx, task)
return
}
acct, err := s.repo.StrmAccount.FindByID(ctx, task.AccountID)
@@ -178,6 +196,48 @@ func (s *StrmService) processUploadTask(ctx context.Context, task *model.StrmUpl
finish(model.StrmTaskDone, "")
}
// processUpload115 115 元数据上传:task.RemotePath 存的是父目录 cid,FileName 为远端文件名。
func (s *StrmService) processUpload115(ctx context.Context, task *model.StrmUploadTask) {
finish := func(status, message string) {
now := time.Now()
task.Status = status
task.Error = message
task.FinishedAt = &now
if err := s.repo.StrmUpload.Update(context.Background(), task); err != nil {
s.log.Warn("update strm upload task failed", zap.Error(err))
}
}
acct, err := s.repo.StrmAccount.FindByID(ctx, task.AccountID)
if err != nil || acct == nil {
finish(model.StrmTaskFailed, "网盘账号不存在")
return
}
provider, err := s.providerFor(ctx, acct)
if err != nil {
s.uploadTaskFailWithRetry(task, err.Error())
return
}
named, ok := provider.(interface {
PutFileNamed(ctx context.Context, parentCID, fileName string, r io.Reader) error
})
if !ok {
finish(model.StrmTaskFailed, "该网盘不支持元数据上传")
return
}
f, err := os.Open(task.LocalPath)
if err != nil {
s.uploadTaskFailWithRetry(task, "打开本地文件失败:"+err.Error())
return
}
if err := named.PutFileNamed(ctx, task.RemotePath, task.FileName, f); err != nil {
_ = f.Close()
s.uploadTaskFailWithRetry(task, "上传失败:"+err.Error())
return
}
_ = f.Close()
finish(model.StrmTaskDone, "")
}
// downloadTaskFailWithRetry 下载失败任务按退避重试,超过上限标记 failed。
func (s *StrmService) downloadTaskFailWithRetry(task *model.StrmDownloadTask, message string) {
if !retryTask(&task.RetryCount, &task.Status, &task.Error, &task.NextTryAt, &task.FinishedAt, message) {
@@ -501,6 +561,16 @@ func (s *StrmService) ClearFinishedDownloadTasks(ctx context.Context) (int64, er
return s.repo.StrmDownload.ClearFinished(ctx)
}
// ClearCanceledDownloadTasks 清空全部已取消的下载记录,返回删除数量。
func (s *StrmService) ClearCanceledDownloadTasks(ctx context.Context) (int64, error) {
return s.repo.StrmDownload.ClearCanceled(ctx)
}
// ClearCanceledUploadTasks 清空全部已取消的上传记录,返回删除数量。
func (s *StrmService) ClearCanceledUploadTasks(ctx context.Context) (int64, error) {
return s.repo.StrmUpload.ClearCanceled(ctx)
}
// RetryAllFailedDownloadTasks 批量重试所有失败下载任务,返回重新入队数量。
func (s *StrmService) RetryAllFailedDownloadTasks(ctx context.Context) (int64, error) {
return s.repo.StrmDownload.RetryAllFailed(ctx)
+53
View File
@@ -90,11 +90,48 @@ type StrmService struct {
running map[string]context.CancelFunc // sync path id -> cancel
oauthSessions map[string]*strm115AuthSession
wafUntil time.Time // 115 风控/限流熔断截止时间(由 mu 保护)
downloadSem chan struct{} // 全局下载并发信号量:限制整个进程同时进行「换直链+下载」的并发数
downloadSemOnce sync.Once
}
// strmWAFCooldown 检测到 115 风控/限流后下载队列的全局冷却时长。
const strmWAFCooldown = 3 * time.Minute
// strmDownloadSemCap 全局同时进行「换直链+下载」的并发上限。
//
// 115 对换直链接口(/open/ufile/downurl)风控极严:过去把全局 QPS 提到 8 或让多
// worker 高并发换链,会瞬时撞上 WAF 返回 405 阻断页并触发 180 秒冷却,反而更慢。
// 因此用信号量把整个进程同时换直链的并发数压到 3,与令牌桶限速共同兜底:
// 宁可下载稍慢,也绝不触发风控。下载本身走 CDN 不限速。
const strmDownloadSemCap = 3
// ensureDownloadSem 惰性初始化全局共享的下载并发信号量。
func (s *StrmService) ensureDownloadSem() {
s.downloadSemOnce.Do(func() {
s.downloadSem = make(chan struct{}, strmDownloadSemCap)
})
}
// acquireDownloadSlot 获取一个下载并发槽位(等待/取消安全)。
func (s *StrmService) acquireDownloadSlot(ctx context.Context) bool {
s.ensureDownloadSem()
select {
case s.downloadSem <- struct{}{}:
return true
case <-ctx.Done():
return false
}
}
// releaseDownloadSlot 释放一个下载并发槽位。
func (s *StrmService) releaseDownloadSlot() {
if s.downloadSem == nil {
return
}
<-s.downloadSem
}
// NewStrmService constructs the STRM service.
func NewStrmService(cfg *config.Config, log *zap.Logger, repos *repository.Container, crypto *CryptoService) *StrmService {
return &StrmService{
@@ -113,6 +150,7 @@ func NewStrmService(cfg *config.Config, log *zap.Logger, repos *repository.Conta
// Start 启动下载/上传队列 worker、定时同步巡检、115 token 刷新与队列清理。
func (s *StrmService) Start(ctx context.Context) {
s.sync115RelayKey(ctx)
s.recoverInterruptedSyncs(ctx)
downloadThreads := s.strmIntSetting(ctx, StrmSettingDownloadThreads, 3)
if downloadThreads < 1 {
downloadThreads = 1
@@ -141,6 +179,21 @@ func (s *StrmService) Start(ctx context.Context) {
zap.Int("upload_threads", uploadThreads))
}
// recoverInterruptedSyncs 在服务启动时自愈重置因服务重启遗留的 running 状态。
func (s *StrmService) recoverInterruptedSyncs(ctx context.Context) {
paths, err := s.repo.StrmSyncPath.List(ctx)
if err == nil {
for i := range paths {
p := &paths[i]
if p.LastSyncStatus == model.StrmSyncRecordRunning {
p.LastSyncStatus = model.StrmSyncRecordCanceled
p.LastSyncMessage = "服务重启,已重置同步状态"
_ = s.repo.StrmSyncPath.Update(ctx, p)
}
}
}
}
func (s *StrmService) Stop() {
s.stopOnce.Do(func() { close(s.stopCh) })
}
+309 -151
View File
@@ -40,11 +40,16 @@ type strmSyncState struct {
seenVideo map[string]bool // "v:"+去掉扩展名的相对路径 → 远端存在该视频
seenMeta map[string]bool // "m:"+相对路径 → 远端存在该元数据
remoteMeta map[string]int64 // 远端元数据大小(上传比对用)
activeDownloadPaths map[string]bool // 本地已在排队/进行的下载任务路径(内存去重)
activeUploadPaths map[string]bool // 本地已在排队/进行的上传任务路径(内存去重)
seenMetaTarget map[string]cloud.FileEntry
seenVideoTarget map[string]cloud.FileEntry
activeDownloadPaths map[string]bool // 本地已在排队/进行的下载任务路径(内存去重)
activeUploadPaths map[string]bool // 本地已在排队/进行的上传任务路径(内存去重)
pendingDownloads []*model.StrmDownloadTask
pendingUploads []*model.StrmUploadTask
dirCache sync.Map // dirID (string) -> relativePath (string)
dirCache sync.Map // dirID (string) -> relativePath (string)
dirPathToID map[string]string // relativePath (string) -> dirID(115 上传父目录寻址用,walk 后构建)
scanIncomplete atomic.Bool // 远端目录树/文件列表本次扫描不完整 → 禁止增量 prune 误删本地文件
}
// StartSync 启动一次同步(异步执行,同一目录同时只允许一个任务)。
@@ -104,15 +109,27 @@ func (s *StrmService) StartSync(ctx context.Context, pathID string, syncType ...
return nil
}
// CancelSync 取消正在进行的同步。
// CancelSync 取消正在进行的同步(若为僵尸运行状态则直接自愈重置)。
func (s *StrmService) CancelSync(ctx context.Context, pathID string) error {
s.mu.Lock()
defer s.mu.Unlock()
cancel, exists := s.running[pathID]
if !exists {
return errors.New("该目录当前没有进行中的同步")
if exists {
delete(s.running, pathID)
}
s.mu.Unlock()
if exists && cancel != nil {
cancel()
}
// 无论内存中是否活跃,确保同步目录状态正确重置为已取消
if p, err := s.repo.StrmSyncPath.FindByID(ctx, pathID); err == nil && p != nil {
if p.LastSyncStatus == model.StrmSyncRecordRunning {
p.LastSyncStatus = model.StrmSyncRecordCanceled
p.LastSyncMessage = "已取消"
_ = s.repo.StrmSyncPath.Update(ctx, p)
}
}
cancel()
return nil
}
@@ -160,15 +177,17 @@ func (s *StrmService) runSync(ctx context.Context, p *model.StrmSyncPath, rec *m
return
}
st := &strmSyncState{
s: s,
ctx: ctx,
p: p,
cfg: cfg,
rec: rec,
syncType: rec.SyncType,
seenVideo: map[string]bool{},
seenMeta: map[string]bool{},
remoteMeta: map[string]int64{},
s: s,
ctx: ctx,
p: p,
cfg: cfg,
rec: rec,
syncType: rec.SyncType,
seenVideo: map[string]bool{},
seenMeta: map[string]bool{},
remoteMeta: map[string]int64{},
seenMetaTarget: map[string]cloud.FileEntry{},
seenVideoTarget: map[string]cloud.FileEntry{},
}
if p.Provider != model.StrmProviderLocal {
acct, err := s.repo.StrmAccount.FindByID(ctx, p.AccountID)
@@ -264,7 +283,21 @@ func (st *strmSyncState) run() error {
}
st.flushPendingDownloads()
st.flushProgress()
if st.cfg.UploadMeta && st.provider != nil && st.p.Provider != model.StrmProvider115 {
if st.cfg.UploadMeta && st.provider != nil {
// 115 上传需要父目录 cid,先用 dirCache 构建「路径 → cid」反向索引
if st.p.Provider == model.StrmProvider115 {
reversed := map[string]string{}
st.dirCache.Range(func(key, value any) bool {
path, ok := value.(string)
if ok && path != "" {
if id, ok2 := key.(string); ok2 {
reversed[path] = id
}
}
return true
})
st.dirPathToID = reversed
}
if err := st.scanLocalMetaForUpload(); err != nil {
return err
}
@@ -423,6 +456,29 @@ func (st *strmSyncState) isMetaExt(ext string) bool {
return false
}
// cleanDirRel 对 115 扁平化拉取的目录相对路径逐段套用目录级文件名清洗,
// 确保与 walkRemote / joinLocalRel(sanitizeRelativePath)使用同一套清洗规则。
// 若不清洗,目录名中的冒号等非法字符会直达 rel,而 seenVideo/seenMeta 的 key
// 与磁盘实际路径不一致,导致 pruneLocal 误删已下载的 strm / 元数据。
// 空 rel(根目录)原样返回。
func cleanDirRel(rel string) string {
if rel == "" {
return ""
}
parts := strings.Split(rel, "/")
out := make([]string, 0, len(parts))
for _, part := range parts {
if part == "" {
continue
}
clean := cleanEntryName(part, true)
if clean != "" && clean != "." && clean != ".." {
out = append(out, clean)
}
}
return strings.Join(out, "/")
}
// walk115Flat 使用 115 开放平台扁平化分页批量拉取机制与目录拓扑缓存(参考 QMediaSync)。
// 极大地降低 API 请求次数并支持毫秒级/秒级增量同步。
func (st *strmSyncState) walk115Flat(open115 *cloud115.OpenClient) error {
@@ -441,11 +497,19 @@ func (st *strmSyncState) walk115Flat(open115 *cloud115.OpenClient) error {
st.s.log.Warn("delete strm dir cache failed", zap.Error(err))
}
} else {
// 增量同步:预加载历史目录缓存
// 增量同步:预加载历史目录缓存(过滤历史一对多塌陷冲突的脏数据以自愈刷新)
cached, err := st.s.repo.StrmDirCache.ListBySyncPathID(ctx, st.p.ID)
if err == nil {
pathCounts := make(map[string]int, len(cached))
for _, item := range cached {
st.dirCache.Store(item.DirID, item.Path)
pathCounts[item.Path]++
}
for _, item := range cached {
// 若同一个 path 对应了多个不同 dir_id,说明包含历史层级塌陷的脏数据,不预加载,让后续步骤重新向 115 获取精确路径
if pathCounts[item.Path] > 1 {
continue
}
st.dirCache.Store(item.DirID, cleanDirRel(item.Path))
}
}
}
@@ -457,6 +521,8 @@ func (st *strmSyncState) walk115Flat(open115 *cloud115.OpenClient) error {
return fmt.Errorf("115: 获取文件列表失败:%w", err)
}
st.updateSyncMessage(fmt.Sprintf("正在拉取远端文件列表 (共 %d 个文件)...", totalCount))
allFiles := make([]cloud115.RemoteFile, 0, totalCount)
allFiles = append(allFiles, firstBatch...)
@@ -484,7 +550,7 @@ func (st *strmSyncState) walk115Flat(open115 *cloud115.OpenClient) error {
}
close(taskCh)
workers := 4
workers := 8
if len(pageTasks) < workers {
workers = len(pageTasks)
}
@@ -549,12 +615,16 @@ func (st *strmSyncState) walk115Flat(open115 *cloud115.OpenClient) error {
var (
pwg sync.WaitGroup
dirWorkers = 4
dirWorkers = 8
doneDirs atomic.Int64
totalDirs = len(pidList)
)
if len(pidList) < dirWorkers {
dirWorkers = len(pidList)
}
st.updateSyncMessage(fmt.Sprintf("正在解析目录树 (0/%d)...", totalDirs))
for i := 0; i < dirWorkers; i++ {
pwg.Add(1)
go func() {
@@ -563,46 +633,58 @@ func (st *strmSyncState) walk115Flat(open115 *cloud115.OpenClient) error {
if ctx.Err() != nil {
return
}
if _, loaded := st.dirCache.Load(pid); loaded {
if n := doneDirs.Add(1); n%20 == 0 || n == int64(totalDirs) {
st.updateSyncMessage(fmt.Sprintf("正在解析目录树 (%d/%d)...", n, totalDirs))
}
continue
}
detail, err := open115.GetFsDetailByCid(ctx, pid)
if err != nil {
st.s.log.Warn("115: 获取目录详情失败", zap.String("pid", pid), zap.Error(err))
continue
}
if detail == nil {
continue
}
// 解析相对路径
relPath := detail.RelativePath(rootCID)
st.dirCache.Store(pid, relPath)
_ = st.s.repo.StrmDirCache.Set(ctx, st.p.ID, pid, relPath)
// 目录详情解析失败会导致下游文件 rel 无法还原真实父路径,
// seen key 与磁盘路径对不上,增量 prune 会误删本地文件,标记本次扫描不完整。
st.scanIncomplete.Store(true)
} else if detail != nil {
// 解析相对路径
relPath := cleanDirRel(detail.RelativePath(rootCID))
st.dirCache.Store(pid, relPath)
_ = st.s.repo.StrmDirCache.Set(ctx, st.p.ID, pid, relPath)
// 顺便解析并缓存 detail.Paths 中包含的中间各层级目录
for _, ancestor := range detail.Paths {
if ancestor.FileId == "0" || ancestor.FileId == rootCID {
continue
}
if _, loaded := st.dirCache.Load(ancestor.FileId); !loaded {
subDetail := &cloud115.RemoteFileDetail{
FileId: ancestor.FileId,
Paths: nil,
// 顺便解析并缓存 detail.Paths 中包含的中间各层级目录
for _, ancestor := range detail.Paths {
if ancestor.FileId == "0" || ancestor.FileId == rootCID {
continue
}
for _, p := range detail.Paths {
subDetail.Paths = append(subDetail.Paths, p)
if p.FileId == ancestor.FileId {
break
if _, loaded := st.dirCache.Load(ancestor.FileId); !loaded {
subDetail := &cloud115.RemoteFileDetail{
FileId: ancestor.FileId,
FileName: ancestor.Name,
Paths: nil,
}
for _, p := range detail.Paths {
subDetail.Paths = append(subDetail.Paths, p)
if p.FileId == ancestor.FileId {
break
}
}
ancestorRel := cleanDirRel(subDetail.RelativePath(rootCID))
st.dirCache.Store(ancestor.FileId, ancestorRel)
_ = st.s.repo.StrmDirCache.Set(ctx, st.p.ID, ancestor.FileId, ancestorRel)
}
ancestorRel := subDetail.RelativePath(rootCID)
st.dirCache.Store(ancestor.FileId, ancestorRel)
_ = st.s.repo.StrmDirCache.Set(ctx, st.p.ID, ancestor.FileId, ancestorRel)
}
}
if n := doneDirs.Add(1); n%10 == 0 || n == int64(totalDirs) {
st.updateSyncMessage(fmt.Sprintf("正在解析目录树 (%d/%d)...", n, totalDirs))
}
}
}()
}
pwg.Wait()
}
st.updateSyncMessage(fmt.Sprintf("正在生成 STRM 与同步文件 (共 %d 个)...", len(allFiles)))
// 5. 分类处理所有文件
for _, f := range allFiles {
if ctx.Err() != nil {
@@ -614,8 +696,13 @@ func (st *strmSyncState) walk115Flat(open115 *cloud115.OpenClient) error {
rel = cleanName
} else {
if parentVal, ok := st.dirCache.Load(f.Pid); ok && parentVal.(string) != "" {
rel = parentVal.(string) + "/" + cleanName
rel = cleanDirRel(parentVal.(string)) + "/" + cleanName
} else {
// 父目录不在目录缓存(目录详情先前解析失败),无法还原真实相对路径。
// 该文件会落到根/错误路径,seen key 与磁盘路径不符,增量 prune 会误删,标记扫描不完整。
if st.syncType == model.StrmSyncTypeIncremental {
st.scanIncomplete.Store(true)
}
rel = cleanName
}
}
@@ -651,6 +738,18 @@ func (st *strmSyncState) handleVideo(entry cloud.FileEntry, rel, ext string) {
return
}
st.mu.Lock()
if st.seenVideoTarget == nil {
st.seenVideoTarget = map[string]cloud.FileEntry{}
}
if _, exists := st.seenVideoTarget[target]; exists {
st.mu.Unlock()
st.touchProgress()
return
}
st.seenVideoTarget[target] = entry
st.mu.Unlock()
// 增量同步模式快速检查:本地 strm 文件存在、非空且修改时间与远端 mtime 一致,直接跳过无需读磁盘
if st.syncType == model.StrmSyncTypeIncremental && entry.MTime > 0 {
if info, err := os.Stat(target); err == nil && info.Size() > 0 && info.ModTime().Unix() == entry.MTime {
@@ -800,6 +899,20 @@ func (st *strmSyncState) handleMeta(entry cloud.FileEntry, rel, ext string) {
if err != nil {
return
}
st.mu.Lock()
if st.seenMetaTarget == nil {
st.seenMetaTarget = map[string]cloud.FileEntry{}
}
if _, exists := st.seenMetaTarget[target]; exists {
// 该本地目标路径在当前批次中已被处理(存在同名/重名冲突),直接忽略重复项,避免多份不同大小的文件在本地交替覆盖导致增量死循环
st.mu.Unlock()
st.touchProgress()
return
}
st.seenMetaTarget[target] = entry
st.mu.Unlock()
if info, err := os.Stat(target); err == nil && info.Size() == entry.Size {
st.touchProgress()
return
@@ -901,124 +1014,124 @@ func (st *strmSyncState) walkLocalSource() error {
if err != nil {
return nil
}
target, err := joinLocalRel(st.p.LocalPath, relSansExt+".strm")
if err != nil {
return nil
}
mTime := info.ModTime()
if st.syncType == model.StrmSyncTypeIncremental {
if tInfo, err := os.Stat(target); err == nil && tInfo.Size() > 0 && tInfo.ModTime().Unix() == mTime.Unix() {
st.mu.Lock()
st.rec.Skipped++
st.mu.Unlock()
st.touchProgress()
return nil
}
}
if data, err := os.ReadFile(target); err == nil && string(data) == content {
_ = os.Chtimes(target, mTime, mTime)
target, err := joinLocalRel(st.p.LocalPath, relSansExt+".strm")
if err != nil {
return nil
}
mTime := info.ModTime()
if st.syncType == model.StrmSyncTypeIncremental {
if tInfo, err := os.Stat(target); err == nil && tInfo.Size() > 0 && tInfo.ModTime().Unix() == mTime.Unix() {
st.mu.Lock()
st.rec.Skipped++
st.mu.Unlock()
st.touchProgress()
return nil
}
if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
return nil
}
tmp := target + ".tmp"
if err := os.WriteFile(tmp, []byte(content), 0o644); err == nil {
_ = os.Rename(tmp, target)
_ = os.Chtimes(target, mTime, mTime)
} else {
_ = os.Remove(tmp)
}
}
if data, err := os.ReadFile(target); err == nil && string(data) == content {
_ = os.Chtimes(target, mTime, mTime)
st.mu.Lock()
st.rec.NewStrm++
st.rec.Skipped++
st.mu.Unlock()
st.touchProgress()
return nil
}
if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
return nil
}
tmp := target + ".tmp"
if err := os.WriteFile(tmp, []byte(content), 0o644); err == nil {
_ = os.Rename(tmp, target)
_ = os.Chtimes(target, mTime, mTime)
} else {
_ = os.Remove(tmp)
}
st.mu.Lock()
st.rec.NewStrm++
st.mu.Unlock()
st.touchProgress()
return nil
})
}
// scanLocalMetaForUpload 扫描本地元数据,与远端比对后入上传队列。
func (st *strmSyncState) scanLocalMetaForUpload() error {
defer st.flushPendingUploads()
if st.activeUploadPaths == nil {
if active, err := st.s.repo.StrmUpload.GetActiveLocalPathMap(st.ctx, st.p.ID); err == nil {
st.activeUploadPaths = active
} else {
st.activeUploadPaths = map[string]bool{}
}
// scanLocalMetaForUpload 扫描本地元数据,与远端比对后入上传队列。
func (st *strmSyncState) scanLocalMetaForUpload() error {
defer st.flushPendingUploads()
if st.activeUploadPaths == nil {
if active, err := st.s.repo.StrmUpload.GetActiveLocalPathMap(st.ctx, st.p.ID); err == nil {
st.activeUploadPaths = active
} else {
st.activeUploadPaths = map[string]bool{}
}
localRoot := filepath.Clean(st.p.LocalPath)
return filepath.WalkDir(localRoot, func(path string, d os.DirEntry, err error) error {
if err != nil {
return nil
}
if path == localRoot {
return nil
}
select {
case <-st.ctx.Done():
return st.ctx.Err()
default:
}
if d.IsDir() {
return nil
}
rel, err := filepath.Rel(localRoot, path)
if err != nil {
return nil
}
rel = filepath.ToSlash(rel)
ext := strings.ToLower(filepath.Ext(rel))
if !st.isMetaExt(ext) {
return nil
}
info, err := d.Info()
if err != nil {
return nil
}
st.mu.Lock()
_, exists := st.remoteMeta["m:"+rel]
st.mu.Unlock()
if exists {
// 网盘端已存在该元数据文件,跳过上传
return nil
}
st.mu.Lock()
if st.activeUploadPaths != nil && st.activeUploadPaths[path] {
st.mu.Unlock()
return nil
}
if st.activeUploadPaths != nil {
st.activeUploadPaths[path] = true
}
st.mu.Unlock()
task := &model.StrmUploadTask{
SyncPathID: st.p.ID,
AccountID: st.p.AccountID,
Provider: st.p.Provider,
FileName: filepath.Base(rel),
LocalPath: path,
RemotePath: st.remoteUploadPath(rel),
Size: info.Size(),
Status: model.StrmTaskPending,
}
st.mu.Lock()
st.pendingUploads = append(st.pendingUploads, task)
shouldFlush := len(st.pendingUploads) >= 100
st.rec.Uploaded++
st.mu.Unlock()
if shouldFlush {
st.flushPendingUploads()
}
return nil
})
}
localRoot := filepath.Clean(st.p.LocalPath)
return filepath.WalkDir(localRoot, func(path string, d os.DirEntry, err error) error {
if err != nil {
return nil
}
if path == localRoot {
return nil
}
select {
case <-st.ctx.Done():
return st.ctx.Err()
default:
}
if d.IsDir() {
return nil
}
rel, err := filepath.Rel(localRoot, path)
if err != nil {
return nil
}
rel = filepath.ToSlash(rel)
ext := strings.ToLower(filepath.Ext(rel))
if !st.isMetaExt(ext) {
return nil
}
info, err := d.Info()
if err != nil {
return nil
}
st.mu.Lock()
_, exists := st.remoteMeta["m:"+rel]
st.mu.Unlock()
if exists {
// 网盘端已存在该元数据文件,跳过上传
return nil
}
st.mu.Lock()
if st.activeUploadPaths != nil && st.activeUploadPaths[path] {
st.mu.Unlock()
return nil
}
if st.activeUploadPaths != nil {
st.activeUploadPaths[path] = true
}
st.mu.Unlock()
task := &model.StrmUploadTask{
SyncPathID: st.p.ID,
AccountID: st.p.AccountID,
Provider: st.p.Provider,
FileName: filepath.Base(rel),
LocalPath: path,
RemotePath: st.uploadRemoteTarget(rel),
Size: info.Size(),
Status: model.StrmTaskPending,
}
st.mu.Lock()
st.pendingUploads = append(st.pendingUploads, task)
shouldFlush := len(st.pendingUploads) >= 100
st.rec.Uploaded++
st.mu.Unlock()
if shouldFlush {
st.flushPendingUploads()
}
return nil
})
}
// remoteUploadPath 远端元数据目标路径 = 同步目录远端根 + 相对路径。
func (st *strmSyncState) remoteUploadPath(rel string) string {
@@ -1029,6 +1142,31 @@ func (st *strmSyncState) remoteUploadPath(rel string) string {
return root + "/" + rel
}
// uploadRemoteTarget 返回上传任务的目标远端描述。
// - 115:返回父目录 cid(供 PutFileNamed 定位),基于 dirPathToID 把父目录相对路径映射到 cid。
// - 网盘桥接(clouddrive2/openlist):返回完整远端路径。
func (st *strmSyncState) uploadRemoteTarget(rel string) string {
if st.p.Provider == model.StrmProvider115 {
dir := rel
if idx := strings.LastIndexByte(dir, '/'); idx >= 0 {
dir = dir[:idx]
} else {
dir = ""
}
if dir == "" {
// 文件在同步根目录下,父目录即 115 同步根目录 ID
return st.p.RemotePath
}
if cid, ok := st.dirPathToID[dir]; ok && cid != "" {
return cid
}
// 父目录未在缓存中(父目录可能本次未扫描到),降级为用户配置的同步根 cid,
// 由上传端尽力处理(可能失败记日志,不影响下载)。
return st.p.RemotePath
}
return st.remoteUploadPath(rel)
}
// taskExists 检查是否已有同目录、同目标的进行中/已完成任务(避免重复入队)。
func (st *strmSyncState) taskExists(kind, syncPathID, localPath string) bool {
ctx := st.ctx
@@ -1044,6 +1182,14 @@ func (st *strmSyncState) taskExists(kind, syncPathID, localPath string) bool {
// pruneLocal 清理本地多余 .strm 与元数据(远端已不存在),可选删除空目录。
func (st *strmSyncState) pruneLocal() error {
// 增量同步保护:本次远端扫描不完整(目录详情解析失败 / 文件父路径降级)时,
// seenVideo/seenMeta 覆盖不全,按"远端不存在"清理会误删刚下载或已存在的本地文件,
// 进而触发"下次增量重新下载"的循环。此时跳过清理,仅做进度落库。
if st.syncType == model.StrmSyncTypeIncremental && st.scanIncomplete.Load() {
st.s.log.Warn("strm 增量同步跳过清理:本次远端扫描不完整,prune 已禁用",
zap.String("path_id", st.p.ID))
return nil
}
localRoot := filepath.Clean(st.p.LocalPath)
var dirs []string
err := filepath.WalkDir(localRoot, func(path string, d os.DirEntry, err error) error {
@@ -1129,6 +1275,18 @@ func (st *strmSyncState) flushProgress() {
}
}
// updateSyncMessage 实时更新同步阶段提示信息,让前端界面清晰了解当前进度。
func (st *strmSyncState) updateSyncMessage(msg string) {
st.mu.Lock()
st.rec.Message = msg
st.p.LastSyncMessage = msg
rec := *st.rec
p := *st.p
st.mu.Unlock()
_ = st.s.repo.StrmSyncRecord.Update(st.ctx, &rec)
_ = st.s.repo.StrmSyncPath.Update(st.ctx, &p)
}
// ─── 定时同步巡检 ──────────────────────────────────────────────────────────────
func (s *StrmService) cronLoop(ctx context.Context) {
+76 -2
View File
@@ -554,8 +554,82 @@ func TestWalkRemoteConcurrent(t *testing.T) {
}
wg.Wait()
if claimedCount != 200 {
t.Fatalf("expected all 200 tasks claimed, got %d", claimedCount)
if claimedCount != 200 {
t.Fatalf("expected all 200 tasks claimed, got %d", claimedCount)
}
}
// TestStrmDuplicateFileConflictResolution 测试远端存在多个同名不同大小文件时,本地确定性仲裁,避免增量死循环
func TestStrmDuplicateFileConflictResolution(t *testing.T) {
svc := testStrmService(t)
localDir := t.TempDir()
p := &model.StrmSyncPath{
Base: model.Base{ID: "dup-test-path"},
Provider: model.StrmProvider115,
RemotePath: "root",
LocalPath: localDir,
DownloadMeta: true,
}
st := &strmSyncState{
s: svc,
ctx: context.Background(),
p: p,
cfg: &strmPathConfig{DownloadMeta: true, MetaExt: []string{"nfo"}},
rec: &model.StrmSyncRecord{},
seenMeta: map[string]bool{},
remoteMeta: map[string]int64{},
seenMetaTarget: map[string]cloud.FileEntry{},
seenVideoTarget: map[string]cloud.FileEntry{},
}
// 模拟远端同目录下存在两个同名不同大小的 nfo 文件 (115 历史重复上传)
// entry1: 较早文件 (MTime: 1000, Size: 100)
entry1 := cloud.FileEntry{ID: "f1", Name: "test.nfo", Size: 100, MTime: 1000, PickCode: "p1"}
// entry2: 较新文件 (MTime: 2000, Size: 200)
entry2 := cloud.FileEntry{ID: "f2", Name: "test.nfo", Size: 200, MTime: 2000, PickCode: "p2"}
// 第一次全量处理:两者都在列表中
st.handleMeta(entry1, "test.nfo", ".nfo")
st.handleMeta(entry2, "test.nfo", ".nfo")
st.flushPendingDownloads()
// 验证仲裁结果:最终只产生 1 个下载任务,且使用的是首个匹配项 (Size 100/p1)
tasks, _, err := svc.repo.StrmDownload.List(context.Background(), "", 1, 10)
if err != nil {
t.Fatal(err)
}
if len(tasks) != 1 {
t.Fatalf("expected 1 download task after conflict resolution, got %d", len(tasks))
}
if tasks[0].Size != 100 || tasks[0].RemoteRef != "p1" {
t.Fatalf("expected task with size 100/p1, got size=%d ref=%s", tasks[0].Size, tasks[0].RemoteRef)
}
// 模拟该任务下载落盘完成
writeFile(t, filepath.Join(localDir, "test.nfo"), strings.Repeat("x", 100))
// 第二次增量同步:两者再次依次扫描
st2 := &strmSyncState{
s: svc,
ctx: context.Background(),
p: p,
cfg: &strmPathConfig{DownloadMeta: true, MetaExt: []string{"nfo"}},
rec: &model.StrmSyncRecord{},
seenMeta: map[string]bool{},
remoteMeta: map[string]int64{},
seenMetaTarget: map[string]cloud.FileEntry{},
seenVideoTarget: map[string]cloud.FileEntry{},
}
st2.handleMeta(entry1, "test.nfo", ".nfo")
st2.handleMeta(entry2, "test.nfo", ".nfo")
st2.flushPendingDownloads()
// 验证:不会新增任何下载任务,NewMeta 为 0,增量跳过
if st2.rec.NewMeta != 0 {
t.Fatalf("expected 0 new meta on incremental sync, got %d", st2.rec.NewMeta)
}
}
+3
View File
@@ -102,6 +102,9 @@ export const libraryAPI = {
createWithRoots: (name: string, type: string, roots: LibraryRootInput[], coverURL = '') =>
api.post<Library>('/libraries', { name, type, roots, cover_url: coverURL }).then((r) => r.data),
createPerSubfolder: (parentPath: string, type: string, coverURL = '') =>
api.post<{ libraries: Library[] }>('/libraries', { path: parentPath, type, cover_url: coverURL, create_per_subfolder: true }).then((r) => r.data),
update: (id: string, payload: { cover_url: string }) =>
api.patch<Library>(`/libraries/${id}`, payload).then((r) => r.data),
+6
View File
@@ -143,6 +143,9 @@ export const strmAPI = {
clearFinishedDownloads: () =>
api.post<{ deleted: number }>('/admin/strm/downloads/clear-finished').then((r) => r.data),
clearCanceledDownloads: () =>
api.post<{ deleted: number }>('/admin/strm/downloads/clear-canceled').then((r) => r.data),
retryFailedDownloads: () =>
api.post<{ retried: number }>('/admin/strm/downloads/retry-failed').then((r) => r.data),
@@ -162,6 +165,9 @@ export const strmAPI = {
cancelPendingUploads: () =>
api.post<{ canceled: number }>('/admin/strm/uploads/cancel-pending').then((r) => r.data),
clearCanceledUploads: () =>
api.post<{ deleted: number }>('/admin/strm/uploads/clear-canceled').then((r) => r.data),
retryUpload: (id: string) =>
api.post(`/admin/strm/uploads/${id}/retry`).then((r) => r.data),
}
+2
View File
@@ -13,9 +13,11 @@ export function AdminLibraryPanel() {
type={createForm.type}
coverURL={createForm.coverURL}
roots={createForm.roots}
createPerSubfolder={createForm.createPerSubfolder}
onNameChange={createForm.setName}
onTypeChange={createForm.setType}
onCoverURLChange={createForm.setCoverURL}
onCreatePerSubfolderChange={createForm.setCreatePerSubfolder}
onRootChange={createForm.updateRoot}
onAddRoot={createForm.addRoot}
onRemoveRoot={createForm.removeRoot}
+26 -6
View File
@@ -9,9 +9,11 @@ type CreateFormProps = {
type: string
coverURL: string
roots: RootDraft[]
createPerSubfolder: boolean
onNameChange: (value: string) => void
onTypeChange: (value: string) => void
onCoverURLChange: (value: string) => void
onCreatePerSubfolderChange: (value: boolean) => void
onRootChange: (index: number, patch: Partial<RootDraft>) => void
onAddRoot: () => void
onRemoveRoot: (index: number) => void
@@ -23,9 +25,11 @@ export function AdminLibraryCreateForm({
type,
coverURL,
roots,
createPerSubfolder,
onNameChange,
onTypeChange,
onCoverURLChange,
onCreatePerSubfolderChange,
onRootChange,
onAddRoot,
onRemoveRoot,
@@ -50,9 +54,9 @@ export function AdminLibraryCreateForm({
<>
<form onSubmit={onSubmit} className="glass-panel grid gap-3 md:grid-cols-4">
<input
required
required={!createPerSubfolder}
className="input-base"
placeholder="名称"
placeholder={createPerSubfolder ? '父级媒体库名(批量模式忽略)' : '名称'}
value={name}
onChange={(e) => onNameChange(e.target.value)}
/>
@@ -81,15 +85,31 @@ export function AdminLibraryCreateForm({
onRemove={onRemoveRoot}
/>
))}
<button type="button" className="inline-flex items-center gap-2 rounded-lg border px-3 py-2 text-sm" onClick={onAddRoot}>
<Plus size={16} /> 添加路径
</button>
{!createPerSubfolder && (
<button type="button" className="inline-flex items-center gap-2 rounded-lg border px-3 py-2 text-sm" onClick={onAddRoot}>
<Plus size={16} /> 添加路径
</button>
)}
</div>
<p className="md:col-span-4 -mt-2 text-xs text-sand-500">
支持直接点选或手动输入;名称和类型与现有媒体库一致时,会自动把这里填写的路径追加到该媒体库。
</p>
<label className="md:col-span-4 flex items-center gap-2 text-sm text-ink-100">
<input
type="checkbox"
className="h-4 w-4 accent-brand-400"
checked={createPerSubfolder}
onChange={(e) => onCreatePerSubfolderChange(e.target.checked)}
/>
<span>按目录下每个子文件夹各建一个媒体库(媒体库名取子文件夹名)</span>
</label>
{createPerSubfolder && (
<p className="md:col-span-4 -mt-2 text-xs text-sand-500">
批处理模式:仅取上方第一个路径作为父级目录,会为其中每个子文件夹分别创建媒体库,可自选类型用于整体推断。
</p>
)}
<button type="submit" className="neon-button md:col-span-4">
新建 / 追加路径
{createPerSubfolder ? '按目录批量创建' : '新建 / 追加路径'}
</button>
</form>
+13
View File
@@ -127,6 +127,19 @@ function StrmQueuePanel({ kind }: { kind: 'download' | 'upload' }) {
'border-amber-200 text-amber-600 hover:bg-amber-50',
cancelAllPendingAction,
)
if (filter === 'canceled')
return batchBtn(
'清空已取消记录',
'trash',
'border-gray-200 text-rose-500 hover:bg-rose-50',
() =>
runBatch(
isDownload
? () => strmAPI.clearCanceledDownloads()
: () => strmAPI.clearCanceledUploads(),
`确定清空所有已取消的${isDownload ? '下载' : '上传'}记录?`,
),
)
if (!isDownload) return null
if (filter === 'done')
return batchBtn(
+20 -6
View File
@@ -32,20 +32,32 @@ function useCreateLibraryForm(refresh: () => Promise<void>) {
const [roots, setRoots] = useState<RootDraft[]>([emptyRootDraft()])
const [type, setType] = useState('movie')
const [coverURL, setCoverURL] = useState('')
const [createPerSubfolder, setCreatePerSubfolder] = useState(false)
const handleCreate = async (e: FormEvent) => {
e.preventDefault()
try {
const payload = createRootPayload(roots)
if (payload.length === 0) {
toast.error('请至少填写一个路径')
return
if (createPerSubfolder) {
const parentPath = roots[0]?.path?.trim()
if (!parentPath) {
toast.error('请先选择或填写父级目录')
return
}
const { libraries } = await libraryAPI.createPerSubfolder(parentPath, type, coverURL.trim())
toast.success(`已按目录创建 ${libraries.length} 个媒体库`)
} else {
const payload = createRootPayload(roots)
if (payload.length === 0) {
toast.error('请至少填写一个路径')
return
}
await libraryAPI.createWithRoots(name, type, payload, coverURL.trim())
toast.success('媒体库已保存')
}
await libraryAPI.createWithRoots(name, type, payload, coverURL.trim())
toast.success('媒体库已保存')
setName('')
setRoots([emptyRootDraft()])
setCoverURL('')
setCreatePerSubfolder(false)
await refresh()
} catch (err: unknown) {
toast.error(apiErrorMessage(err, '创建失败'))
@@ -61,9 +73,11 @@ function useCreateLibraryForm(refresh: () => Promise<void>) {
type,
coverURL,
roots,
createPerSubfolder,
setName,
setType,
setCoverURL,
setCreatePerSubfolder,
updateRoot,
addRoot: () => setRoots((prev) => [...prev, emptyRootDraft()]),
removeRoot: (index: number) => setRoots((prev) => (prev.length <= 1 ? prev : prev.filter((_, i) => i !== index))),