Compare commits

..

3 Commits

13 changed files with 279 additions and 79 deletions
+18 -7
View File
@@ -660,14 +660,25 @@ func streamHandler(svc *service.Container) gin.HandlerFunc {
}
return
}
target, err := svc.EmbyRemote.WebStreamURL(ctx, acct, remoteID)
if err != nil {
c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()})
target, err := svc.EmbyRemote.WebStreamURL(ctx, acct, remoteID)
if err != nil {
c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()})
return
}
// 现代浏览器在 HTTPS 页面中请求不安全源(HTTP 视频流)会直接报 Mixed Content 拦截导致播放失败。
// 仅当当前前端请求为 HTTPS 且远程直连目标为 HTTP 时,自动降级通过本机反向代理传输流,避免播放被浏览器阻断;
// 其它场景(HTTP 页面访问 HTTP/HTTPS,或 HTTPS 访问 HTTPS)继续 302 直连,最大化节省服务器带宽与流量。
if requestIsHTTPS(c) && strings.HasPrefix(strings.ToLower(target), "http://") {
if err := svc.Emby.ProxyRemoteVideoStream(ctx, c.Writer, c.Request, mountID, remoteID); err != nil {
if !c.Writer.Written() {
c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()})
}
}
return
}
setRedirectNoStoreHeaders(c)
c.Redirect(http.StatusFound, target)
return
}
setRedirectNoStoreHeaders(c)
c.Redirect(http.StatusFound, target)
return
}
m, err := svc.Media.GetMedia(ctx, id)
if err != nil || m == nil || !mediaVisibleForRequest(c, svc, m) {
+26 -20
View File
@@ -123,35 +123,41 @@ func (p *openAPI115Provider) ResolveBatch(ctx context.Context, fileRefs []string
// OpenClient 暴露底层客户端(token 刷新用)。
func (p *openAPI115Provider) OpenClient() *cloud115.OpenClient { return p.c }
// PutLocalFile 直接上传本地文件,避免通过 io.Reader 复制临时文件产生的磁盘开销与并发重命名碰撞。
func (p *openAPI115Provider) PutLocalFile(ctx context.Context, parentCID, localPath string) error {
_, err := p.c.Upload(ctx, localPath, parentCID, "", "")
return err
}
// PutFileNamed 把本地元数据上传到 115 指定父目录(parentCID 为父目录 cid)。
// io.Reader 无法携带文件名,因此走独立的 named 上传接口。将内容落为临时文件后
// 重命名为目标文件名,再交给 115 上传(/open/upload/init 的 file_name 取真实文件名)。
// 为防止多并发上传线程在同一临时目录下发生同名文件(如 poster.jpg)碰撞覆盖与误删,
// 为每个上传任务分配专属临时子目录。
func (p *openAPI115Provider) PutFileNamed(ctx context.Context, parentCID, fileName string, r io.Reader) error {
tmp, err := os.CreateTemp("", "mebox-upload-*")
tmpDir, err := os.MkdirTemp("", "mebox-upload-*")
if err != nil {
return fmt.Errorf("115: 创建临时目录失败:%w", err)
}
defer func() {
_ = os.RemoveAll(tmpDir)
}()
safeName := filepath.Base(fileName)
if safeName == "" || safeName == "." {
safeName = "file"
}
tmpPath := filepath.Join(tmpDir, safeName)
dst, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644)
if err != nil {
return fmt.Errorf("115: 创建临时文件失败:%w", err)
}
tmpPath := tmp.Name()
defer func() {
_ = tmp.Close()
_ = os.Remove(tmpPath)
}()
if _, err := io.Copy(tmp, r); err != nil {
if _, err := io.Copy(dst, r); err != nil {
_ = dst.Close()
return fmt.Errorf("115: 写入临时文件失败:%w", err)
}
if err := tmp.Close(); err != nil {
if err := dst.Close(); err != nil {
return fmt.Errorf("115: 关闭临时文件失败:%w", err)
}
// 重命名为目标文件名,保证上传到 115 后保留原始文件名。
// 重命名失败必须 fail fast:静默用随机临时名上传会导致 115 上的文件名
// 变成 mebox-upload-xxx,破坏元数据文件名契约。
if fileName != "" && fileName != filepath.Base(tmpPath) {
namedPath := filepath.Join(filepath.Dir(tmpPath), fileName)
if err := os.Rename(tmpPath, namedPath); err != nil {
return fmt.Errorf("115: 重命名临时文件为 %s 失败:%w", fileName, err)
}
tmpPath = namedPath
}
_, err = p.c.Upload(ctx, tmpPath, parentCID, "", "")
if err != nil {
return err
+31
View File
@@ -139,6 +139,37 @@ func (r *EmbyRemoteService) ListAccounts(ctx context.Context) ([]model.StrmAccou
return out, nil
}
// ConfiguredRemoteHosts 返回所有已配置的远程 Emby 线路的主机名/IP(去重、不含端口)。
func (r *EmbyRemoteService) ConfiguredRemoteHosts(ctx context.Context) []string {
if r == nil || r.repo == nil || r.repo.StrmAccount == nil {
return nil
}
accounts, err := r.ListAccounts(ctx)
if err != nil || len(accounts) == 0 {
return nil
}
seen := make(map[string]bool)
var hosts []string
for _, acct := range accounts {
lines, _, err := r.LinesOf(&acct)
if err != nil {
continue
}
for _, line := range lines {
u, err := url.Parse(line.URL)
if err != nil || u.Hostname() == "" {
continue
}
h := strings.ToLower(u.Hostname())
if !seen[h] {
seen[h] = true
hosts = append(hosts, h)
}
}
}
return hosts
}
// AccountByID 按 ID 查找远程 Emby 挂载账号(不存在或类型不符返回 nil)。
func (r *EmbyRemoteService) AccountByID(ctx context.Context, id string) *model.StrmAccount {
if strings.TrimSpace(id) == "" {
+62 -6
View File
@@ -17,6 +17,7 @@ import (
"net"
"net/http"
"path/filepath"
"strings"
"sync"
"syscall"
"time"
@@ -42,6 +43,13 @@ type ImageProxy struct {
libRootsMu sync.Mutex
libRootsCache []string
libRootsAt time.Time
// allowedRemoteHostsFn returns hostnames or IPs of explicitly configured
// upstream services (e.g. remote Emby mounts) that should bypass SSRF private IP checks.
allowedRemoteHostsFn func() []string
allowedHostsMu sync.Mutex
allowedHostsCache map[string]bool
allowedHostsAt time.Time
}
const (
@@ -51,6 +59,12 @@ const (
// NewImageProxy is the constructor.
func NewImageProxy(cfg *config.Config, log *zap.Logger) *ImageProxy {
proxy := &ImageProxy{
cfg: cfg,
log: log,
cacheDir: filepath.Join(cfg.Cache.CacheDir, "images"),
}
// Honor HTTP(S)_PROXY env vars so deployments behind GFW can pull
// from image.tmdb.org via their HTTP proxy without extra config. On
// Windows we also honor the current user's system proxy settings.
@@ -63,6 +77,7 @@ func NewImageProxy(cfg *config.Config, log *zap.Logger) *ImageProxy {
// 仅 URL 解析层的 isPrivateHost 可被十进制/十六进制 IP、解析到
// 私网的域名与 DNS rebinding 绕过;在拨号层对最终连接 IP 做二次
// 校验(含重定向后的每条连接)堵住该旁路。
// 用户明确配置的远程挂载源(如内网 Emby)豁免该私网限制。
dialer := &net.Dialer{
Timeout: 15 * time.Second,
Control: func(_, address string, _ syscall.RawConn) error {
@@ -70,6 +85,9 @@ func NewImageProxy(cfg *config.Config, log *zap.Logger) *ImageProxy {
if err != nil {
return err
}
if proxy.isAllowedRemoteHost(host) {
return nil
}
ip := net.ParseIP(host)
if ip == nil {
return errors.New("image proxy: refusing non-IP dial target")
@@ -82,12 +100,9 @@ func NewImageProxy(cfg *config.Config, log *zap.Logger) *ImageProxy {
}
transport.DialContext = dialer.DialContext
}
return &ImageProxy{
cfg: cfg,
log: log,
cacheDir: filepath.Join(cfg.Cache.CacheDir, "images"),
client: &http.Client{Timeout: 30 * time.Second, Transport: transport},
}
proxy.client = &http.Client{Timeout: 30 * time.Second, Transport: transport}
return proxy
}
// proxyConfiguredForImageFetch 探测环境变量或系统代理是否会影响图片抓取。
@@ -125,6 +140,47 @@ func (p *ImageProxy) libraryRoots() []string {
return p.libRootsCache
}
// SetAllowedRemoteHostsProvider injects a callback that returns hostnames or IPs
// of explicitly configured remote services (e.g. remote Emby mounts). Requests to
// these hosts bypass SSRF private-IP restrictions.
func (p *ImageProxy) SetAllowedRemoteHostsProvider(fn func() []string) {
p.allowedRemoteHostsFn = fn
}
func (p *ImageProxy) isAllowedRemoteHost(host string) bool {
if p == nil || p.allowedRemoteHostsFn == nil {
return false
}
host = strings.ToLower(strings.TrimSpace(host))
if host == "" {
return false
}
// Strip port if present
if h, _, err := net.SplitHostPort(host); err == nil {
host = strings.ToLower(strings.TrimSpace(h))
}
p.allowedHostsMu.Lock()
defer p.allowedHostsMu.Unlock()
if p.allowedHostsCache == nil || time.Since(p.allowedHostsAt) >= 30*time.Second {
rawList := p.allowedRemoteHostsFn()
cache := make(map[string]bool, len(rawList))
for _, item := range rawList {
item = strings.ToLower(strings.TrimSpace(item))
if item == "" {
continue
}
if h, _, err := net.SplitHostPort(item); err == nil {
item = strings.ToLower(strings.TrimSpace(h))
}
cache[item] = true
}
p.allowedHostsCache = cache
p.allowedHostsAt = time.Now()
}
return p.allowedHostsCache[host]
}
// Prune removes oldest cached images until disk usage is within the configured limit.
func (p *ImageProxy) Prune() (PruneImageCacheResult, error) {
if p.cfg == nil || p.cfg.Cache.ImagesMaxSizeMB <= 0 {
+1 -1
View File
@@ -22,7 +22,7 @@ func (p *ImageProxy) validateURL(raw string) (*url.URL, error) {
if scheme != "http" && scheme != "https" {
return nil, errors.New("unsupported scheme")
}
if isPrivateHost(u.Hostname()) {
if !p.isAllowedRemoteHost(u.Hostname()) && isPrivateHost(u.Hostname()) {
return nil, errors.New("requests to private/internal hosts are not allowed")
}
return u, nil
+12 -8
View File
@@ -51,7 +51,7 @@ func (p *ImageProxy) fetchRemoteImageOnce(ctx context.Context, raw, host string,
p.log.Warn("imageproxy: build request failed", zap.String("url", raw), zap.Error(err))
return nil, "", "", errImageProxyRequestSetup
}
applyRemoteImageHeaders(req, host)
applyRemoteImageHeaders(req, host, raw)
resp, err := candidate.client.Do(req)
if err != nil {
@@ -79,7 +79,7 @@ func (p *ImageProxy) fetchRemoteImageOnce(ctx context.Context, raw, host string,
return data, ctype, resp.Header.Get("Content-Length"), nil
}
func applyRemoteImageHeaders(req *http.Request, host string) {
func applyRemoteImageHeaders(req *http.Request, host, raw string) {
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0 Safari/537.36")
req.Header.Set("Accept", "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8")
req.Header.Set("Accept-Language", "zh-CN,zh;q=0.9,ja;q=0.8,en;q=0.7")
@@ -88,7 +88,7 @@ func applyRemoteImageHeaders(req *http.Request, host string) {
if cookie := remoteImageCookie(host); cookie != "" {
req.Header.Set("Cookie", cookie)
}
if referer := remoteImageReferer(host); referer != "" {
if referer := remoteImageReferer(host, raw); referer != "" {
req.Header.Set("Referer", referer)
}
}
@@ -105,7 +105,7 @@ func remoteImageCookie(host string) string {
}
}
func remoteImageReferer(host string) string {
func remoteImageReferer(host, raw string) string {
h := strings.ToLower(strings.TrimSpace(host))
switch {
case strings.Contains(h, "doubanio.com"):
@@ -125,7 +125,11 @@ func remoteImageReferer(host string) string {
case strings.Contains(h, "fc2.com"):
return "https://adult.contents.fc2.com/"
case h != "":
return "https://" + h + "/"
scheme := "https"
if strings.HasPrefix(strings.ToLower(strings.TrimSpace(raw)), "http://") {
scheme = "http"
}
return scheme + "://" + h + "/"
default:
return ""
}
@@ -158,9 +162,9 @@ func fetchRemoteImageWithCurl(ctx context.Context, raw, host string) ([]byte, st
"--header", "Cache-Control: no-cache",
"--header", "Pragma: no-cache",
}
if referer := remoteImageReferer(host); referer != "" {
args = append(args, "--referer", referer)
}
if referer := remoteImageReferer(host, raw); referer != "" {
args = append(args, "--referer", referer)
}
if cookie := remoteImageCookie(host); cookie != "" {
args = append(args, "--cookie", cookie)
}
+9 -4
View File
@@ -317,9 +317,14 @@ func TestRemoteImageRefererForAdultHosts(t *testing.T) {
{"example.com", "https://example.com/"},
{"", ""},
}
for _, tt := range tests {
if got := remoteImageReferer(tt.host); got != tt.want {
t.Errorf("remoteImageReferer(%q) = %q, want %q", tt.host, got, tt.want)
for _, tt := range tests {
if got := remoteImageReferer(tt.host, "https://"+tt.host+"/img.jpg"); got != tt.want {
t.Errorf("remoteImageReferer(%q) = %q, want %q", tt.host, got, tt.want)
}
}
// Also verify HTTP protocol preservation for generic hosts
if got := remoteImageReferer("192.168.1.100", "http://192.168.1.100:8096/image"); got != "http://192.168.1.100/" {
t.Errorf("remoteImageReferer for HTTP host = %q, want http://192.168.1.100/", got)
}
}
}
+26 -3
View File
@@ -117,9 +117,32 @@ func TestIsPrivateHost(t *testing.T) {
// Hostnames must NOT be blocked even though GFW DNS poisoning may resolve
// them to private/loopback IPs — blocking them broke legitimate posters.
allowed := []string{"image.tmdb.org", "lain.bgm.tv", "example.com", "8.8.8.8"}
for _, h := range allowed {
if isPrivateHost(h) {
t.Errorf("isPrivateHost(%q) = true, want false", h)
for _, h := range allowed {
if isPrivateHost(h) {
t.Errorf("isPrivateHost(%q) = true, want false", h)
}
}
}
func TestImageProxyAllowedRemoteHostBypassesPrivateCheck(t *testing.T) {
proxy := NewImageProxy(&config.Config{Cache: config.CacheConfig{CacheDir: filepath.Join(t.TempDir(), "cache")}}, zap.NewNop())
rawURL := "http://192.168.1.100:8096/emby/Items/123/Images/Primary"
// Before setting allowed remote hosts, private host is rejected by validateURL
if _, err := proxy.validateURL(rawURL); err == nil {
t.Fatal("expected validateURL to reject private IP before whitelist")
}
// After configuring whitelist with the Emby host
proxy.SetAllowedRemoteHostsProvider(func() []string {
return []string{"192.168.1.100:8096"}
})
u, err := proxy.validateURL(rawURL)
if err != nil {
t.Fatalf("expected validateURL to allow whitelisted host, got: %v", err)
}
if u.Hostname() != "192.168.1.100" {
t.Fatalf("hostname = %s, want 192.168.1.100", u.Hostname())
}
}
+5
View File
@@ -177,6 +177,11 @@ func (b *serviceContainerBuilder) initIdentityServices() {
func (b *serviceContainerBuilder) initImageProxy() {
b.c.ImageProxy = NewImageProxy(b.cfg, b.log)
b.c.ImageProxy.SetLibraryRootsProvider(b.libraryRoots)
if b.c.EmbyRemote != nil {
b.c.ImageProxy.SetAllowedRemoteHostsProvider(func() []string {
return b.c.EmbyRemote.ConfiguredRemoteHosts(context.Background())
})
}
b.c.Scan.SetImageProxy(b.c.ImageProxy)
b.c.Scraper.SetImageProxy(b.c.ImageProxy)
}
+30 -18
View File
@@ -378,27 +378,39 @@ func (s *StrmService) processUpload115(ctx context.Context, task *model.StrmUplo
return
}
refs := strings.Split(task.RemoteRef, ",")
if err := open115.OpenClient().DeleteFiles(ctx, task.RemotePath, refs...); err != nil {
s.log.Warn("删除网盘旧元数据失败,跳过删除继续上传新文件",
zap.String("task_id", task.ID),
zap.String("local_path", task.LocalPath),
zap.Error(err))
// 不 return:继续上传新文件,旧副本由下次同步清理
if err := open115.OpenClient().DeleteFiles(ctx, task.RemotePath, refs...); err != nil {
s.log.Warn("删除网盘旧元数据失败,跳过删除继续上传新文件",
zap.String("task_id", task.ID),
zap.String("local_path", task.LocalPath),
zap.Error(err))
// 不 return:继续上传新文件,旧副本由下次同步清理
}
}
// 优先使用直接本地文件上传接口,零拷贝且彻底根除并发临时文件同名碰撞
if localUploader, ok := provider.(interface {
PutLocalFile(ctx context.Context, parentCID, localPath string) error
}); ok {
if err := localUploader.PutLocalFile(ctx, task.RemotePath, task.LocalPath); err != nil {
s.uploadTaskFailWithRetry(task, "上传失败:"+err.Error())
return
}
finish(model.StrmTaskDone, "")
return
}
f, err := os.Open(task.LocalPath)
if err != nil {
s.uploadTaskFailWithRetry(task, "打开本地文件失败:"+err.Error())
return
}
if err := named.PutFileNamed(ctx, task.RemotePath, task.FileName, f); err != nil {
_ = f.Close()
s.uploadTaskFailWithRetry(task, "上传失败:"+err.Error())
return
}
}
f, err := os.Open(task.LocalPath)
if err != nil {
s.uploadTaskFailWithRetry(task, "打开本地文件失败:"+err.Error())
return
}
if err := named.PutFileNamed(ctx, task.RemotePath, task.FileName, f); err != nil {
_ = f.Close()
s.uploadTaskFailWithRetry(task, "上传失败:"+err.Error())
return
finish(model.StrmTaskDone, "")
}
_ = f.Close()
finish(model.StrmTaskDone, "")
}
// downloadTaskFailWithRetry 下载失败任务按退避重试,超过上限标记 failed。
func (s *StrmService) downloadTaskFailWithRetry(task *model.StrmDownloadTask, message string) {
+21 -11
View File
@@ -1341,6 +1341,7 @@ func (st *strmSyncState) localSha1Matches(path, remoteSha1 string) bool {
}
// recordRemoteMeta 记录远端存在的元数据索引、文件大小、文件引用及内容 SHA1(多副本聚合追加)。
// 对同一文件 ID 严格去重,避免当 download_meta 与 upload_meta 同时开启时因重复记录引发误判与自杀式删除。
func (st *strmSyncState) recordRemoteMeta(entry cloud.FileEntry, rel string) {
st.mu.Lock()
defer st.mu.Unlock()
@@ -1349,6 +1350,13 @@ func (st *strmSyncState) recordRemoteMeta(entry cloud.FileEntry, rel string) {
}
key := "m:" + rel
st.seenMeta[key] = true
if entry.ID != "" {
for _, existing := range st.remoteMeta[key] {
if existing.ID == entry.ID {
return
}
}
}
st.remoteMeta[key] = append(st.remoteMeta[key], remoteMetaItem{
ID: entry.ID,
Size: entry.Size,
@@ -1624,21 +1632,23 @@ func (st *strmSyncState) scanLocalMetaForUpload() error {
}
}
}
if matchedIdx >= 0 {
// 远端已存在完全一致的副本,跳过上传!
// 若远端还存在其他同名脏副本(副本总数 > 1),在 115 下收集待删除 ID,稍后按目录批量删除
if len(entries) > 1 && st.p.Provider == model.StrmProvider115 {
parentCID := st.uploadRemoteTarget(rel)
if parentCID != "" {
for i, it := range entries {
if i != matchedIdx && it.ID != "" {
pendingDeletes[parentCID] = append(pendingDeletes[parentCID], it.ID)
if matchedIdx >= 0 {
// 远端已存在完全一致的副本,跳过上传!
// 若远端还存在其他同名脏副本(副本总数 > 1),在 115 下收集待删除 ID,稍后按目录批量删除。
// 严禁将已命中的最新副本 ID (matchedID) 放入待删列表,杜绝误杀唯一有效副本。
if len(entries) > 1 && st.p.Provider == model.StrmProvider115 {
parentCID := st.uploadRemoteTarget(rel)
if parentCID != "" {
matchedID := entries[matchedIdx].ID
for i, it := range entries {
if i != matchedIdx && it.ID != "" && it.ID != matchedID {
pendingDeletes[parentCID] = append(pendingDeletes[parentCID], it.ID)
}
}
}
}
return nil
}
return nil
}
}
remoteTarget := st.uploadRemoteTarget(rel)
+24
View File
@@ -1320,6 +1320,30 @@ func TestScanLocalMetaForUploadMultipleCopiesAllStale(t *testing.T) {
}
}
// TestRecordRemoteMetaDeduplication 验证当 download_meta 与 upload_meta 同时开启时,
// 同一远端元数据被多次入账不会在 remoteMeta 中生成重复副本,杜绝误判自杀式删除。
func TestRecordRemoteMetaDeduplication(t *testing.T) {
st := &strmSyncState{
seenMeta: map[string]bool{},
remoteMeta: map[string][]remoteMetaItem{},
}
entry := cloud.FileEntry{
ID: "unique-fid-1",
Name: "test.nfo",
Size: 100,
Sha1: "AAAABBBBCCCC",
MTime: 12345,
}
// 连续记录两次同一文件
st.recordRemoteMeta(entry, "dir/test.nfo")
st.recordRemoteMeta(entry, "dir/test.nfo")
items := st.remoteMeta["m:dir/test.nfo"]
if len(items) != 1 {
t.Fatalf("expected 1 item after duplicate record, got %d", len(items))
}
}
// TestWalk115AdaptiveHierarchicalFlatScan 验证自适应分治扁平化扫描:
// 当根目录探测总数 >= 9500 时,系统自动分治展开单层直接子项,对各子目录分别执行扁平拉取,
// 正确合并根目录直属文件与各子目录深层文件,突破 115 开放平台 10000 深度限制。
+14 -1
View File
@@ -2,6 +2,7 @@ import { useEffect, useRef, useState, type DragEvent, type MouseEvent, type Reac
import { createPortal } from 'react-dom'
import { Folder, GripVertical, Image, MoreVertical, Plus, Power, PowerOff, RefreshCw, Save, Trash2 } from 'lucide-react'
import { imageURL } from '../api/client'
import { LocalDirBrowserDialog } from '../components/LocalDirBrowserDialog'
import type { Library, LibraryRoot } from '../types'
import type { RootDraft } from './adminLibraryPanelModel'
@@ -169,7 +170,19 @@ function LibraryTableRow({ library, dragging, dragOver, onDragStart, onDragOver,
</td>
<td className="py-2 pr-3 font-medium text-ink-600">
<div className="flex items-center gap-2">
{library.cover_url && <img src={library.cover_url} alt="" loading="lazy" decoding="async" className="h-10 w-8 rounded object-cover" />}
{library.cover_url && (
<img
src={imageURL(library.cover_url, library.updated_at)}
alt=""
loading="lazy"
decoding="async"
referrerPolicy="no-referrer"
className="h-10 w-8 rounded object-cover"
onError={(e) => {
e.currentTarget.style.visibility = 'hidden'
}}
/>
)}
<span>{library.name}</span>
</div>
</td>