mirror of
https://github.com/truewhile/MeBox.git
synced 2026-09-28 03:06:38 +08:00
优化
This commit is contained in:
@@ -44,7 +44,7 @@ func setDefaults(v *viper.Viper) {
|
||||
v.SetDefault("logging.max_backups", 10)
|
||||
|
||||
v.SetDefault("cache.cache_dir", "./cache")
|
||||
v.SetDefault("cache.images_max_size_mb", 500)
|
||||
v.SetDefault("cache.images_max_size_mb", 2000)
|
||||
v.SetDefault("cache.memory_max_size_mb", DefaultCacheMemoryMaxSizeMB)
|
||||
v.SetDefault("cache.cleanup_interval_min", 60)
|
||||
v.SetDefault("cache.redis_url", "")
|
||||
|
||||
@@ -38,6 +38,12 @@ type ImageProxy struct {
|
||||
mu sync.Mutex
|
||||
fetchGroup singleflight.Group
|
||||
|
||||
// directClient bypasses HTTP_PROXY / OS proxy settings. It is built once and
|
||||
// shared: rebuilding the transport on every fetch discarded all keep-alive
|
||||
// connections, so every burst of poster requests paid a fresh TCP (and TLS)
|
||||
// handshake per image.
|
||||
directClient *http.Client
|
||||
|
||||
// resizeSem bounds concurrent decode/resize jobs. Emby TV clients request
|
||||
// poster grids in bursts; letting every request decode a source image at
|
||||
// once causes CPU and memory spikes that make the whole UI feel sluggish.
|
||||
@@ -113,6 +119,7 @@ func NewImageProxy(cfg *config.Config, log *zap.Logger) *ImageProxy {
|
||||
}
|
||||
|
||||
proxy.client = &http.Client{Timeout: 30 * time.Second, Transport: transport}
|
||||
proxy.directClient = &http.Client{Timeout: 30 * time.Second, Transport: NewInternalTransport()}
|
||||
return proxy
|
||||
}
|
||||
|
||||
|
||||
@@ -88,7 +88,7 @@ func (p *ImageProxy) remoteImageCachePaths(raw string) (string, string, string,
|
||||
}
|
||||
|
||||
func (p *ImageProxy) remoteImageCachePathsForValidated(raw string) (string, string, string) {
|
||||
sum := sha256.Sum256([]byte(raw))
|
||||
sum := sha256.Sum256([]byte(imageCacheKeyURL(raw)))
|
||||
key := hex.EncodeToString(sum[:])
|
||||
cachePath := filepath.Join(p.cacheDir, key)
|
||||
return key, cachePath, cachePath + ".fail"
|
||||
|
||||
@@ -87,7 +87,10 @@ func (p *ImageProxy) serveRemoteImage(ctx context.Context, w http.ResponseWriter
|
||||
return err
|
||||
}
|
||||
host := strings.ToLower(u.Host)
|
||||
key, cachePath, failPath := p.remoteImageCachePathsForValidated(raw)
|
||||
// 已配置的远程 Emby 挂载:把尺寸直接转发给远端生成缩略图,缓存键也用
|
||||
// 带尺寸的地址,这样不同尺寸各自缓存互不覆盖。
|
||||
fetchURL := p.upstreamImageFetchURL(raw, opts)
|
||||
key, cachePath, failPath := p.remoteImageCachePathsForValidated(fetchURL)
|
||||
forceRefresh := r.URL.Query().Get("refresh") != ""
|
||||
p.removeUnusableImageCache(cachePath, failPath)
|
||||
if !forceRefresh && p.serveCachedImage(w, r, key, cachePath, opts) {
|
||||
@@ -95,7 +98,7 @@ func (p *ImageProxy) serveRemoteImage(ctx context.Context, w http.ResponseWriter
|
||||
}
|
||||
// No negative caching: a previously failed fetch is retried on every
|
||||
// subsequent request, so the image recovers as soon as upstream does.
|
||||
data, ctype, contentLength, err := p.fetchAndCacheRemoteImageShared(ctx, raw, host, cachePath, failPath)
|
||||
data, ctype, contentLength, err := p.fetchAndCacheRemoteImageShared(ctx, fetchURL, host, cachePath, failPath)
|
||||
if err != nil {
|
||||
if forceRefresh && p.serveCachedImage(w, r, key, cachePath, opts) {
|
||||
return nil
|
||||
@@ -179,7 +182,7 @@ func (p *ImageProxy) fetchAndCacheRemoteImage(ctx context.Context, raw, host, ca
|
||||
return nil, "", "", errImageProxyRequestSetup
|
||||
}
|
||||
var lastErr error
|
||||
for _, candidate := range p.remoteImageFetchClients() {
|
||||
for _, candidate := range p.remoteImageFetchClients(host) {
|
||||
data, ctype, contentLength, err := p.fetchRemoteImageOnce(ctx, raw, host, candidate)
|
||||
if err == nil {
|
||||
p.writeImageCache(cachePath, failPath, "img-*.tmp", data)
|
||||
@@ -263,9 +266,11 @@ func (p *ImageProxy) Fetch(ctx context.Context, raw string) ([]byte, string, err
|
||||
return data, ctype, err
|
||||
}
|
||||
|
||||
// writeImageCache atomically writes the fetched original. The global mutex is
|
||||
// deliberately not held: os.CreateTemp already yields a unique name and
|
||||
// os.Rename is atomic, so the lock only serialized multi-megabyte disk writes
|
||||
// and made one poster's write block every other image in flight.
|
||||
func (p *ImageProxy) writeImageCache(cachePath, failPath, pattern string, data []byte) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
tmp, tmpErr := os.CreateTemp(p.cacheDir, pattern)
|
||||
if tmpErr != nil {
|
||||
return
|
||||
|
||||
@@ -18,23 +18,32 @@ type remoteImageFetchClient struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func (p *ImageProxy) remoteImageFetchClients() []remoteImageFetchClient {
|
||||
func (p *ImageProxy) remoteImageFetchClients(host string) []remoteImageFetchClient {
|
||||
client := p.client
|
||||
if client == nil {
|
||||
client = NewExternalHTTPClient(30 * time.Second)
|
||||
}
|
||||
clients := []remoteImageFetchClient{{name: "default", client: client}}
|
||||
if _, ok := client.Transport.(*http.Transport); ok {
|
||||
timeout := client.Timeout
|
||||
if timeout <= 0 {
|
||||
timeout = 30 * time.Second
|
||||
}
|
||||
clients = append(clients, remoteImageFetchClient{
|
||||
name: "direct",
|
||||
client: &http.Client{Timeout: timeout, Transport: NewInternalTransport()},
|
||||
})
|
||||
if _, ok := client.Transport.(*http.Transport); !ok {
|
||||
return []remoteImageFetchClient{{name: "default", client: client}}
|
||||
}
|
||||
return clients
|
||||
timeout := client.Timeout
|
||||
if timeout <= 0 {
|
||||
timeout = 30 * time.Second
|
||||
}
|
||||
direct := p.directClient
|
||||
if direct == nil {
|
||||
direct = &http.Client{Timeout: timeout, Transport: NewInternalTransport()}
|
||||
}
|
||||
directCandidate := remoteImageFetchClient{name: "direct", client: direct}
|
||||
defaultCandidate := remoteImageFetchClient{name: "default", client: client}
|
||||
// Hosts the user configured themselves — remote Emby mounts and their image
|
||||
// endpoints — are reached over the LAN or a dedicated line. Routing those
|
||||
// through the OS/env proxy first costs a failed attempt before every single
|
||||
// image, so try the direct client first for them.
|
||||
if p.isAllowedRemoteHost(host) {
|
||||
return []remoteImageFetchClient{directCandidate, defaultCandidate}
|
||||
}
|
||||
return []remoteImageFetchClient{defaultCandidate, directCandidate}
|
||||
}
|
||||
|
||||
func (p *ImageProxy) canUseExternalImageFallback() bool {
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// imageCacheKeyURL normalizes an upstream image URL before it is hashed into a
|
||||
// disk cache key. Credential query parameters are dropped: a remote Emby token
|
||||
// is per-account, so removing it cannot make two different images collide, but
|
||||
// it stops a token rotation from invalidating every cached poster at once.
|
||||
func imageCacheKeyURL(raw string) string {
|
||||
u, err := url.Parse(strings.TrimSpace(raw))
|
||||
if err != nil || u.Host == "" {
|
||||
return raw
|
||||
}
|
||||
q := u.Query()
|
||||
dropped := false
|
||||
for key := range q {
|
||||
switch strings.ToLower(key) {
|
||||
case "api_key", "apikey", "x-emby-token", "x-mediabrowser-token":
|
||||
q.Del(key)
|
||||
dropped = true
|
||||
}
|
||||
}
|
||||
if !dropped {
|
||||
return raw
|
||||
}
|
||||
u.RawQuery = q.Encode()
|
||||
return u.String()
|
||||
}
|
||||
|
||||
// upstreamImageFetchURL forwards the client's thumbnail request to a configured
|
||||
// remote Emby mount. The remote server can produce the thumbnail itself from
|
||||
// maxWidth/maxHeight/quality, so the proxy transfers a few dozen KB instead of
|
||||
// the full-size original and skips the local decode+scale entirely (measured at
|
||||
// ~400ms for a single 2892×4096 poster).
|
||||
//
|
||||
// Other upstreams (TMDb, Douban, adult sites, ...) do not honour these
|
||||
// parameters, so their originals are still fetched and scaled locally.
|
||||
func (p *ImageProxy) upstreamImageFetchURL(raw string, opts imageResizeOptions) string {
|
||||
if p == nil || !opts.active() {
|
||||
return raw
|
||||
}
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil || u.Host == "" {
|
||||
return raw
|
||||
}
|
||||
if !p.isAllowedRemoteHost(u.Hostname()) {
|
||||
return raw
|
||||
}
|
||||
q := u.Query()
|
||||
if opts.MaxWidth > 0 {
|
||||
q.Set("maxWidth", strconv.Itoa(opts.MaxWidth))
|
||||
}
|
||||
if opts.MaxHeight > 0 {
|
||||
q.Set("maxHeight", strconv.Itoa(opts.MaxHeight))
|
||||
}
|
||||
q.Set("quality", strconv.Itoa(opts.encodingQuality()))
|
||||
u.RawQuery = q.Encode()
|
||||
return u.String()
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/config"
|
||||
)
|
||||
|
||||
func newUpstreamImageProxy(t *testing.T, allowedHosts ...string) *ImageProxy {
|
||||
t.Helper()
|
||||
proxy := NewImageProxy(&config.Config{Cache: config.CacheConfig{CacheDir: filepath.Join(t.TempDir(), "cache")}}, zap.NewNop())
|
||||
if len(allowedHosts) > 0 {
|
||||
proxy.SetAllowedRemoteHostsProvider(func() []string { return allowedHosts })
|
||||
}
|
||||
return proxy
|
||||
}
|
||||
|
||||
func TestUpstreamImageFetchURLForwardsResizeToConfiguredMount(t *testing.T) {
|
||||
proxy := newUpstreamImageProxy(t, "192.168.1.50:8096")
|
||||
raw := "http://192.168.1.50:8096/emby/Items/abc/Images/Primary?api_key=secret"
|
||||
|
||||
got := proxy.upstreamImageFetchURL(raw, imageResizeOptions{MaxWidth: 480, MaxHeight: 600, Quality: 82})
|
||||
u, err := url.Parse(got)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
q := u.Query()
|
||||
if q.Get("maxWidth") != "480" || q.Get("maxHeight") != "600" || q.Get("quality") != "82" {
|
||||
t.Fatalf("forwarded query = %q, want maxWidth=480 maxHeight=600 quality=82", u.RawQuery)
|
||||
}
|
||||
if q.Get("api_key") != "secret" {
|
||||
t.Fatalf("api_key = %q, want the credential preserved", q.Get("api_key"))
|
||||
}
|
||||
if u.Path != "/emby/Items/abc/Images/Primary" {
|
||||
t.Fatalf("path = %q, want the item image path unchanged", u.Path)
|
||||
}
|
||||
|
||||
// The default quality is applied when the client does not send one.
|
||||
got = proxy.upstreamImageFetchURL(raw, imageResizeOptions{MaxWidth: 160})
|
||||
if !strings.Contains(got, "quality="+strconv.Itoa(imageResizeDefaultQuality)) {
|
||||
t.Fatalf("url = %q, want the default encoding quality forwarded", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpstreamImageFetchURLLeavesOtherUpstreamsAlone(t *testing.T) {
|
||||
proxy := newUpstreamImageProxy(t, "192.168.1.50:8096")
|
||||
opts := imageResizeOptions{MaxWidth: 480, MaxHeight: 600, Quality: 82}
|
||||
|
||||
// Not a configured mount: TMDb/Douban do not honour these parameters, so the
|
||||
// proxy must keep fetching their originals and resize locally.
|
||||
tmdb := "https://image.tmdb.org/t/p/original/poster.jpg"
|
||||
if got := proxy.upstreamImageFetchURL(tmdb, opts); got != tmdb {
|
||||
t.Fatalf("url = %q, want %q unchanged", got, tmdb)
|
||||
}
|
||||
// Configured mount but no size requested: keep the full original.
|
||||
mount := "http://192.168.1.50:8096/emby/Items/abc/Images/Primary?api_key=secret"
|
||||
if got := proxy.upstreamImageFetchURL(mount, imageResizeOptions{}); got != mount {
|
||||
t.Fatalf("url = %q, want %q unchanged", got, mount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteImageCacheKeyIgnoresCredentialRotation(t *testing.T) {
|
||||
proxy := newUpstreamImageProxy(t)
|
||||
first := "http://192.168.1.50:8096/emby/Items/abc/Images/Primary?api_key=old-token"
|
||||
second := "http://192.168.1.50:8096/emby/Items/abc/Images/Primary?api_key=new-token"
|
||||
|
||||
_, firstPath, _ := proxy.remoteImageCachePathsForValidated(first)
|
||||
_, secondPath, _ := proxy.remoteImageCachePathsForValidated(second)
|
||||
if firstPath != secondPath {
|
||||
t.Fatalf("token rotation changed the cache key: %q vs %q", firstPath, secondPath)
|
||||
}
|
||||
|
||||
other := "http://192.168.1.50:8096/emby/Items/zzz/Images/Primary?api_key=new-token"
|
||||
_, otherPath, _ := proxy.remoteImageCachePathsForValidated(other)
|
||||
if otherPath == secondPath {
|
||||
t.Fatal("different items must not share a cache key")
|
||||
}
|
||||
|
||||
// A generic `token` parameter can be part of a signed URL's identity, so it
|
||||
// must stay in the key. Only credential-ish names are dropped.
|
||||
signed := "http://cdn.example.com/img/1.jpg?token=aaa"
|
||||
signedOther := "http://cdn.example.com/img/1.jpg?token=bbb"
|
||||
_, signedPath, _ := proxy.remoteImageCachePathsForValidated(signed)
|
||||
_, signedOtherPath, _ := proxy.remoteImageCachePathsForValidated(signedOther)
|
||||
if signedPath == signedOtherPath {
|
||||
t.Fatal("generic token query parameters must remain part of the cache key")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteImageFetchClientsPreferDirectForConfiguredMount(t *testing.T) {
|
||||
proxy := newUpstreamImageProxy(t, "192.168.1.50:8096")
|
||||
|
||||
mountClients := proxy.remoteImageFetchClients("192.168.1.50:8096")
|
||||
if len(mountClients) != 2 || mountClients[0].name != "direct" || mountClients[1].name != "default" {
|
||||
t.Fatalf("mount client order = %+v, want direct first", clientNames(mountClients))
|
||||
}
|
||||
if mountClients[0].client != proxy.directClient {
|
||||
t.Fatal("direct fetches must reuse the shared no-proxy client")
|
||||
}
|
||||
|
||||
cdnClients := proxy.remoteImageFetchClients("image.tmdb.org")
|
||||
if len(cdnClients) != 2 || cdnClients[0].name != "default" || cdnClients[1].name != "direct" {
|
||||
t.Fatalf("cdn client order = %+v, want default first", clientNames(cdnClients))
|
||||
}
|
||||
}
|
||||
|
||||
func clientNames(clients []remoteImageFetchClient) []string {
|
||||
out := make([]string, 0, len(clients))
|
||||
for _, c := range clients {
|
||||
out = append(out, c.name)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// TestServeRemoteImageForwardsResizeAndCachesPerSize is the regression test for
|
||||
// sluggish artwork on mounted Emby libraries: MeBox used to download the remote
|
||||
// original and scale it locally for every requested size. The size must now be
|
||||
// forwarded to the mount, and each size must get its own cache entry.
|
||||
func TestServeRemoteImageForwardsResizeAndCachesPerSize(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
queries := []url.Values{}
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
mu.Lock()
|
||||
queries = append(queries, r.URL.Query())
|
||||
mu.Unlock()
|
||||
w.Header().Set("Content-Type", "image/jpeg")
|
||||
_, _ = w.Write(testJPEG)
|
||||
}))
|
||||
defer upstream.Close()
|
||||
|
||||
u, err := url.Parse(upstream.URL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
proxy := newUpstreamImageProxy(t, u.Host)
|
||||
raw := upstream.URL + "/emby/Items/abc/Images/Primary?api_key=secret"
|
||||
|
||||
serve := func(t *testing.T, query string) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/img?"+query, nil)
|
||||
if err := proxy.Serve(t.Context(), rec, req, raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200", rec.Code)
|
||||
}
|
||||
return rec
|
||||
}
|
||||
|
||||
rec := serve(t, "maxWidth=480&maxHeight=600&quality=82")
|
||||
if got := rec.Body.Bytes(); !bytes.Equal(got, testJPEG) {
|
||||
t.Fatalf("body = %x, want the upstream image", got)
|
||||
}
|
||||
|
||||
mu.Lock()
|
||||
first := append([]url.Values(nil), queries...)
|
||||
mu.Unlock()
|
||||
if len(first) != 1 {
|
||||
t.Fatalf("upstream calls = %d, want 1", len(first))
|
||||
}
|
||||
if first[0].Get("maxWidth") != "480" || first[0].Get("maxHeight") != "600" || first[0].Get("quality") != "82" {
|
||||
t.Fatalf("upstream query = %q, want the client's thumbnail request forwarded", first[0].Encode())
|
||||
}
|
||||
|
||||
// Same size again: served from the disk cache, no second upstream call.
|
||||
serve(t, "maxWidth=480&maxHeight=600&quality=82")
|
||||
mu.Lock()
|
||||
if len(queries) != 1 {
|
||||
mu.Unlock()
|
||||
t.Fatalf("upstream calls = %d, want 1 after a cache hit", len(queries))
|
||||
}
|
||||
mu.Unlock()
|
||||
|
||||
// A different size must be its own cache entry, not a re-use of the 480px file.
|
||||
serve(t, "maxWidth=160&quality=60")
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if len(queries) != 2 {
|
||||
t.Fatalf("upstream calls = %d, want 2 for two distinct sizes", len(queries))
|
||||
}
|
||||
if got := queries[1].Get("maxWidth"); got != "160" {
|
||||
t.Fatalf("second upstream maxWidth = %q, want 160", got)
|
||||
}
|
||||
if got := queries[1].Get("quality"); got != "60" {
|
||||
t.Fatalf("second upstream quality = %q, want 60", got)
|
||||
}
|
||||
}
|
||||
@@ -344,14 +344,14 @@ func isCompactWebImage(header []byte) bool {
|
||||
}
|
||||
|
||||
// writeResizeCache 原子写入缩放结果;失败只记日志,不影响本次响应。
|
||||
// 与 writeImageCache 一样不再持有全局锁:临时文件名唯一、rename 原子,
|
||||
// 持锁只会把缩略图的写盘和原图的写盘串成一条队。
|
||||
func (p *ImageProxy) writeResizeCache(cachePath string, data []byte) {
|
||||
dir := filepath.Dir(cachePath)
|
||||
if err := os.MkdirAll(dir, 0o750); err != nil {
|
||||
p.warn("imageproxy: resize cache mkdir failed", err)
|
||||
return
|
||||
}
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
tmp, err := os.CreateTemp(dir, "resized-*.tmp")
|
||||
if err != nil {
|
||||
p.warn("imageproxy: resize cache temp failed", err)
|
||||
|
||||
@@ -111,9 +111,9 @@ export const generalSettingsGroup: SettingGroup = {
|
||||
key: 'cache.images_max_size_mb',
|
||||
label: '图片缓存上限 (MB)',
|
||||
type: 'number',
|
||||
hint: '海报、剧照等图片代理缓存目录 (cache/images) 的最大占用空间。超过上限时自动按修改时间清理最旧的文件。设为 0 表示不限制,默认 500MB',
|
||||
defaultValue: '500',
|
||||
placeholder: '500',
|
||||
hint: '海报、剧照等图片代理缓存目录 (cache/images) 的最大占用空间。超过上限时自动按修改时间清理最旧的文件;上限过小会导致旧图被淘汰后又要重新从上游/挂载的 Emby 下载。设为 0 表示不限制,默认 2000MB',
|
||||
defaultValue: '2000',
|
||||
placeholder: '2000',
|
||||
},
|
||||
{
|
||||
key: 'https.enabled',
|
||||
|
||||
Reference in New Issue
Block a user