mirror of
https://github.com/truewhile/MeBox.git
synced 2026-11-15 20:34:23 +08:00
Compare commits
28 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 3d7d661199 | |||
| 8686d8cae6 | |||
| 43cfeabea1 | |||
| d112e8e4db | |||
| 6604691c19 | |||
| 23937d8137 | |||
| 6864a9c7f1 | |||
| 079b3ff063 | |||
| 3c30c64a39 | |||
| 6186c8b848 | |||
| 0db8698fce | |||
| 42703c6760 | |||
| 4ed2edab5b | |||
| 374ee28f01 | |||
| 3a6b8db0b3 | |||
| b50cb7c31e | |||
| 5f4b613bde | |||
| 67ef9c95be | |||
| 2bf1b27e23 | |||
| 48a1c36a7a | |||
| d7665e4ef0 | |||
| 99916c0b97 | |||
| 6adc3cb3b2 | |||
| 2b2dc697ce | |||
| 8bfdd75d47 | |||
| 1a3248f4bb | |||
| 0be7f8b97e | |||
| 6984158f24 |
+6
-3
@@ -84,12 +84,15 @@ ENV MEBOX_APP_PORT=8080 \
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
HEALTHCHECK --interval=30s --timeout=5s --start-period=15s --retries=3 \
|
||||
CMD busybox wget -q --spider http://127.0.0.1:8080/api/health || exit 1
|
||||
# 探测协议不能写死:开启 HTTPS 后 8080 只接受 TLS。脚本按数据目录里的
|
||||
# .http_scheme 选择协议并在失败时回退,无需用户改配置。
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \
|
||||
CMD ["/usr/local/bin/mebox-healthcheck"]
|
||||
|
||||
# Tiny entrypoint that lets us run as a NAS host UID/GID via PUID/PGID without
|
||||
# rewriting /etc/passwd or /etc/group on every container start.
|
||||
COPY docker-entrypoint.sh /entrypoint.sh
|
||||
RUN chmod +x /entrypoint.sh
|
||||
COPY docker-healthcheck.sh /usr/local/bin/mebox-healthcheck
|
||||
RUN chmod +x /entrypoint.sh /usr/local/bin/mebox-healthcheck
|
||||
|
||||
CMD ["/entrypoint.sh"]
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -76,7 +77,15 @@ func main() {
|
||||
fatal("未从输入中识别到书源")
|
||||
}
|
||||
|
||||
svc := reader.NewReaderService(&config.Config{}, zap.NewNop(), &repository.Container{})
|
||||
// 书源文件缓存根目录:java.downloadFile / cacheFile 需要它才能落盘,
|
||||
// 否则「拷贝轻小说」这类把整卷文本缓存到本地再读的源会被判为不支持。
|
||||
// 用固定子目录而不是每次 MkdirTemp,避免反复运行在临时目录里留一堆垃圾。
|
||||
cfg := &config.Config{}
|
||||
cacheDir := filepath.Join(os.TempDir(), "reader-smoke-cache")
|
||||
if err := os.MkdirAll(cacheDir, 0o750); err == nil {
|
||||
cfg.Cache.CacheDir = cacheDir
|
||||
}
|
||||
svc := reader.NewReaderService(cfg, zap.NewNop(), &repository.Container{})
|
||||
ctx := context.Background()
|
||||
|
||||
results := make([]*reader.SmokeChainResult, len(sources))
|
||||
|
||||
@@ -88,7 +88,7 @@ func (a *application) start() error {
|
||||
a.logger.Error("database not ready", zap.Error(err))
|
||||
return fmt.Errorf("database not ready: %w", err)
|
||||
}
|
||||
if err := database.AutoMigrate(db); err != nil {
|
||||
if err := database.AutoMigrateWithConfig(db, a.cfg, a.logger); err != nil {
|
||||
a.logger.Error("auto-migrate failed", zap.Error(err))
|
||||
return fmt.Errorf("auto-migrate failed: %w", err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"net"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// 开启 HTTPS 后,同一个端口会同时面对两类连接:
|
||||
//
|
||||
// - TLS 客户端(浏览器、播放器)——正常提供 HTTPS 服务;
|
||||
// - 明确访问 http:// 的客户端——以前只会撞出一句
|
||||
// "Client sent an HTTP request to an HTTPS server.",用户看不懂。
|
||||
//
|
||||
// 一个端口上只有 TLS 监听时无法区分两者,所以在 Accept 之后先偷看第一个字节:
|
||||
// TLS 握手的记录类型固定是 0x16(handshake),而 HTTP 请求行必然是可见字符。
|
||||
// 据此把连接分流成两路,明文那一路交给只做跳转的 http.Server,于是 http:// 会被
|
||||
// 自动重定向到 https://(见 server_manager.go 的 httpsRedirectHandler)。
|
||||
const tlsRecordHandshake = 0x16
|
||||
|
||||
// defaultPeekTimeout 限制"偷看首字节"阶段的等待,避免只连不发的半开连接长期占住
|
||||
// goroutine。分流器按实例持有该值,便于测试用更短的超时。
|
||||
const defaultPeekTimeout = 10 * time.Second
|
||||
|
||||
// splitListener 是监听分流器:Accept 到底层连接后按首字节投递给对应的一路。
|
||||
type splitListener struct {
|
||||
ln net.Listener
|
||||
tlsCh chan net.Conn
|
||||
plainCh chan net.Conn
|
||||
quit chan struct{}
|
||||
once sync.Once
|
||||
peek time.Duration
|
||||
}
|
||||
|
||||
func newSplitListener(ln net.Listener) *splitListener {
|
||||
return newSplitListenerWithTimeout(ln, defaultPeekTimeout)
|
||||
}
|
||||
|
||||
func newSplitListenerWithTimeout(ln net.Listener, peek time.Duration) *splitListener {
|
||||
s := &splitListener{
|
||||
ln: ln,
|
||||
tlsCh: make(chan net.Conn),
|
||||
plainCh: make(chan net.Conn),
|
||||
quit: make(chan struct{}),
|
||||
peek: peek,
|
||||
}
|
||||
go s.acceptLoop()
|
||||
return s
|
||||
}
|
||||
|
||||
// tlsListener 返回只收到 TLS 连接的 listener。
|
||||
func (s *splitListener) tlsListener() net.Listener {
|
||||
return &channelListener{addr: s.ln.Addr(), conns: s.tlsCh, quit: s.quit}
|
||||
}
|
||||
|
||||
// plainListener 返回只收到明文 HTTP 连接的 listener。
|
||||
func (s *splitListener) plainListener() net.Listener {
|
||||
return &channelListener{addr: s.ln.Addr(), conns: s.plainCh, quit: s.quit}
|
||||
}
|
||||
|
||||
func (s *splitListener) acceptLoop() {
|
||||
for {
|
||||
conn, err := s.ln.Accept()
|
||||
if err != nil {
|
||||
// 底层监听关闭(换证、关停)后两路 Accept 一起返回 net.ErrClosed,
|
||||
// 由持有它们的 http.Server 自行收尾。
|
||||
s.stop()
|
||||
return
|
||||
}
|
||||
go s.dispatch(conn)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *splitListener) dispatch(conn net.Conn) {
|
||||
br := bufio.NewReader(conn)
|
||||
_ = conn.SetReadDeadline(time.Now().Add(s.peek))
|
||||
first, err := br.Peek(1)
|
||||
_ = conn.SetReadDeadline(time.Time{})
|
||||
if err != nil {
|
||||
_ = conn.Close()
|
||||
return
|
||||
}
|
||||
|
||||
peeked := &peekedConn{Conn: conn, r: br}
|
||||
ch := s.plainCh
|
||||
if first[0] == tlsRecordHandshake {
|
||||
ch = s.tlsCh
|
||||
}
|
||||
select {
|
||||
case ch <- peeked: // 交给对应的 http.Server,由它负责读写与关闭
|
||||
case <-s.quit:
|
||||
_ = conn.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func (s *splitListener) stop() {
|
||||
s.once.Do(func() { close(s.quit) })
|
||||
}
|
||||
|
||||
// channelListener 把一路 channel 包装成 net.Listener,供 http.Server.Serve 使用。
|
||||
type channelListener struct {
|
||||
addr net.Addr
|
||||
conns <-chan net.Conn
|
||||
quit <-chan struct{}
|
||||
}
|
||||
|
||||
func (l *channelListener) Accept() (net.Conn, error) {
|
||||
select {
|
||||
case conn := <-l.conns:
|
||||
return conn, nil
|
||||
case <-l.quit:
|
||||
return nil, net.ErrClosed
|
||||
}
|
||||
}
|
||||
|
||||
// Close 是空实现:真正要释放的端口在 splitListener 的底层监听上,它一旦关闭,
|
||||
// 这里的 Accept 就会返回 net.ErrClosed,http.Server 随之退出。
|
||||
func (l *channelListener) Close() error { return nil }
|
||||
|
||||
func (l *channelListener) Addr() net.Addr { return l.addr }
|
||||
|
||||
// peekedConn 把已经读进 bufio 缓冲的首字节还给上层读取者。
|
||||
type peekedConn struct {
|
||||
net.Conn
|
||||
r *bufio.Reader
|
||||
}
|
||||
|
||||
func (c *peekedConn) Read(p []byte) (int, error) { return c.r.Read(p) }
|
||||
@@ -0,0 +1,140 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestSplitListenerRoutesByFirstByte(t *testing.T) {
|
||||
raw, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
split := newSplitListener(raw)
|
||||
addr := raw.Addr().String()
|
||||
|
||||
// TLS:首字节 0x16。分流器偷看过的字节必须还能被上层读到,否则握手会失败。
|
||||
handshake := []byte{0x16, 0x03, 0x01, 0x00, 0x2c}
|
||||
tlsClient, err := net.Dial("tcp", addr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = tlsClient.Close() }()
|
||||
if _, err := tlsClient.Write(handshake); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tlsConn := <-split.tlsCh
|
||||
defer func() { _ = tlsConn.Close() }()
|
||||
got := make([]byte, len(handshake))
|
||||
if _, err := io.ReadFull(tlsConn, got); err != nil {
|
||||
t.Fatalf("read peeked tls bytes: %v", err)
|
||||
}
|
||||
if string(got) != string(handshake) {
|
||||
t.Fatalf("tls bytes = %v, want %v", got, handshake)
|
||||
}
|
||||
|
||||
// 明文 HTTP:请求行走另一路,同样要读到偷看过的首字节。
|
||||
request := "GET /api/health HTTP/1.1\r\nHost: example.com\r\n\r\n"
|
||||
plainClient, err := net.Dial("tcp", addr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = plainClient.Close() }()
|
||||
if _, err := plainClient.Write([]byte(request)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plainConn := <-split.plainCh
|
||||
defer func() { _ = plainConn.Close() }()
|
||||
head := make([]byte, len("GET /"))
|
||||
if _, err := io.ReadFull(plainConn, head); err != nil {
|
||||
t.Fatalf("read peeked plain bytes: %v", err)
|
||||
}
|
||||
if string(head) != request[:len(head)] {
|
||||
t.Fatalf("plain bytes = %q, want prefix of %q", head, request)
|
||||
}
|
||||
|
||||
// 底层监听关闭后两路 Accept 都要返回 net.ErrClosed,http.Server 才能收尾。
|
||||
_ = raw.Close()
|
||||
for _, ln := range []net.Listener{split.tlsListener(), split.plainListener()} {
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for {
|
||||
_, err := ln.Accept()
|
||||
if err == net.ErrClosed {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("Accept after close = %v, want net.ErrClosed", err)
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 只连不发的半开连接不应被分给任何一路,而应在偷看超时后被丢弃。
|
||||
func TestSplitListenerDropsIdleConnection(t *testing.T) {
|
||||
raw, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = raw.Close() }()
|
||||
split := newSplitListenerWithTimeout(raw, 200*time.Millisecond)
|
||||
|
||||
conn, err := net.Dial("tcp", raw.Addr().String())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = conn.Close() }()
|
||||
|
||||
select {
|
||||
case c := <-split.plainCh:
|
||||
_ = c.Close()
|
||||
t.Fatal("idle connection should not be dispatched")
|
||||
case c := <-split.tlsCh:
|
||||
_ = c.Close()
|
||||
t.Fatal("idle connection should not be dispatched")
|
||||
case <-time.After(time.Second):
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPSRedirectTarget(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
host string
|
||||
rawURL string
|
||||
want string
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "keeps host port path and query", host: "10.0.0.2:8080", rawURL: "/library?tab=1&q=a%20b", want: "https://10.0.0.2:8080/library?tab=1&q=a%20b"},
|
||||
{name: "root path", host: "nas.local:18080", rawURL: "/", want: "https://nas.local:18080/"},
|
||||
{name: "keeps escaping of user input", host: "10.0.0.2", rawURL: "/a%2Fb", want: "https://10.0.0.2/a%2Fb"},
|
||||
{name: "absolute form request line", host: "10.0.0.2:8080", rawURL: "http://10.0.0.2:8080/api/x?y=1", want: "https://10.0.0.2:8080/api/x?y=1"},
|
||||
{name: "rejects empty host", host: "", rawURL: "/", wantErr: true},
|
||||
{name: "rejects host with slash", host: "evil.com/path", rawURL: "/", wantErr: true},
|
||||
{name: "rejects userinfo host", host: "user@evil.com", rawURL: "/", wantErr: true},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
req, err := http.NewRequest(http.MethodGet, tc.rawURL, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req.Host = tc.host
|
||||
|
||||
if host := redirectHostHeader(req); host == "" {
|
||||
if !tc.wantErr {
|
||||
t.Fatalf("redirectHostHeader(%q) rejected, want accepted", tc.host)
|
||||
}
|
||||
return
|
||||
}
|
||||
if tc.wantErr {
|
||||
t.Fatalf("redirectHostHeader(%q) accepted, want rejected", tc.host)
|
||||
}
|
||||
if got := httpsRedirectURL(tc.host, req); got != tc.want {
|
||||
t.Fatalf("redirect target = %q, want %q", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+123
-18
@@ -7,6 +7,8 @@ import (
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -17,6 +19,14 @@ import (
|
||||
"github.com/truewhile/MeBox/internal/service"
|
||||
)
|
||||
|
||||
// httpSchemeFileName 是数据目录下记录当前监听协议的小文件(内容为 http 或 https)。
|
||||
//
|
||||
// 容器健康检查无法读取后台保存的 HTTPS 设置(它存在数据库里),而同一端口在开启
|
||||
// HTTPS 后只接受 TLS。若健康检查固定用 http:// 探测,每轮都会撞出
|
||||
// "client sent an HTTP request to an HTTPS server",既刷日志又让容器永远 unhealthy。
|
||||
// 因此由服务进程在每次成功绑定监听后写下真实协议,供 docker-healthcheck.sh 选择。
|
||||
const httpSchemeFileName = ".http_scheme"
|
||||
|
||||
// tlsPair 记录当前正在服务的证书,用于判断是否需要重新绑定监听。
|
||||
type tlsPair struct {
|
||||
cert tls.Certificate
|
||||
@@ -35,6 +45,9 @@ type tlsPair struct {
|
||||
// - 旧服务器随后优雅退出,正在进行的播放/请求不会被立刻掐断。
|
||||
//
|
||||
// 任何校验失败都会中止切换并保留旧监听,保证用户不会被锁在服务外面。
|
||||
//
|
||||
// 开启 HTTPS 时端口上会同时接两类连接:TLS 走主服务器,明文 HTTP 走
|
||||
// redirectSrv 并跳转到 HTTPS(见 protocol_mux.go)。
|
||||
type serverManager struct {
|
||||
cfg *config.Config
|
||||
log *zap.Logger
|
||||
@@ -43,6 +56,7 @@ type serverManager struct {
|
||||
|
||||
mu sync.Mutex
|
||||
srv *http.Server
|
||||
redirectSrv *http.Server
|
||||
ln net.Listener
|
||||
pair *tlsPair
|
||||
stopCh chan struct{}
|
||||
@@ -92,11 +106,11 @@ func (m *serverManager) Reload() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
oldSrv, oldLn, oldPair := m.srv, m.ln, m.pair
|
||||
oldSrv, oldRedirect, oldLn, oldPair := m.srv, m.redirectSrv, m.ln, m.pair
|
||||
if oldLn != nil {
|
||||
_ = oldLn.Close() // 释放端口后再绑定新监听
|
||||
}
|
||||
m.srv, m.ln, m.pair = nil, nil, nil
|
||||
m.srv, m.redirectSrv, m.ln, m.pair = nil, nil, nil, nil
|
||||
|
||||
firstErr := m.bind(pair)
|
||||
if firstErr != nil {
|
||||
@@ -107,6 +121,7 @@ func (m *serverManager) Reload() error {
|
||||
}
|
||||
// 新监听已就绪,让旧服务器在新连接切换到新监听后优雅退出。
|
||||
m.drain(oldSrv)
|
||||
m.drain(oldRedirect)
|
||||
m.logServerReady()
|
||||
m.maybeStartAutoReloadLocked()
|
||||
return firstErr
|
||||
@@ -120,11 +135,19 @@ func (m *serverManager) Shutdown(ctx context.Context) error {
|
||||
close(m.stopCh)
|
||||
}
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.srv == nil {
|
||||
return nil
|
||||
targets := []*http.Server{m.srv, m.redirectSrv}
|
||||
m.mu.Unlock()
|
||||
|
||||
var firstErr error
|
||||
for _, srv := range targets {
|
||||
if srv == nil {
|
||||
continue
|
||||
}
|
||||
if err := srv.Shutdown(ctx); err != nil && firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
}
|
||||
return m.srv.Shutdown(ctx)
|
||||
return firstErr
|
||||
}
|
||||
|
||||
// desiredPair 根据当前配置计算目标监听形态:nil 表示明文 HTTP,非 nil 表示 TLS。
|
||||
@@ -217,25 +240,107 @@ func (m *serverManager) pairEquals(pair *tlsPair) bool {
|
||||
return pair.version == m.pair.version
|
||||
}
|
||||
|
||||
// bind 创建并按需启用 TLS 的监听,异步开始服务。
|
||||
// bind 创建监听,异步开始服务。
|
||||
//
|
||||
// 未开启 HTTPS 时端口只服务明文 HTTP;开启后同一个端口两种流量都要接:TLS 连接
|
||||
// 交给主服务器,明文 HTTP 连接交给只做跳转的 redirect 服务器,让 http:// 自动
|
||||
// 跳到 https://,而不是甩给用户一句 "Client sent an HTTP request to an HTTPS
|
||||
// server."。
|
||||
func (m *serverManager) bind(pair *tlsPair) error {
|
||||
ln, err := net.Listen("tcp", m.addr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("listen %s: %w", m.addr, err)
|
||||
}
|
||||
srv := &http.Server{
|
||||
Handler: m.handler,
|
||||
|
||||
if pair == nil {
|
||||
srv := m.newHTTPServer(m.handler)
|
||||
m.srv, m.redirectSrv, m.ln, m.pair = srv, nil, ln, nil
|
||||
m.writeSchemeState(nil)
|
||||
go m.serve(srv, ln)
|
||||
return nil
|
||||
}
|
||||
|
||||
split := newSplitListener(ln)
|
||||
srv := m.newHTTPServer(m.handler)
|
||||
tlsLn := tls.NewListener(split.tlsListener(), &tls.Config{
|
||||
Certificates: []tls.Certificate{pair.cert},
|
||||
MinVersion: tls.VersionTLS12,
|
||||
})
|
||||
redirect := m.newHTTPServer(m.httpsRedirectHandler())
|
||||
|
||||
m.srv, m.redirectSrv, m.ln, m.pair = srv, redirect, ln, pair
|
||||
m.writeSchemeState(pair)
|
||||
go m.serve(srv, tlsLn)
|
||||
go m.serve(redirect, split.plainListener())
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *serverManager) newHTTPServer(handler http.Handler) *http.Server {
|
||||
return &http.Server{
|
||||
Handler: handler,
|
||||
ReadHeaderTimeout: 15 * time.Second,
|
||||
}
|
||||
if pair != nil {
|
||||
ln = tls.NewListener(ln, &tls.Config{
|
||||
Certificates: []tls.Certificate{pair.cert},
|
||||
MinVersion: tls.VersionTLS12,
|
||||
})
|
||||
}
|
||||
|
||||
// httpsRedirectHandler 处理"开着 HTTPS 却用 http:// 访问"的请求:跳转到同端口、
|
||||
// 同路径的 https://。
|
||||
//
|
||||
// 用 302/307 而不是 301/308:HTTPS 开关可以在后台随时改,永久跳转会被浏览器长期
|
||||
// 缓存,等用户关掉 HTTPS 之后 http 访问会被带到已经不提供 TLS 的端口上打不开。
|
||||
func (m *serverManager) httpsRedirectHandler() http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
host := redirectHostHeader(r)
|
||||
if host == "" {
|
||||
// 没有可用的 Host(HTTP/1.0 或畸形请求头)时宁可明说,也不跳到错的地方。
|
||||
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
fmt.Fprintf(w, "HTTPS is enabled; open https://<host>:%d instead.\n", m.cfg.App.Port)
|
||||
return
|
||||
}
|
||||
status := http.StatusFound
|
||||
if r.Method != http.MethodGet && r.Method != http.MethodHead {
|
||||
status = http.StatusTemporaryRedirect // 保住方法与请求体
|
||||
}
|
||||
w.Header().Set("Location", httpsRedirectURL(host, r))
|
||||
w.WriteHeader(status)
|
||||
})
|
||||
}
|
||||
|
||||
// httpsRedirectURL 拼出同端口同路径的 HTTPS 地址。
|
||||
func httpsRedirectURL(host string, r *http.Request) string {
|
||||
target := "https://" + host + r.URL.EscapedPath()
|
||||
if r.URL.RawQuery != "" {
|
||||
target += "?" + r.URL.RawQuery
|
||||
}
|
||||
return target
|
||||
}
|
||||
|
||||
// redirectHostHeader 取重定向目标主机。HTTP/1.1 里 Host 是唯一的主机来源(端口映射
|
||||
// 到容器 8080 时它同时也是 HTTPS 端口),这里只做字符过滤,避免把请求头里的怪字符
|
||||
// 带进 Location 造成任意跳转。
|
||||
func redirectHostHeader(r *http.Request) string {
|
||||
host := strings.TrimSpace(r.Host)
|
||||
if host == "" || strings.ContainsAny(host, " \t\r\n/\\@?#") {
|
||||
return ""
|
||||
}
|
||||
return host
|
||||
}
|
||||
|
||||
// writeSchemeState 把当前实际监听的协议写入数据目录,供容器健康检查按协议探测。
|
||||
// 写失败只降级为告警:健康检查脚本在状态文件缺失时会回退试另一种协议。
|
||||
func (m *serverManager) writeSchemeState(pair *tlsPair) {
|
||||
dataDir := strings.TrimSpace(m.cfg.App.DataDir)
|
||||
if dataDir == "" {
|
||||
return
|
||||
}
|
||||
scheme := "http"
|
||||
if pair != nil {
|
||||
scheme = "https"
|
||||
}
|
||||
path := filepath.Join(dataDir, httpSchemeFileName)
|
||||
if err := os.WriteFile(path, []byte(scheme+"\n"), 0o644); err != nil { // #nosec G304 -- path is DataDir + fixed filename.
|
||||
m.log.Warn("write http scheme state failed", zap.String("path", path), zap.Error(err))
|
||||
}
|
||||
m.srv, m.ln, m.pair = srv, ln, pair
|
||||
go m.serve(srv, ln)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *serverManager) serve(s *http.Server, ln net.Listener) {
|
||||
@@ -271,7 +376,7 @@ func (m *serverManager) logServerReady() {
|
||||
zap.String("listen", m.addr),
|
||||
)
|
||||
if m.pair != nil {
|
||||
m.log.Info("HTTPS is enabled; plain HTTP is no longer served on this port",
|
||||
m.log.Info("HTTPS is enabled; plain HTTP requests on this port are redirected to HTTPS",
|
||||
zap.String("addr", m.addr),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1,13 +1,17 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/pem"
|
||||
"io"
|
||||
"math/big"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -32,6 +36,8 @@ func makeTestPairPEM(t *testing.T) (certPEM, keyPEM string) {
|
||||
NotBefore: time.Now().Add(-time.Hour),
|
||||
NotAfter: time.Now().Add(24 * time.Hour),
|
||||
DNSNames: []string{"localhost"},
|
||||
// 带上回环 IP,HTTPS 客户端就能在校验证书的前提下连 127.0.0.1:port。
|
||||
IPAddresses: []net.IP{net.ParseIP("127.0.0.1")},
|
||||
}
|
||||
der, err := x509.CreateCertificate(rand.Reader, tpl, tpl, &priv.PublicKey, priv)
|
||||
if err != nil {
|
||||
@@ -105,3 +111,136 @@ func TestDesiredPairModes(t *testing.T) {
|
||||
t.Fatal("version should change after files replaced")
|
||||
}
|
||||
}
|
||||
|
||||
// 健康检查依赖该文件判断该用 http 还是 https 探测,内容与当前监听协议必须一致,
|
||||
// 否则容器会一直显示 unhealthy 并刷 TLS handshake error 日志。
|
||||
func TestWriteSchemeState(t *testing.T) {
|
||||
m := newTestServerManager(t)
|
||||
m.cfg.App.DataDir = t.TempDir()
|
||||
path := filepath.Join(m.cfg.App.DataDir, httpSchemeFileName)
|
||||
|
||||
m.writeSchemeState(nil)
|
||||
if data, err := os.ReadFile(path); err != nil || strings.TrimSpace(string(data)) != "http" {
|
||||
t.Fatalf("plain HTTP state: data=%q err=%v", data, err)
|
||||
}
|
||||
|
||||
m.writeSchemeState(&tlsPair{})
|
||||
if data, err := os.ReadFile(path); err != nil || strings.TrimSpace(string(data)) != "https" {
|
||||
t.Fatalf("HTTPS state: data=%q err=%v", data, err)
|
||||
}
|
||||
|
||||
// DataDir 为空时不应写到工作目录(测试/裸跑二进制时会污染仓库)。
|
||||
m.cfg.App.DataDir = ""
|
||||
_ = os.Remove(path)
|
||||
m.writeSchemeState(nil)
|
||||
if _, err := os.Stat(httpSchemeFileName); !os.IsNotExist(err) {
|
||||
t.Fatalf("empty DataDir should not create %s in cwd: %v", httpSchemeFileName, err)
|
||||
}
|
||||
}
|
||||
|
||||
// 开启 HTTPS 后同一个端口要同时接 TLS 与明文 HTTP:明文请求跳到 HTTPS,而不是
|
||||
// 给用户看 "Client sent an HTTP request to an HTTPS server."。
|
||||
func TestServerManagerRedirectsPlainHTTPToHTTPS(t *testing.T) {
|
||||
certPEM, keyPEM := makeTestPairPEM(t)
|
||||
cfg := &config.Config{}
|
||||
cfg.App.Port = 8080
|
||||
cfg.App.DataDir = t.TempDir()
|
||||
cfg.App.HTTPSEnabled = true
|
||||
cfg.App.SSLCert, cfg.App.SSLKey = certPEM, keyPEM
|
||||
|
||||
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = io.WriteString(w, "secure:"+r.URL.Path)
|
||||
})
|
||||
m := newServerManager(cfg, zap.NewNop(), handler)
|
||||
m.addr = "127.0.0.1:0" // 端口 0 交给内核分配,测试之间不会撞端口
|
||||
if err := m.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = m.Shutdown(context.Background()) })
|
||||
|
||||
addr := m.ln.Addr().String()
|
||||
// 端口 0 只在第一次分配时用;热切换要绑回同一个端口(生产里端口本来就是固定的)。
|
||||
m.addr = addr
|
||||
schemeFile := filepath.Join(cfg.App.DataDir, httpSchemeFileName)
|
||||
if data, err := os.ReadFile(schemeFile); err != nil || strings.TrimSpace(string(data)) != "https" {
|
||||
t.Fatalf("scheme state = %q err=%v, want https", data, err)
|
||||
}
|
||||
|
||||
plain := &http.Client{
|
||||
Timeout: 10 * time.Second,
|
||||
CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
|
||||
}
|
||||
|
||||
resp, err := plain.Get("http://" + addr + "/library?tab=1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.StatusCode != http.StatusFound {
|
||||
t.Fatalf("plain GET status = %d, want %d", resp.StatusCode, http.StatusFound)
|
||||
}
|
||||
if got, want := resp.Header.Get("Location"), "https://"+addr+"/library?tab=1"; got != want {
|
||||
t.Fatalf("Location = %q, want %q", got, want)
|
||||
}
|
||||
|
||||
postResp, err := plain.Post("http://"+addr+"/api/me/login", "application/json", strings.NewReader("{}"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = postResp.Body.Close() }()
|
||||
if postResp.StatusCode != http.StatusTemporaryRedirect {
|
||||
t.Fatalf("plain POST status = %d, want %d", postResp.StatusCode, http.StatusTemporaryRedirect)
|
||||
}
|
||||
if got, want := postResp.Header.Get("Location"), "https://"+addr+"/api/me/login"; got != want {
|
||||
t.Fatalf("POST Location = %q, want %q", got, want)
|
||||
}
|
||||
|
||||
// 同一端口上的 TLS 连接照常拿到正常响应(校验证书,不做跳过校验)。
|
||||
pool := x509.NewCertPool()
|
||||
if !pool.AppendCertsFromPEM([]byte(certPEM)) {
|
||||
t.Fatal("append test certificate to pool failed")
|
||||
}
|
||||
secure := &http.Client{
|
||||
Timeout: 10 * time.Second,
|
||||
Transport: &http.Transport{
|
||||
TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12},
|
||||
},
|
||||
}
|
||||
secureResp, err := secure.Get("https://" + addr + "/library")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = secureResp.Body.Close() }()
|
||||
body, err := io.ReadAll(secureResp.Body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if secureResp.StatusCode != http.StatusOK || string(body) != "secure:/library" {
|
||||
t.Fatalf("https response = %d %q, want 200 %q", secureResp.StatusCode, body, "secure:/library")
|
||||
}
|
||||
|
||||
// 关掉 HTTPS 后同一端口回到纯明文服务,不再有多余的跳转服务器。
|
||||
cfg.App.HTTPSEnabled = false
|
||||
if err := m.Reload(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m.redirectSrv != nil {
|
||||
t.Fatal("redirect server should be released when HTTPS is off")
|
||||
}
|
||||
if data, err := os.ReadFile(schemeFile); err != nil || strings.TrimSpace(string(data)) != "http" {
|
||||
t.Fatalf("scheme state = %q err=%v, want http", data, err)
|
||||
}
|
||||
plainResp, err := plain.Get("http://" + addr + "/library")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = plainResp.Body.Close() }()
|
||||
plainBody, err := io.ReadAll(plainResp.Body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if plainResp.StatusCode != http.StatusOK || string(plainBody) != "secure:/library" {
|
||||
t.Fatalf("plain response after disabling https = %d %q", plainResp.StatusCode, plainBody)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -62,7 +62,7 @@ services:
|
||||
# NAS 示例:source: /vol1/1000/Media
|
||||
# Windows Docker Desktop 示例:source: D:/Media
|
||||
# create_host_path=false 可以避免路径写错时 Docker 自动创建空文件夹。
|
||||
- ./media: /media
|
||||
- ./media:/media
|
||||
|
||||
environment:
|
||||
TZ: Asia/Shanghai
|
||||
@@ -105,7 +105,9 @@ services:
|
||||
MEBOX_TRANSCODER_IDLE_TIMEOUT_SECONDS: "120"
|
||||
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "busybox wget -qO- http://127.0.0.1:8080/api/health || exit 1"]
|
||||
# 不能写死 http://:开启 HTTPS 后 8080 只接受 TLS,固定明文探测会
|
||||
# 每轮刷一条 TLS handshake error 并让容器一直 unhealthy。
|
||||
test: ["CMD", "/usr/local/bin/mebox-healthcheck"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
|
||||
@@ -115,7 +115,9 @@ services:
|
||||
MEBOX_TRANSCODER_IDLE_TIMEOUT_SECONDS: "120"
|
||||
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "busybox wget -qO- http://127.0.0.1:8080/api/health || exit 1"]
|
||||
# 不能写死 http://:开启 HTTPS 后 8080 只接受 TLS,固定明文探测会
|
||||
# 每轮刷一条 TLS handshake error 并让容器一直 unhealthy。
|
||||
test: ["CMD", "/usr/local/bin/mebox-healthcheck"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
|
||||
@@ -73,7 +73,9 @@ services:
|
||||
- "host.docker.internal:host-gateway"
|
||||
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "busybox wget -qO- http://127.0.0.1:8080/api/health || exit 1"]
|
||||
# 不能写死 http://:开启 HTTPS 后 8080 只接受 TLS,固定明文探测会
|
||||
# 每轮刷一条 TLS handshake error 并让容器一直 unhealthy。
|
||||
test: ["CMD", "/usr/local/bin/mebox-healthcheck"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
|
||||
@@ -44,7 +44,7 @@ services:
|
||||
# NAS 示例:source: /vol1/1000/Media
|
||||
# Windows Docker Desktop 示例:source: D:/Media
|
||||
# create_host_path=false 可以避免路径写错时 Docker 自动创建空文件夹。
|
||||
- ./media: /media
|
||||
- ./media:/media
|
||||
|
||||
environment:
|
||||
TZ: Asia/Shanghai
|
||||
@@ -87,7 +87,9 @@ services:
|
||||
MEBOX_TRANSCODER_IDLE_TIMEOUT_SECONDS: "120"
|
||||
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "busybox wget -qO- http://127.0.0.1:8080/api/health || exit 1"]
|
||||
# 不能写死 http://:开启 HTTPS 后 8080 只接受 TLS,固定明文探测会
|
||||
# 每轮刷一条 TLS handshake error 并让容器一直 unhealthy。
|
||||
test: ["CMD", "/usr/local/bin/mebox-healthcheck"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
|
||||
+3
-1
@@ -131,7 +131,9 @@ services:
|
||||
MEBOX_TRANSCODER_IDLE_TIMEOUT_SECONDS: "120"
|
||||
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "busybox wget -qO- http://127.0.0.1:8080/api/health || exit 1"]
|
||||
# 不能写死 http://:开启 HTTPS 后 8080 只接受 TLS,固定明文探测会
|
||||
# 每轮刷一条 TLS handshake error 并让容器一直 unhealthy。
|
||||
test: ["CMD", "/usr/local/bin/mebox-healthcheck"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
#!/bin/sh
|
||||
# MeBox 容器健康检查。
|
||||
#
|
||||
# 背景:MeBox 只监听一个端口,且开启 HTTPS 后该端口只接受 TLS(明文 HTTP 不再
|
||||
# 服务)。健康检查若固定用 http:// 探测,每轮都会被 TLS 监听拒绝并在服务日志里
|
||||
# 留下一条 "client sent an HTTP request to an HTTPS server",同时容器状态永远
|
||||
# 是 unhealthy。反过来固定用 https:// 探测,则在未开启 HTTPS 的部署上产生
|
||||
# "first record does not look like a TLS handshake"。
|
||||
#
|
||||
# 当前协议由服务进程写入 $MEBOX_APP_DATA_DIR/.http_scheme(见
|
||||
# cmd/server/server_manager.go 的 writeSchemeState)。脚本优先按该文件选择协议,
|
||||
# 文件缺失或该协议探测失败时再试另一种,避免状态文件过期导致误报不健康。
|
||||
set -u
|
||||
|
||||
port="${MEBOX_APP_PORT:-8080}"
|
||||
data_dir="${MEBOX_APP_DATA_DIR:-/data}"
|
||||
|
||||
scheme_file="$data_dir/.http_scheme"
|
||||
scheme=""
|
||||
if [ -r "$scheme_file" ]; then
|
||||
scheme="$(tr -d '[:space:]' <"$scheme_file")"
|
||||
fi
|
||||
if [ "$scheme" = "https" ]; then
|
||||
first="https"
|
||||
second="http"
|
||||
else
|
||||
first="http"
|
||||
second="https"
|
||||
fi
|
||||
|
||||
for proto in "$first" "$second"; do
|
||||
# --no-check-certificate:用户常用自签证书或 IP 访问,校验域名必然会失败。
|
||||
if busybox wget -q --spider -T 3 --no-check-certificate \
|
||||
"$proto://127.0.0.1:$port/api/health" >/dev/null 2>&1; then
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
|
||||
exit 1
|
||||
+24
-1
@@ -38,7 +38,7 @@ web/src/
|
||||
components/reader/
|
||||
```
|
||||
|
||||
**基建复用**:`internal/helper/http.go`(浏览器 UA + 代理回退 HTTP 客户端)、`internal/service/runtime_cache.go`(正文/目录/搜索缓存,内存+Redis)、`internal/service/image_proxy*`(封面与漫画图片代理)、`internal/handler/ws.go` 的 WSHub(搜索进度、追更任务推送,新增 `reader:*` topic)。
|
||||
**基建复用**:`internal/helper/http.go`(浏览器 UA + 代理回退 HTTP 客户端)、`internal/service/runtime_cache.go`(影视侧 JSON/对象缓存)、`internal/service/image_proxy*`(封面与漫画图片代理)、`internal/handler/ws.go` 的 WSHub(搜索进度、追更任务推送,新增 `reader:*` topic)。
|
||||
|
||||
**数据流**:书源 JSON 存库 → 搜索/发现时按启用的源并发抓取(errgroup + 信号量限流,超时熔断)→ 结果聚合 → 前端。正文、播放地址、图片列表由服务端组装(含 `nextContentUrl` 翻页合并)后带 TTL 缓存下发;音频流与漫画图片按需经服务端代理补 UA/Referer 头。
|
||||
|
||||
@@ -126,8 +126,31 @@ web/src/
|
||||
| P4 漫画/图片源 ✅ | 图片列表绝对化 + 经签名代理(带书源 Referer 防盗链头)+ 漫画阅读器(上下滚动/左右单页双模式、图片懒加载与加载失败占位、点击分区翻页/呼菜单、菜单进度条按图片序号、进度按图片序号记忆、下一章预取)+ imageStyle 透传 | 漫画源可看 |
|
||||
| P4.5 书源登录 ✅ | `jsLib` 一次装载(对应 legado SharedJsScope)+ `source.*` 会话方法(getVariable/setVariable/getLoginInfo/putLoginInfo/getLoginHeader/putLoginHeader/get/put)+ `cookie.*`(getCookie/setCookie/replaceCookie/removeCookie/getKey,按 eTLD+1 隔离)+ 请求自动携带 Cookie 与 loginHeader + `loginUrl`/`loginUi`(解析表单 → 按钮 action 拼在 loginUrl 后执行 → result 为表单值)+ `loginCheckJs`(会话失效自动重登/重取)+ 源变量落库 + 登录信息/Cookie 加密存储 + 服务端 toast/startBrowser 回传前端 + 登录面板与变量编辑器 | 登录类书源可登录、可留存登录态 |
|
||||
| P4.6 本地书籍 ✅ | `POST /api/reader/local/books` 上传 TXT / EPUB(上限 64MB)→ 落盘 `data/reader/local/<bookID>.<txt\|epub>` + 解析目录入 `reader_chapters` + 落库为 `origin=""`、`is_local=true` 的书架条目;TXT 自动识别 BOM/UTF-8/GBK/Big5/UTF-16 并按 legado 默认 TXT 目录规则切章(RE2 无 lookbehind,改行首锚定 + 句子启发式过滤),章定位信息存字节区间 `start:end`,读章只读该区间;EPUB 走 container.xml → OPF spine,标题优先取 NCX/NAV,正文去标签与实体;移出书架同步删落盘文件;同名重复导入覆盖更新并尽量保留进度 | 本地书可上传、可读、可删 |
|
||||
| P4.10 书源按用户独立 ✅ | 书源从全局唯一改为按用户独立:`ReaderBookSource` 增加 `user_id`,唯一键改为 `(user_id, source_url)`;导入/列表/启停/删除/搜索全部按当前用户限定;老库迁移自动删旧唯一索引、按「谁书架上有这本书」回填归属并给多引用用户复制副本、清理软删历史行;删除改为物理删除(软删会占住唯一键导致「删掉再导入」失败);正文缓存键随之改为「书源行 ID + 书本地址」,不同用户改过规则的同名书源不再共享缓存 | 用户的启停/规则/变量/删除互不影响 |
|
||||
| P4.7 正文持久缓存 ✅ | 服务端正文持久缓存(`reader_content_cache.go`:书源侧产物 + 索引落库 + 内容落盘 `cache_dir/reader-content/`、TTL/容量 LRU、目录刷新 remap、换源/移出书架清理、读穿透 singleflight)+ 批量取正文接口 `POST /api/reader/books/:id/content-batch`(对应 CacheBook 协议)+ 前端窗口预取(默认 3 章) | 慢源连续翻章不再每章等待;重复阅读零网络 |
|
||||
| P4.8 图片/字体兼容 ✅ | `coverDecodeJs`(封面解密代理,含搜索结果未入库场景)/ `ruleContent.imageDecode`(正文图片解密)+ 图片 URL 尾部 `,{headers}` options 逐图应用(options 优先于书源 header 与 Cookie)+ `java.queryTTF` / `queryBase64TTF` / `replaceFont`(sfnt cmap 0/4/6、loca 短/长、glyf 简单+复合字形) | 加密图与字体混淆正文可读 |
|
||||
| P4.9 规则稳定性 ✅ | `@put` 回写书源变量(修复跨请求丢失);`book.getVariable/putVariable` 真正注入书籍上下文并落库(对应 Book.upVariable);全部书源 `regexp2` 统一 3s 匹配超时;解压输出 32MB 上限;非 UTF-8 TXT 托管副本按 UTF-8 落盘(外部引用读时解码);换源单事务;JS `cache` 按书源命名空间隔离并落盘;HLS 双端识别(服务端 `#EXTM3U` 嗅探 + `hls` 标志下发) | 与阅读 App 行为对齐,坏源不再拖死服务 |
|
||||
| P5 体验完善 | 换源(ChangeBookSourceDialog 四档排序)、追更(定时刷新目录 + 缓存清理)、发现页(exploreUrl 标签条)、阅读器高级设置(页眉页脚提示、点击区域自定义)、书源编辑器六 Tab、备份导出 | 完整体验 |
|
||||
|
||||
### 实施记录:书源会话状态仍是跨用户共享(下一步)
|
||||
|
||||
书源本身已按用户独立,但 `ReaderSourceState`(Cookie / 登录信息 / loginHeader / 源变量)
|
||||
仍以 `source_url` 为唯一键、跨用户共享一份。两个用户导入同一书源时:
|
||||
|
||||
- 一方登录后,另一方复用同一份 Cookie;一方的登出会清掉另一方的登录态。
|
||||
|
||||
彻底隔离需要给状态表也加 `user_id`(或按书源行 ID 组织),涉及登录面板与变量读写
|
||||
的一批调用点,留作单独一轮处理。
|
||||
|
||||
### 实施记录:关于「宽松 @put 触发 panic」的复核
|
||||
|
||||
前期评估曾报告 `sourcerule.go` 的宽松 `@put` 回退正则含 RE2 不支持的反向引用、可能 panic。
|
||||
逐行复核后该结论**不成立**:`splitPutRule` 的下标全部由固定捕获分组数保证。
|
||||
真实缺陷是另外两个并已修复:
|
||||
|
||||
- `AnalyzeRule.Put` 没有 `sourcePutter` 分支,`@put` 只写进一次性的 `bookVars`,跨请求丢失;
|
||||
- `applyBookContext` 的三处调用点都在传 `book=nil`,`book.getVariable/putVariable` 与 `book.name` 长期是死代码。
|
||||
|
||||
P0–P2 是主体(约全部工作量 60–70%),P3/P4 相对独立可并行。
|
||||
|
||||
## 8. 风险与对策
|
||||
|
||||
@@ -34,7 +34,9 @@
|
||||
- 排序六档,对齐 legado `AppConfig.getBookSortByGroupId`:最近阅读(dur_chapter_time,默认)/ 最近更新(latest_chapter_time)/ 综合 / 按书名 / 按作者 / 手动顺序(order)。顶栏有快捷排序下拉,弹窗里也可选。
|
||||
- 显示项开关:未读章数徽标、更新时间。
|
||||
- 「更新目录」(对应 legado `menu_update_toc`):`POST /reader/shelf/refresh-toc` 并发重抓书架内网络书籍的目录,覆盖章节缓存;末章标题变化时刷新 `latest_chapter_time`,用于「最近更新」排序与更新时间展示。本地书籍与无书源信息的书跳过。返回 `total/updated/unchanged/failed` 四档——抓到但章数没变(完结书)记 `unchanged`「已是最新」,只有抓取或写入真的失败才记 `failed`,避免把正常结果报成「更新失败」;失败会打 Warn 日志(含书名与书源)。
|
||||
- 未实现(与 legado 的差距):书籍二级分组网格(进入分组后的封面网格)、导出/导入书架、离线下载。
|
||||
- 未实现(与 legado 的差距):书籍二级分组网格(进入分组后的封面网格)、导出/导入书架、离线下载管理界面(服务端批量缓存接口与窗口预取已可用,见下)。
|
||||
- 章节预取:阅读页对当前章之后默认 3 章走 `POST /reader/books/:id/content-batch` 批量抓取(服务端并发 + 持久缓存),翻到后续章直接命中;单章串行预取在高延迟书源上赶不上连续翻章。
|
||||
- 书源按用户独立:书源管理(列表/导入/启停/删除)与搜索都只作用于当前用户自己的书源。同一份书源不同用户可以各导入一份,各自启停、排序、改规则与变量,互不可见;Web 端无需额外交互,接口按登录用户自动隔离。
|
||||
|
||||
**书架分组(`/reader/book-groups`,仿影视模块的媒体库标签)**
|
||||
- 按用户存服务端:`reader_book_groups` 表每个用户一条,`groups` 列是 `[{name, book_ids}]` 的 JSON。结构、语义与影视的媒体库标签(`User.LibraryTags`)同构——组名 → 成员 ID、整份替换、一个成员只归一个组、组内顺序即展示顺序。
|
||||
@@ -95,11 +97,16 @@
|
||||
- 菜单打开时正文整体下移一个顶栏高度(用 `transform`,不改视口高度、不触发重新分页),顶栏不再压住开头 1–2 行。
|
||||
- 界面面板的字号/行距/段距三个调节组用 `flex-wrap`,窄屏自动折行,不会把「段距」挤出屏幕。
|
||||
- 点击分区:翻页模式渲染一层 `absolute inset-0` 的三分区按钮(左 30% 上一页 / 中 40% 呼出菜单 / 右 30% 下一页);**滚动模式不能渲染这层覆盖层**——它不是滚动容器的子节点,手机上手指落在覆盖层按钮上时浏览器找不到可滚动的祖先,纵向滑动完全失效、只剩点左右能翻屏。滚动模式改为在滚动容器自身的 `onClick` 上按 x 坐标分区(`handleZoneTap`),手指拖动不会产生 click,原生纵向滚动照常。
|
||||
- 手机横向滑动翻页(`hooks/useHorizontalSwipe.ts`):翻页模式(文本/漫画)在三分区覆盖层上再接一层触摸手势——**从左往右滑=上一页、从右往左滑=下一页**,并且**跟手**:位移超过 12px 且横向明显大于纵向(1.2 倍)之后,内容就跟着手指走,松手再按「位移 ≥45px」决定翻页还是回弹(跟手手势不再看手势时长,慢慢拖过去同样是拖动)。跟手手势一律吞掉随之而来的 click,否则短距离拖动会「回弹 + 再翻一页」;没走跟手的手势仍按原来的「≥45px + 时长 ≤800ms」判定一次滑动,短距离当点击交给分区按钮。`consumeSwipe()` 让三个分区按钮的 `onClick` 先判断这次 click 是否已被手势吃掉,标记在每次 `touchstart` 重置。
|
||||
- 跟手拖拽平移的是两个不同的元素。文本:正文列外面单挂一层「跟手层」(`pageDragRef`),分页平移由 `applyTextPageTransform` 命令式写在里层(`contentRef`),跟手层只管手指位移;松手时两层同时动(跟手位移归零 + 页码变化),合起来就是一次连续滑动。正文用 `ReaderTextPageBody` memo,避免长章每次 `setPage` 重渲染整棵段落树(手机会先卡一下再翻页)。漫画:平移 `ReaderComic` 的舞台,舞台左右各摆一页(见下一条)。
|
||||
- 漫画单页翻页的跟手轨道与预取(`ReaderComic` 的 `draggable` 分支):翻页模式且没有并排铺开两页时,当前页左右各摆一页,手指横滑整条舞台跟着走。单元格按图片序号做 key、位置用 `left` 百分比铺开(不是固定三个槽位换内容),所以翻页时变的只是 `left`,`<img>` 的 src 不变,浏览器不会重新解码、也不会先闪一下转圈。松手要翻页时先把新页瞬移到手指离开的位置(这一步必须关掉过渡,否则会先反着滑一段),读一次布局把这一帧定成过渡起点,再打开过渡滑回 0;同时页码 +1/-1,单元格的 `left` 在同一次提交里跟着换,两边相抵,屏幕上看不出切换。**两侧那两页同时就是下一页/上一页的预取**(带 `eager`:它们被 `overflow-hidden` 裁在屏幕外,懒加载会认为「离视口还远」而不去取,横滑过去就会看到转圈)——这正是「手机上翻一页要等一秒、电脑上(双页开着会顺带往后探测 8 张)不卡」的根因。
|
||||
- 滚动模式的滚轮(`useSmoothWheelScroll`):把滚轮格数累加成一个目标位置,再逐帧向它逼近,滚动连续、松手后自己滑行一段,仿手机上下滑动,而不是浏览器整格跳变。只在文本/漫画的上下滚动模式接管;翻页模式仍走下面的翻页监听器。按真实帧间隔换算逼近比例,120Hz 屏上速度不会翻倍;`ctrl/cmd+滚轮` 保留缩放;手指按下(滑动、拖滚动条、点分区)立刻停掉动画把控制权交还用户。
|
||||
- 漫画双页铺开(`hooks/useComicSpreads.ts` + `utils/comicSpread.ts`,legado 无对应项,是桌面端补充):窗口宽度 ≥900px 且漫画处于翻页模式时,一屏并排显示两页(`settings.comicDoublePage` 为设备级偏好,默认开,可在菜单里关;窗口不够宽时自动退回单页)。分组规则同 Tachiyomi 的 dual page——宽度明显大于高度(比值 ≥1.1)的图视为「跨页宽图」独占一屏,其余连续的竖版页两两配对;列表接口只给 URL,尺寸靠 `new Image()` 探测(`<img>` 渲染时顺带回填),只探测当前页往后 8 张的窗口,避免整章预载。进度仍按图片序号记,翻页时换算成「第几屏」,所以关掉再开双页进度不会错位。菜单动作行改为 `flex` 等分,容纳「双页/单页」入口(`grid-cols-N` 得写死列数字面量,多一个按钮无法复用)。
|
||||
- 漫画图片显示尺寸(`utils/comicImageFit.ts`,仅上下滚动模式):legado 的漫画阅读有「缩放」,Web 版原先只有一种写死的宽度(正文列封顶 900px),桌面端放不大也缩不小,所以补上一组档位,在界面面板的「图片尺寸」一行切换。`default` 就是老样子(900px 居中),`width` 适应宽度(正文列不再封顶、铺满窗口),`height` 适应高度(一屏一页),`long` 适应长边(整页完整可见),`original` 原图(原始像素 1:1,超出部分横向滚动)。`default`/`width` 由外层列宽决定,其余三个由 `<img>` 自身约束,**所以后三种必须放开 900px 列宽上限**,否则会被列再压回去。`height`/`long` 要按「一屏多高」换算,所以量的是滚动容器本身而不是 `window.innerHeight`(正文区是 root 里的 `flex-1`,另挂上下条之后就不等于窗口高度了),并用 `ResizeObserver` 跟随;只在启用了自尺寸档位时才挂观察器,默认档位不白白重渲染整章。`settings.comicImageFit` 是设备级偏好(不同屏幕和源分辨率合适的尺寸不同),和 `comicDoublePage` 一样只存本机、不参与账号同步。翻页模式是整页缩放进视口,本来就不存在「太大/太小」,档位对它不生效、也不显示入口。`ctrl/cmd+滚轮` 依然留给浏览器缩放,本功能只用档位做粗调。
|
||||
- **文本型漫画源按图片下发**(`internal/service/reader` 的 `imageMarkersOnly`):拷贝漫画这类书源 `bookSourceType=0`(文本),正文规则直接给 `<img>`,服务端把整行标签转成 `[img]<签名地址>` 标记行后类型仍是 `text`,前端就会走文本阅读器——分页把每张图当成一列,一屏只看得到一张,桌面端也谈不上双页铺开。现在 `GetContentForBook` / `LocalChapterContent` 在标记化之后判断「非空行全部是 `[img]` 标记」,是则把 `type` 改成 `image` 并给出 `images`,交由漫画阅读器渲染(整页缩放、双页铺开、进度按图片序号)。判定只认「全是标记」:混了正文的章节一律保持 `text`,孤立的 HTML 标签行(`<div>`/`</div>`)与空行当作无内容忽略,绝不吞掉可读文字。
|
||||
- 鼠标滚轮翻页(仅翻页模式):向上滚=上一页,向下滚=下一页;菜单打开时也不翻页。鼠标滚轮一格一页(间隔至少 220ms,与翻页动画对齐),触控板小步长累计到阈值翻一页且一次手势只翻一页(避免惯性连翻)。`ctrl/cmd+滚轮` 保留浏览器缩放。
|
||||
- **浏览器全屏(沉浸式)**:阅读页本身已经是 `fixed inset-0` 的独立布局(`/reader/*` 不套影视 Layout),所以屏幕上的留白只剩浏览器自带的地址栏/工具栏。底部动作行加「全屏 / 还原」,对 `readerRef` 调标准 Fullscreen API(`requestFullscreen` / `exitFullscreen`),进全屏前先收起菜单。按钮只在 `document.fullscreenEnabled === true` 时出现——**iOS 上的 Safari 至今不支持元素全屏**(只有 `<video>` 能全屏),所以那里不显示按钮,而不是显示一个点了没反应的死按钮;iPhone 想彻底去掉浏览器界面只能「添加到主屏幕」。离开阅读页时卸载 effect 主动 `exitFullscreen`,否则返回书架后浏览器仍停在全屏、整站都被罩住。全屏状态由 `fullscreenchange` 事件回写,不做乐观更新(用户按 Esc / 系统手势退出时也要跟着变)。
|
||||
- **手机端双击中间区域进出全屏**(`hooks/useDoubleTap.ts` + `hooks/useIsTouchDevice.ts`):中间区域单击本来就是呼出菜单,现在双击再切一次全屏,两个动作在同一块屏幕上,得先等一小会儿看有没有第二下。`useDoubleTap` 把这段窗口做成纯函数 `decideDoubleTap`(第一下进 `tap` 窗口等第二下;第二下到达取消单击、改判双击),**双击回调在第二次点击的事件里同步执行**——浏览器只认用户手势上下文里的 `requestFullscreen`,挪进定时器再调会被直接拒绝。只在「触摸设备(`(hover: none)`)+ 支持元素全屏」时启用:桌面端根本不装这套,单击立刻生效,点中间呼出菜单的手感与以前完全一致;iOS 也不装(它不支持元素全屏)。三条中间点击路径(翻页模式的三分区覆盖层、文本滚动模式的 `handleZoneTap`、漫画滚动模式的 `onZone`)统一走 `centerTap()`,左右分区点击会 `cancel()` 掉待判定的单击,免得点完翻页又弹出菜单。菜单打开时整屏遮罩先吃掉那次点击,遮罩在中间区域关闭菜单的同时 `arm()` 一个双击窗口,「双击中间切全屏」在菜单开着时也成立。手机端再给阅读根节点挂 `touch-action: manipulation`,关掉浏览器自带的双击缩放(纵向滚动与双指缩放不受影响),避免同一手势被浏览器抢着缩一次。
|
||||
- 目录:整屏面板(顶部返回 + 书名 + 章数,Virtuoso 虚拟列表,定位并高亮当前章,点章跳转)。**必须渲染在底部菜单之外**:菜单带 `backdrop-blur`,会成为 `fixed` 后代的包含块,放里面 `h-full` 只能拿到菜单高度;历史上它写的是 `top-0 + bottom-full`,两者同时存在时高度被算成 0,整块目录完全看不见。
|
||||
|
||||
**本地书籍(对应 legado 本地 TXT / EPUB)**
|
||||
|
||||
@@ -33,6 +33,7 @@ func setDefaults(v *viper.Viper) {
|
||||
v.SetDefault("database.cache_size", -40000)
|
||||
v.SetDefault("database.max_open_conns", defaultDatabaseMaxOpenConns)
|
||||
v.SetDefault("database.max_idle_conns", defaultDatabaseMaxIdleConns)
|
||||
v.SetDefault("database.postgres_trgm_index", true)
|
||||
|
||||
v.SetDefault("secrets.jwt_secret", "")
|
||||
|
||||
@@ -52,6 +53,9 @@ func setDefaults(v *viper.Viper) {
|
||||
v.SetDefault("cache.redis_prefix", "mebox")
|
||||
v.SetDefault("cache.media_ttl_seconds", 90)
|
||||
v.SetDefault("cache.emby_latest_ttl_seconds", 300)
|
||||
v.SetDefault("cache.reader_content_max_size_mb", 1024)
|
||||
v.SetDefault("cache.reader_content_ttl_hours", 168)
|
||||
v.SetDefault("cache.reader_files_ttl_hours", 720)
|
||||
|
||||
v.SetDefault("search.backend", "")
|
||||
v.SetDefault("search.opensearch_url", "")
|
||||
|
||||
@@ -62,6 +62,15 @@ func (c *Config) normalize() error {
|
||||
if c.Cache.MediaTTLSeconds < 1 {
|
||||
c.Cache.MediaTTLSeconds = 90
|
||||
}
|
||||
if c.Cache.ReaderContentMaxSizeMB < 0 {
|
||||
c.Cache.ReaderContentMaxSizeMB = 0
|
||||
}
|
||||
if c.Cache.ReaderContentTTLHours < 0 {
|
||||
c.Cache.ReaderContentTTLHours = 0
|
||||
}
|
||||
if c.Cache.ReaderFilesTTLHours < 0 {
|
||||
c.Cache.ReaderFilesTTLHours = 0
|
||||
}
|
||||
c.Search.Backend = strings.ToLower(strings.TrimSpace(c.Search.Backend))
|
||||
if c.Search.Index == "" {
|
||||
c.Search.Index = "mebox_media"
|
||||
|
||||
@@ -50,7 +50,8 @@ type AppConfig struct {
|
||||
DataDir string `mapstructure:"data_dir"`
|
||||
WebDir string `mapstructure:"web_dir"`
|
||||
// HTTPSEnabled 是否仅通过 HTTPS 提供访问。启用时必须同时配置
|
||||
// SSLCert / SSLKey(或 SSLCertPath / SSLKeyPath),保存后服务会热切换到 HTTPS。
|
||||
// SSLCert / SSLKey(或 SSLCertPath / SSLKeyPath),保存后服务会热切换到 HTTPS;
|
||||
// 同一端口上的明文 HTTP 请求会被跳转到 HTTPS,而不是直接拒绝。
|
||||
HTTPSEnabled bool `mapstructure:"https_enabled"`
|
||||
// SSLCert 是 PEM 编码的 SSL 证书内容。
|
||||
SSLCert string `mapstructure:"ssl_cert"`
|
||||
@@ -86,6 +87,10 @@ type DatabaseConfig struct {
|
||||
CacheSize int `mapstructure:"cache_size"`
|
||||
MaxOpenConns int `mapstructure:"max_open_conns"`
|
||||
MaxIdleConns int `mapstructure:"max_idle_conns"`
|
||||
// PostgresTrgmIndex 决定是否用 pg_trgm 的 GIN 三元组索引替代 SQLite 的
|
||||
// FTS5 全文索引。开启后 ILIKE '%term%' 搜索能走索引而不是整表扫描,代价是
|
||||
// media 表写入时要多维护 4 个 GIN 索引。仅对 PostgreSQL 生效。
|
||||
PostgresTrgmIndex bool `mapstructure:"postgres_trgm_index"`
|
||||
}
|
||||
|
||||
// SecretsConfig 保存 JWT / 第三方 API 密钥(不要提交值)。
|
||||
@@ -133,6 +138,13 @@ type CacheConfig struct {
|
||||
// 并发),缓存过短会让这批请求同时穿透并各自重建 payload,在低配主机
|
||||
// 上造成秒级延迟。默认 300 秒,新入库内容最迟 5 分钟后出现在最新列表。
|
||||
EmbyLatestTTLSeconds int `mapstructure:"emby_latest_ttl_seconds"`
|
||||
// ReaderContentMaxSizeMB 阅读正文持久缓存的总容量(MB),0 表示不限制。
|
||||
ReaderContentMaxSizeMB int `mapstructure:"reader_content_max_size_mb"`
|
||||
// ReaderContentTTLHours 阅读正文缓存的保留时长(小时),0 表示不过期。
|
||||
ReaderContentTTLHours int `mapstructure:"reader_content_ttl_hours"`
|
||||
// ReaderFilesTTLHours 书源文件缓存(java.cacheFile / downloadFile)的保留时长(小时)。
|
||||
// 0 表示使用内置兜底值(30 天)。
|
||||
ReaderFilesTTLHours int `mapstructure:"reader_files_ttl_hours"`
|
||||
}
|
||||
|
||||
type SearchConfig struct {
|
||||
|
||||
@@ -165,7 +165,7 @@ func MigrateCurrentToPostgres(src *gorm.DB, targetDSN string, batchSize int, log
|
||||
}
|
||||
|
||||
// 1. 初始化目标库 Schema、类型与索引
|
||||
if err := AutoMigrate(targetDB); err != nil {
|
||||
if err := AutoMigrateWithConfig(targetDB, nil, log); err != nil {
|
||||
return nil, fmt.Errorf("auto migrate target PostgreSQL: %w", err)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/glebarez/sqlite"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
)
|
||||
|
||||
// legacyReaderSource 旧版书源结构(全局唯一,没有 user_id)。
|
||||
type legacyReaderSource struct {
|
||||
ID string `gorm:"primaryKey"`
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
DeletedAt gorm.DeletedAt `gorm:"index"`
|
||||
Name string
|
||||
SourceURL string `gorm:"type:varchar(512);uniqueIndex"`
|
||||
RawJSON string
|
||||
Enabled bool
|
||||
EnabledExplore bool
|
||||
Variables string
|
||||
Header string
|
||||
}
|
||||
|
||||
func (legacyReaderSource) TableName() string { return "reader_book_sources" }
|
||||
|
||||
// 迁移路径:老库(书源全局唯一、无 user_id)升级后
|
||||
// 1. 旧唯一索引被删除,两个用户可以各持一份同 URL 的书源副本;
|
||||
// 2. 书源按「谁书架上有这本书」回填归属,多个引用者各得一份副本;
|
||||
// 3. 软删的历史书源行被清理,删除后重新导入不再撞唯一键。
|
||||
func TestReaderSourceMigrationBackfill(t *testing.T) {
|
||||
db, err := gorm.Open(sqlite.Open(filepath.Join(t.TempDir(), "legacy.db")), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Windows 上 sqlite 文件句柄会阻塞 TempDir 清理,测试结束前先关连接。
|
||||
t.Cleanup(func() {
|
||||
if sqlDB, err := db.DB(); err == nil {
|
||||
_ = sqlDB.Close()
|
||||
}
|
||||
})
|
||||
// 造老结构:只有书源表(带旧唯一索引),书籍表用当前模型建。
|
||||
if err := db.AutoMigrate(&legacyReaderSource{}, &model.ReaderBook{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
shared := &legacyReaderSource{ID: "legacy-1", Name: "共享源", SourceURL: "https://shared.example.com", Enabled: true}
|
||||
if err := db.Create(shared).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
softDeleted := &legacyReaderSource{ID: "legacy-2", Name: "已删源", SourceURL: "https://gone.example.com"}
|
||||
if err := db.Create(softDeleted).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Delete(softDeleted).Error; err != nil { // 旧版 DeleteSource 的软删
|
||||
t.Fatal(err)
|
||||
}
|
||||
// u1 两本、u2 一本都引用共享源;孤儿源无人引用。
|
||||
books := []model.ReaderBook{
|
||||
{UserID: "u1", Origin: "https://shared.example.com", BookURL: "https://shared.example.com/b/1", Name: "书1"},
|
||||
{UserID: "u1", Origin: "https://shared.example.com", BookURL: "https://shared.example.com/b/2", Name: "书2"},
|
||||
{UserID: "u2", Origin: "https://shared.example.com", BookURL: "https://shared.example.com/b/1", Name: "书1"},
|
||||
{UserID: "u2", Origin: "https://orphan.example.com", BookURL: "https://orphan.example.com/b/1", Name: "孤儿"},
|
||||
}
|
||||
for i := range books {
|
||||
if err := db.Create(&books[i]).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// 迁移。
|
||||
if err := dropLegacyReaderSourceUniqueIndex(db); err != nil {
|
||||
t.Fatalf("删旧索引失败: %v", err)
|
||||
}
|
||||
if err := db.AutoMigrate(model.AllModels()...); err != nil {
|
||||
t.Fatalf("AutoMigrate 失败: %v", err)
|
||||
}
|
||||
if err := backfillReaderSourceOwnership(db); err != nil {
|
||||
t.Fatalf("回填失败: %v", err)
|
||||
}
|
||||
|
||||
// 1. 旧唯一索引已删除:两个用户同 URL 各自一份。
|
||||
var rows []model.ReaderBookSource
|
||||
if err := db.Order("user_id ASC").Find(&rows).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
byUser := map[string]model.ReaderBookSource{}
|
||||
for _, r := range rows {
|
||||
if r.SourceURL == "https://shared.example.com" {
|
||||
byUser[r.UserID] = r
|
||||
}
|
||||
}
|
||||
if len(byUser) != 2 {
|
||||
t.Fatalf("共享源应有两份副本,实际 %d 份: %+v", len(byUser), byUser)
|
||||
}
|
||||
if byUser["u1"].ID == "" || byUser["u2"].ID == "" || byUser["u1"].ID == byUser["u2"].ID {
|
||||
t.Fatalf("两份副本应有不同行 ID: u1=%q u2=%q", byUser["u1"].ID, byUser["u2"].ID)
|
||||
}
|
||||
if byUser["u1"].RawJSON != shared.RawJSON || byUser["u1"].Name != "共享源" {
|
||||
t.Fatal("副本应保留原书源内容")
|
||||
}
|
||||
// 引用最多的用户保留原行(u1 有两本书)。
|
||||
if byUser["u1"].ID != "legacy-1" {
|
||||
t.Fatalf("原行应归引用最多的用户,实际 %q", byUser["u1"].ID)
|
||||
}
|
||||
|
||||
// 2. 软删的历史行被清理。
|
||||
var softLeft int64
|
||||
if err := db.Unscoped().Model(&model.ReaderBookSource{}).
|
||||
Where("source_url = ?", "https://gone.example.com").Count(&softLeft).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if softLeft != 0 {
|
||||
t.Fatalf("软删的历史书源应被清理,剩余 %d 行", softLeft)
|
||||
}
|
||||
|
||||
// 3. 无人引用的书源保持无归属(不会凭空分配给谁)。
|
||||
var orphans int64
|
||||
if err := db.Model(&model.ReaderBookSource{}).
|
||||
Where("source_url = ? AND user_id <> ''", "https://orphan.example.com").Count(&orphans).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if orphans != 0 {
|
||||
t.Fatal("无人引用的书源不应被分配归属")
|
||||
}
|
||||
|
||||
// 4. 回填是幂等的(再次执行不产生重复副本)。
|
||||
if err := backfillReaderSourceOwnership(db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var total int64
|
||||
if err := db.Model(&model.ReaderBookSource{}).Where("source_url = ?", "https://shared.example.com").Count(&total).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if total != 2 {
|
||||
t.Fatalf("重复回填不应新增副本,实际 %d 份", total)
|
||||
}
|
||||
}
|
||||
@@ -1,27 +1,52 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"go.uber.org/zap"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/config"
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
)
|
||||
|
||||
// AutoMigrate creates tables for every model registered in the model package.
|
||||
func AutoMigrate(db *gorm.DB) error {
|
||||
return AutoMigrateWithConfig(db, nil, nil)
|
||||
}
|
||||
|
||||
// AutoMigrateWithConfig 与 AutoMigrate 相同,另外接受配置与日志:
|
||||
// 配置用来决定可选的 PostgreSQL 搜索索引,日志用来记录耗时步骤
|
||||
// (首次在已有大库上重建索引可能需要数十秒)。两者都可以为 nil。
|
||||
func AutoMigrateWithConfig(db *gorm.DB, cfg *config.Config, log *zap.Logger) error {
|
||||
// 必须先于 AutoMigrate:旧库中可能已有重复的 (user_id, media_id) 历史行,
|
||||
// 不去重会导致唯一索引 uniq_user_history 创建失败。
|
||||
if err := dedupePlaybackHistories(db); err != nil {
|
||||
return err
|
||||
}
|
||||
// 书源从「全局唯一」改成「按用户独立」:先删掉 source_url 上的旧唯一索引。
|
||||
// AutoMigrate 只会按索引名创建新索引,不会删除已改名的旧唯一索引,留着它
|
||||
// 第二个用户导入同一书源就会撞唯一约束。
|
||||
if err := dropLegacyReaderSourceUniqueIndex(db); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := db.AutoMigrate(model.AllModels()...); err != nil {
|
||||
return err
|
||||
}
|
||||
// 回填书源归属:老库的书源没有 user_id,按「谁书架上有这本书」分配或复制副本。
|
||||
if err := backfillReaderSourceOwnership(db); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ensurePostgresColumnCompatibility(db); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ensurePerformanceIndexes(db); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ensureMediaPathIndex(db); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ensureLibraryRootsCompatibility(db); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -34,6 +59,111 @@ func AutoMigrate(db *gorm.DB) error {
|
||||
}
|
||||
return ensureSQLiteQueryOptimizer(db)
|
||||
}
|
||||
return ensurePostgresSearchIndexes(db, cfg, log)
|
||||
}
|
||||
|
||||
// ensureMediaPathIndex 建立 media.path 的唯一约束与等值查找索引。
|
||||
//
|
||||
// 模型上不再用 gorm:"uniqueIndex"(见 model.Media.Path 的注释):PostgreSQL 的
|
||||
// btree 单条索引项上限是 2704 字节,varchar(1024) 在中文路径下可达 3072 字节,
|
||||
// 对整列建唯一索引会让这类行插入时报
|
||||
//
|
||||
// index row size ... exceeds btree version 4 maximum 2704
|
||||
//
|
||||
// 所以 PostgreSQL 改用 md5(path) 的表达式唯一索引(定长 32 字节,永远塞得下),
|
||||
// 同时保留一条普通索引 idx_media_path 供 path = ? 等值查找(扫描与 upsert 的
|
||||
// 热路径)使用。SQLite 没有这个长度限制,沿用整列唯一索引。
|
||||
func ensureMediaPathIndex(db *gorm.DB) error {
|
||||
if db == nil || !db.Migrator().HasTable("media") {
|
||||
return nil
|
||||
}
|
||||
if !isPostgres(db) {
|
||||
return db.Exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_media_path ON media(path)`).Error
|
||||
}
|
||||
// 先建哈希唯一索引:此时旧索引还在,唯一性不会出现空窗。
|
||||
if err := db.Exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_media_path_hash ON media(md5(path))`).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
unique, found, err := postgresIndexIsUnique(db, "idx_media_path")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if found && unique {
|
||||
if err := db.Exec(`DROP INDEX IF EXISTS idx_media_path`).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return db.Exec(`CREATE INDEX IF NOT EXISTS idx_media_path ON media(path)`).Error
|
||||
}
|
||||
|
||||
// postgresIndexIsUnique 报告当前 schema 下是否存在该索引、以及它是否唯一。
|
||||
func postgresIndexIsUnique(db *gorm.DB, indexName string) (unique bool, found bool, err error) {
|
||||
// 用指针接 bool:没有匹配行时 Scan 不报错,nil 即代表索引不存在。
|
||||
var row struct {
|
||||
IsUnique *bool
|
||||
}
|
||||
if err := db.Raw(`
|
||||
SELECT i.indisunique AS is_unique
|
||||
FROM pg_class c
|
||||
JOIN pg_index i ON i.indexrelid = c.oid
|
||||
JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||
WHERE n.nspname = current_schema() AND c.relname = ?
|
||||
`, indexName).Scan(&row).Error; err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
if row.IsUnique == nil {
|
||||
return false, false, nil
|
||||
}
|
||||
return *row.IsUnique, true, nil
|
||||
}
|
||||
|
||||
// ensurePostgresSearchIndexes 给 PostgreSQL 补上 SQLite FTS5 的替代品。
|
||||
//
|
||||
// 全文检索只在 SQLite 下存在(FTS5 虚表 + 触发器),PostgreSQL 下
|
||||
// media_search_repository 会退回 title/original_name/path/genres 的 ILIKE
|
||||
// 全表扫描。pg_trgm 的 GIN 三元组索引能让这类 '%term%' 模式(≥3 字符)走索引,
|
||||
// 把搜索从整表扫描降到索引扫描。
|
||||
//
|
||||
// 之所以可关闭:media 表写入非常频繁(每次扫描批量入库),多 4 个 GIN 索引会
|
||||
// 明显放大写入成本。默认开启以对齐 SQLite 的搜索能力,写入压力大的部署可以关掉。
|
||||
//
|
||||
// 整个过程是尽力而为:没有建扩展的权限、Postgres 版本不支持等情况下只记一条
|
||||
// 警告并继续启动,绝不让一个搜索加速项把服务挡在门外。
|
||||
func ensurePostgresSearchIndexes(db *gorm.DB, cfg *config.Config, log *zap.Logger) error {
|
||||
if !isPostgres(db) {
|
||||
return nil
|
||||
}
|
||||
if cfg != nil && !cfg.Database.PostgresTrgmIndex {
|
||||
if log != nil {
|
||||
log.Info("postgres trigram search index disabled; media search falls back to sequential scan")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if err := db.Exec(`CREATE EXTENSION IF NOT EXISTS pg_trgm`).Error; err != nil {
|
||||
if log != nil {
|
||||
log.Warn("create pg_trgm extension failed; media search falls back to sequential scan", zap.Error(err))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
// 首次在已有大库上执行会重建索引(服务尚未开始接受请求),之后每次启动都因
|
||||
// IF NOT EXISTS 直接跳过。
|
||||
if log != nil {
|
||||
log.Info("ensuring postgres trigram search indexes")
|
||||
}
|
||||
for _, stmt := range []string{
|
||||
`CREATE INDEX IF NOT EXISTS idx_media_title_trgm ON media USING gin (title gin_trgm_ops)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_media_original_name_trgm ON media USING gin (original_name gin_trgm_ops)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_media_path_trgm ON media USING gin (path gin_trgm_ops)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_media_genres_trgm ON media USING gin (genres gin_trgm_ops)`,
|
||||
} {
|
||||
if err := db.Exec(stmt).Error; err != nil {
|
||||
if log != nil {
|
||||
log.Warn("create trigram index failed; media search may fall back to sequential scan",
|
||||
zap.String("statement", stmt), zap.Error(err))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -42,6 +172,105 @@ func ensureSQLiteQueryOptimizer(db *gorm.DB) error {
|
||||
return db.Exec("ANALYZE").Error
|
||||
}
|
||||
|
||||
// dropLegacyReaderSourceUniqueIndex 删除书源表上旧版 source_url 唯一索引。
|
||||
//
|
||||
// 书源改为按用户独立后唯一键是 (user_id, source_url);SQLite 的 RenameTable 与
|
||||
// AutoMigrate 都不会删除按旧名创建的索引(实测旧唯一索引在改名后依然生效),
|
||||
// 必须显式删除,否则第二个用户导入同一书源会撞唯一约束。
|
||||
func dropLegacyReaderSourceUniqueIndex(db *gorm.DB) error {
|
||||
if !db.Migrator().HasTable(&model.ReaderBookSource{}) {
|
||||
return nil
|
||||
}
|
||||
for _, name := range []string{"idx_reader_book_sources_source_url", "reader_book_sources_source_url_key"} {
|
||||
if db.Migrator().HasIndex(&model.ReaderBookSource{}, name) {
|
||||
if err := db.Migrator().DropIndex(&model.ReaderBookSource{}, name); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
// 旧版 DeleteSource 是软删:这些历史行会继续占用 (user_id, source_url) 唯一键,
|
||||
// 让「删掉再导入同一书源」失败。书源是可重新导入的数据,这里一次性清掉。
|
||||
return db.Unscoped().Where("deleted_at IS NOT NULL").Delete(&model.ReaderBookSource{}).Error
|
||||
}
|
||||
|
||||
// backfillReaderSourceOwnership 给老库的书源补上归属用户。
|
||||
//
|
||||
// 书源原先全局唯一,现在按用户独立:按「谁的书架上有这本书」把书源分配给该用户;
|
||||
// 同一书源被多个用户引用时,给每个用户复制一份副本(各自启停/排序/改规则),
|
||||
// 这样谁都不会因为别人删源而失去书源。
|
||||
//
|
||||
// 没有 user_id 且无人引用的书源保持原样(user_id 为空)——它不再出现在任何人的
|
||||
// 列表里,但数据仍在,需要时可以人工认领。
|
||||
func backfillReaderSourceOwnership(db *gorm.DB) error {
|
||||
if !db.Migrator().HasTable(&model.ReaderBookSource{}) {
|
||||
return nil
|
||||
}
|
||||
var pending []model.ReaderBookSource
|
||||
if err := db.Where("user_id = '' OR user_id IS NULL").Find(&pending).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range pending {
|
||||
src := pending[i]
|
||||
if err := assignOrCloneReaderSource(db, &src); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func assignOrCloneReaderSource(db *gorm.DB, src *model.ReaderBookSource) error {
|
||||
if src == nil || src.SourceURL == "" {
|
||||
return nil
|
||||
}
|
||||
// 引用该书源的书籍分布(去重到用户)。
|
||||
type ownerCount struct {
|
||||
UserID string
|
||||
N int64
|
||||
}
|
||||
var owners []ownerCount
|
||||
if err := db.Model(&model.ReaderBook{}).
|
||||
Select("user_id, COUNT(*) AS n").
|
||||
Where("origin = ?", src.SourceURL).
|
||||
Group("user_id").Order("n DESC").Scan(&owners).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if len(owners) == 0 {
|
||||
return nil // 无人引用:保持无归属
|
||||
}
|
||||
// 第一个用户(引用最多)直接接手这一行。
|
||||
owner := owners[0].UserID
|
||||
if owner == "" {
|
||||
return nil
|
||||
}
|
||||
if err := db.Model(&model.ReaderBookSource{}).
|
||||
Where("id = ?", src.ID).
|
||||
Updates(map[string]any{"user_id": owner}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
// 其余用户各复制一份副本(新行 ID,其余字段全量拷贝)。
|
||||
for _, o := range owners[1:] {
|
||||
if o.UserID == "" || o.UserID == owner {
|
||||
continue
|
||||
}
|
||||
var exists int64
|
||||
if err := db.Model(&model.ReaderBookSource{}).
|
||||
Where("user_id = ? AND source_url = ?", o.UserID, src.SourceURL).
|
||||
Count(&exists).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if exists > 0 {
|
||||
continue
|
||||
}
|
||||
clone := *src
|
||||
clone.ID = ""
|
||||
clone.UserID = o.UserID
|
||||
if err := db.Create(&clone).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// dedupePlaybackHistories removes duplicate (user_id, media_id) rows left by
|
||||
// the former read-then-write upsert, so the uniq_user_history composite unique
|
||||
// index can be created on existing databases. Keeps the most recent row per
|
||||
@@ -67,16 +296,35 @@ func ensurePostgresColumnCompatibility(db *gorm.DB) error {
|
||||
if !isPostgres(db) {
|
||||
return nil
|
||||
}
|
||||
statements := []string{
|
||||
`ALTER TABLE media ALTER COLUMN container TYPE varchar(128)`,
|
||||
`ALTER TABLE media ALTER COLUMN genres TYPE text`,
|
||||
`ALTER TABLE media ALTER COLUMN series_id TYPE varchar(128)`,
|
||||
`ALTER TABLE media ALTER COLUMN duplicate_of TYPE varchar(128)`,
|
||||
`ALTER TABLE playback_histories ALTER COLUMN media_id TYPE varchar(128)`,
|
||||
`ALTER TABLE favorites ALTER COLUMN media_id TYPE varchar(128)`,
|
||||
`ALTER TABLE playlist_items ALTER COLUMN media_id TYPE varchar(128)`,
|
||||
// 只在类型确实不匹配时才 ALTER。
|
||||
//
|
||||
// PostgreSQL 的 ALTER COLUMN ... TYPE 只要类型声明与目标不同就会整表重写并
|
||||
// 重建相关索引,执行期间持有 ACCESS EXCLUSIVE 锁。此前这里每次启动都无条件
|
||||
// 发这 7 条语句:类型其实早已一致,但几十万行的 media 表仍然被反复重写,
|
||||
// 启动会卡住数分钟,且期间这张表连读都被阻塞。
|
||||
changes := []struct {
|
||||
table string
|
||||
column string
|
||||
target string
|
||||
}{
|
||||
{"media", "container", "varchar(128)"},
|
||||
{"media", "genres", "text"},
|
||||
{"media", "series_id", "varchar(128)"},
|
||||
{"media", "duplicate_of", "varchar(128)"},
|
||||
{"playback_histories", "media_id", "varchar(128)"},
|
||||
{"favorites", "media_id", "varchar(128)"},
|
||||
{"playlist_items", "media_id", "varchar(128)"},
|
||||
}
|
||||
for _, stmt := range statements {
|
||||
for _, change := range changes {
|
||||
current, found, err := postgresColumnType(db, change.table, change.column)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !found || current == change.target {
|
||||
continue
|
||||
}
|
||||
stmt := "ALTER TABLE " + quoteIdent(change.table) +
|
||||
" ALTER COLUMN " + quoteIdent(change.column) + " TYPE " + change.target
|
||||
if err := db.Exec(stmt).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -84,6 +332,35 @@ func ensurePostgresColumnCompatibility(db *gorm.DB) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// postgresColumnType 返回列在当前 schema 下的规范化类型描述,例如
|
||||
// "varchar(128)" / "text"。列不存在时 found 为 false。
|
||||
func postgresColumnType(db *gorm.DB, table, column string) (value string, found bool, err error) {
|
||||
var row struct {
|
||||
DataType string
|
||||
MaxLength *int64
|
||||
}
|
||||
if err := db.Raw(`
|
||||
SELECT data_type AS data_type, character_maximum_length AS max_length
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = current_schema() AND table_name = ? AND column_name = ?
|
||||
`, table, column).Scan(&row).Error; err != nil {
|
||||
return "", false, fmt.Errorf("inspect postgres column %s.%s: %w", table, column, err)
|
||||
}
|
||||
dataType := strings.ToLower(strings.TrimSpace(row.DataType))
|
||||
if dataType == "" {
|
||||
return "", false, nil
|
||||
}
|
||||
// information_schema 用的是 SQL 标准名,映射回 GORM 标签里的写法再比较。
|
||||
switch dataType {
|
||||
case "character varying", "varchar":
|
||||
dataType = "varchar"
|
||||
}
|
||||
if row.MaxLength != nil && *row.MaxLength > 0 {
|
||||
dataType = fmt.Sprintf("%s(%d)", dataType, *row.MaxLength)
|
||||
}
|
||||
return dataType, true, nil
|
||||
}
|
||||
|
||||
func ensurePerformanceIndexes(db *gorm.DB) error {
|
||||
statements := []string{
|
||||
// 完整多级排序索引:媒体库分页与首页预览的 ORDER BY
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/glebarez/sqlite"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/logger"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
)
|
||||
|
||||
// TestAutoMigrateCreatesMediaPathUniqueIndex 守住 media.path 的唯一约束。
|
||||
//
|
||||
// 模型上不再声明 gorm:"uniqueIndex"(PostgreSQL 的 btree 单条索引项上限是
|
||||
// 2704 字节,而 varchar(1024) 在中文路径下可达 3072 字节,对整列建唯一索引会让
|
||||
// 这类行插入直接失败),改由 AutoMigrate 里的 ensureMediaPathIndex 按方言显式
|
||||
// 建立。一旦这个调用被漏掉,扫描会静默产生重复的媒体行(同一文件被入库多次)。
|
||||
func TestAutoMigrateCreatesMediaPathUniqueIndex(t *testing.T) {
|
||||
db, err := gorm.Open(sqlite.Open(filepath.Join(t.TempDir(), "mebox.db")), &gorm.Config{
|
||||
Logger: logger.Default.LogMode(logger.Silent),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// 必须在 TempDir 清理之前关掉连接,否则 Windows 上删除库文件会失败。
|
||||
// t.Cleanup 是后进先出:这里注册的关闭早于 t.TempDir 注册的删除。
|
||||
if sqlDB, err := db.DB(); err == nil {
|
||||
t.Cleanup(func() { _ = sqlDB.Close() })
|
||||
}
|
||||
if err := AutoMigrate(db); err != nil {
|
||||
t.Fatalf("AutoMigrate: %v", err)
|
||||
}
|
||||
|
||||
var definition string
|
||||
if err := db.Raw(`SELECT COALESCE(sql, '') FROM sqlite_master WHERE type = 'index' AND name = 'idx_media_path'`).Scan(&definition).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if definition == "" {
|
||||
t.Fatal("AutoMigrate did not create idx_media_path")
|
||||
}
|
||||
if !strings.Contains(strings.ToUpper(definition), "UNIQUE") {
|
||||
t.Fatalf("idx_media_path is not unique: %q", definition)
|
||||
}
|
||||
|
||||
if err := db.Create(&model.Media{Base: model.Base{ID: "m-1"}, Path: "/media/a.mkv"}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Create(&model.Media{Base: model.Base{ID: "m-2"}, Path: "/media/a.mkv"}).Error; err == nil {
|
||||
t.Fatal("a duplicate media path was accepted; the unique constraint is not enforced")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/config"
|
||||
"github.com/truewhile/MeBox/internal/service"
|
||||
)
|
||||
|
||||
// 云 HLS 分片代理的错误映射:
|
||||
// - 会话不存在/过期要给前端可判断的 code,前端据此重拉 master.m3u8 建会话续播,
|
||||
// 而不是当成播放失败直接退回本地转码;
|
||||
// - 客户端主动断开(seek、切清晰度、关页面取消在途请求)不是网关故障,记 499,
|
||||
// 与 nginx 的口径一致。以前统一记 502,既污染监控又会让前端拿到误导性错误。
|
||||
func newCloud115HLSSessionTestRouter() *gin.Engine {
|
||||
gin.SetMode(gin.TestMode)
|
||||
svc := &service.Container{
|
||||
Cloud115: service.NewCloud115PlaybackService(&config.Config{}, zap.NewNop(), nil, nil),
|
||||
}
|
||||
router := gin.New()
|
||||
router.GET("/api/cloud115/hls/:session/:key", cloud115HLSSessionHandler(svc))
|
||||
return router
|
||||
}
|
||||
|
||||
func TestCloud115HLSSessionUnknownSessionReturnsCodedNotFound(t *testing.T) {
|
||||
router := newCloud115HLSSessionTestRouter()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/cloud115/hls/unknown-session/abc?media_id=media-1", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusNotFound {
|
||||
t.Fatalf("status = %d, want 404; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if !strings.Contains(rec.Body.String(), "hls_session_not_found") {
|
||||
t.Fatalf("expected machine-readable code in body, got %s", rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloud115HLSSessionClientAbortReturns499(t *testing.T) {
|
||||
router := newCloud115HLSSessionTestRouter()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/cloud115/hls/unknown-session/abc?media_id=media-1", nil).
|
||||
WithContext(ctx)
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != 499 {
|
||||
t.Fatalf("status = %d, want 499 (client closed request); body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"strconv"
|
||||
@@ -136,11 +137,21 @@ func cloud115HLSSessionHandler(svc *service.Container) gin.HandlerFunc {
|
||||
if err == nil || c.Writer.Written() {
|
||||
return
|
||||
}
|
||||
// 客户端主动断开(播放器 seek、切切清晰度、关页面时取消在途请求)会让
|
||||
// 上游请求以 context.Canceled 结束。这是正常现象,不是网关故障;以前
|
||||
// 统一记成 502,既污染监控又和 nginx 的 499 对不上,这里按 499 记。
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) ||
|
||||
c.Request.Context().Err() != nil {
|
||||
c.AbortWithStatus(499)
|
||||
return
|
||||
}
|
||||
switch {
|
||||
case errors.Is(err, service.ErrCloud115HLSSessionNotFound):
|
||||
c.JSON(http.StatusNotFound, gin.H{"error": "hls session not found"})
|
||||
// code 供前端判断「会话没了,需要重新拉一次 master.m3u8 建会话」,
|
||||
// 而不是直接放弃云端播放退回本地转码。
|
||||
c.JSON(http.StatusNotFound, gin.H{"code": "hls_session_not_found", "error": "hls session not found"})
|
||||
case errors.Is(err, service.ErrCloud115HLSUpstreamExpired):
|
||||
c.JSON(http.StatusGone, gin.H{"error": "hls upstream expired"})
|
||||
c.JSON(http.StatusGone, gin.H{"code": "hls_session_expired", "error": "hls upstream expired"})
|
||||
default:
|
||||
c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()})
|
||||
}
|
||||
|
||||
@@ -810,6 +810,20 @@ func streamHandler(svc *service.Container) gin.HandlerFunc {
|
||||
c.JSON(http.StatusNotFound, gin.H{"error": "not found"})
|
||||
return
|
||||
}
|
||||
// 远程条目可能是「整部剧/季/目录」:这类条目自身没有媒体源,远程 Emby 对它
|
||||
// 们的 /Videos/{id}/stream 会直接回 500(Unable to cast object of type
|
||||
// 'Series' to type 'IHasMediaSources')。网页端、外部播放器、第三方客户端都
|
||||
// 可能把这种 ID 送进来,这里统一改投第一集:302 让客户端自己跟过去,播放器
|
||||
// 原有的 token / proxy 查询参数一并保留。
|
||||
if playable := svc.EmbyRemote.PlayableRemoteID(ctx, acct, remoteID); playable != "" && playable != remoteID {
|
||||
target := "/api/stream/" + service.EncodeEmbyRemoteID(mountID, playable)
|
||||
if rawQuery := c.Request.URL.RawQuery; rawQuery != "" {
|
||||
target += "?" + rawQuery
|
||||
}
|
||||
setRedirectNoStoreHeaders(c)
|
||||
c.Redirect(http.StatusFound, target)
|
||||
return
|
||||
}
|
||||
// 远程 Emby 条目的同源转发:VR 全景需要浏览器读帧(见 stream_proxy.go),
|
||||
// 与挂载账号的 proxy_play 开关等价。
|
||||
if mount.ProxyPlay || wantSameOriginProxy(c) {
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
"github.com/truewhile/MeBox/internal/service"
|
||||
"github.com/truewhile/MeBox/internal/service/reader"
|
||||
readerrule "github.com/truewhile/MeBox/internal/service/reader/rule"
|
||||
)
|
||||
|
||||
func registerReaderRoutes(authed *gin.RouterGroup, svc *service.Container) {
|
||||
@@ -77,6 +78,8 @@ func registerReaderRoutes(authed *gin.RouterGroup, svc *service.Container) {
|
||||
g.GET("/books/:id/chapters", readerListChaptersHandler(svc))
|
||||
g.POST("/books/:id/chapters", readerReplaceChaptersHandler(svc))
|
||||
g.GET("/books/:id/content", readerBookContentHandler(svc))
|
||||
// 批量取正文(离线缓存协议,对应 legado CacheBook):返回每章内容与命中统计
|
||||
g.POST("/books/:id/content-batch", readerBookContentBatchHandler(svc))
|
||||
|
||||
// 替换净化规则
|
||||
g.GET("/replace-rules", readerListReplaceRulesHandler(svc))
|
||||
@@ -87,7 +90,8 @@ func registerReaderRoutes(authed *gin.RouterGroup, svc *service.Container) {
|
||||
|
||||
func readerListSourcesHandler(svc *service.Container) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
sources, err := svc.Reader.ListSources(c.Request.Context())
|
||||
userID := c.GetString(middleware.CtxUserID)
|
||||
sources, err := svc.Reader.ListSources(c.Request.Context(), userID)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
@@ -105,7 +109,8 @@ func readerImportSourcesHandler(svc *service.Container) gin.HandlerFunc {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
imported, err := svc.Reader.ImportSources(c.Request.Context(), body.Text)
|
||||
userID := c.GetString(middleware.CtxUserID)
|
||||
imported, err := svc.Reader.ImportSources(c.Request.Context(), userID, body.Text)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
@@ -123,7 +128,8 @@ func readerUpdateSourceHandler(svc *service.Container) gin.HandlerFunc {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "enabled 字段必填"})
|
||||
return
|
||||
}
|
||||
if err := svc.Reader.UpdateSourceEnabled(c.Request.Context(), c.Param("id"), *body.Enabled); err != nil {
|
||||
userID := c.GetString(middleware.CtxUserID)
|
||||
if err := svc.Reader.UpdateSourceEnabled(c.Request.Context(), userID, c.Param("id"), *body.Enabled); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
@@ -133,7 +139,8 @@ func readerUpdateSourceHandler(svc *service.Container) gin.HandlerFunc {
|
||||
|
||||
func readerDeleteSourceHandler(svc *service.Container) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
if err := svc.Reader.DeleteSource(c.Request.Context(), c.Param("id")); err != nil {
|
||||
userID := c.GetString(middleware.CtxUserID)
|
||||
if err := svc.Reader.DeleteSource(c.Request.Context(), userID, c.Param("id")); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
@@ -150,7 +157,8 @@ func readerDebugSourceHandler(svc *service.Container) gin.HandlerFunc {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
logs, err := svc.Reader.Debug(c.Request.Context(), c.Param("id"), body.Key)
|
||||
userID := c.GetString(middleware.CtxUserID)
|
||||
logs, err := svc.Reader.Debug(c.Request.Context(), userID, c.Param("id"), body.Key)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
@@ -315,7 +323,8 @@ func readerBrowserAssetHandler(svc *service.Container) gin.HandlerFunc {
|
||||
|
||||
func readerSourceLogoutHandler(svc *service.Container) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
if err := svc.Reader.ClearSourceLogin(c.Request.Context(), c.Param("id")); err != nil {
|
||||
userID := c.GetString(middleware.CtxUserID)
|
||||
if err := svc.Reader.ClearSourceLogin(c.Request.Context(), userID, c.Param("id")); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
@@ -332,7 +341,8 @@ func readerSetSourceVariableHandler(svc *service.Container) gin.HandlerFunc {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
if err := svc.Reader.SetSourceVariable(c.Request.Context(), c.Param("id"), body.Variable); err != nil {
|
||||
userID := c.GetString(middleware.CtxUserID)
|
||||
if err := svc.Reader.SetSourceVariable(c.Request.Context(), userID, c.Param("id"), body.Variable); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
@@ -349,7 +359,8 @@ func readerSetSourceLoginInfoHandler(svc *service.Container) gin.HandlerFunc {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
if err := svc.Reader.SetSourceLoginInfo(c.Request.Context(), c.Param("id"), body.Fields); err != nil {
|
||||
userID := c.GetString(middleware.CtxUserID)
|
||||
if err := svc.Reader.SetSourceLoginInfo(c.Request.Context(), userID, c.Param("id"), body.Fields); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
@@ -370,7 +381,8 @@ func readerSearchHandler(svc *service.Container) gin.HandlerFunc {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
books, skipped, err := svc.Reader.Search(c.Request.Context(), body.Key, body.SourceIDs, body.Page)
|
||||
userID := c.GetString(middleware.CtxUserID)
|
||||
books, skipped, err := svc.Reader.Search(c.Request.Context(), userID, body.Key, body.SourceIDs, body.Page)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
@@ -389,6 +401,10 @@ func readerBookInfoHandler(svc *service.Container) gin.HandlerFunc {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
// 声明了封面解密的书源:详情封面也要走解密代理(尚未入库,按书源 URL 签)。
|
||||
if info != nil {
|
||||
info.CoverURL = svc.Reader.RewriteBookCover(c.Request.Context(), "", c.Query("source_url"), info.CoverURL)
|
||||
}
|
||||
c.JSON(http.StatusOK, info)
|
||||
}
|
||||
}
|
||||
@@ -435,6 +451,10 @@ func readerListBooksHandler(svc *service.Container) gin.HandlerFunc {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
// 封面解密代理只在下发时改写:库里仍存原始地址,换源/换签名密钥后不会失效。
|
||||
for i := range books {
|
||||
books[i].CoverURL = svc.Reader.RewriteBookCover(c.Request.Context(), books[i].ID, books[i].Origin, books[i].CoverURL)
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"books": books})
|
||||
}
|
||||
}
|
||||
@@ -819,12 +839,37 @@ func readerMediaProxyHandler(svc *service.Container) gin.HandlerFunc {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "仅支持 http(s) 媒体地址"})
|
||||
return
|
||||
}
|
||||
// 封面解密代理(d=cover):签名主体可能是书源 URL(搜索结果尚未入库)。
|
||||
// 封面不读取书籍正文,因此不需要书籍记录;统一整段读取后解密下发。
|
||||
if c.Query("d") == "cover" {
|
||||
resp, fetchErr := svc.Reader.FetchCover(c.Request.Context(), bookID, rawURL)
|
||||
if fetchErr != nil {
|
||||
c.JSON(http.StatusBadGateway, gin.H{"error": "封面拉取失败: " + fetchErr.Error()})
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
data, readErr := io.ReadAll(io.LimitReader(resp.Body, readerImageDecodeMaxBytes+1))
|
||||
if readErr != nil || len(data) > readerImageDecodeMaxBytes {
|
||||
c.JSON(http.StatusBadGateway, gin.H{"error": "封面过大或读取失败"})
|
||||
return
|
||||
}
|
||||
decoded := svc.Reader.DecodeCoverBytes(c.Request.Context(), bookID, rawURL, data)
|
||||
contentType := resp.Header.Get("Content-Type")
|
||||
if contentType == "" {
|
||||
contentType = http.DetectContentType(decoded)
|
||||
}
|
||||
c.Data(resp.StatusCode, contentType, decoded)
|
||||
return
|
||||
}
|
||||
book, err := svc.Reader.GetBook(c.Request.Context(), bookID)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusNotFound, gin.H{"error": "书籍不存在"})
|
||||
return
|
||||
}
|
||||
resp, err := svc.Reader.FetchMedia(c.Request.Context(), book, rawURL, c.GetHeader("Range"))
|
||||
// 图片地址可自带 ",{headers:{...}}" 选项段:拆分后 headers 逐图应用,
|
||||
// 代理地址里只保留不带选项的地址(见 ProxyURL)。
|
||||
rawURL, mediaOptions, _ := readerrule.ParseMediaOptions(rawURL)
|
||||
resp, err := svc.Reader.FetchMediaWithOptions(c.Request.Context(), book, rawURL, c.GetHeader("Range"), mediaOptions.Headers)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadGateway, gin.H{"error": "媒体拉取失败: " + err.Error()})
|
||||
return
|
||||
@@ -832,13 +877,51 @@ func readerMediaProxyHandler(svc *service.Container) gin.HandlerFunc {
|
||||
defer resp.Body.Close()
|
||||
|
||||
ct := resp.Header.Get("Content-Type")
|
||||
isPlaylist := strings.Contains(ct, "mpegurl") || strings.Contains(ct, "m3u8") ||
|
||||
strings.HasSuffix(strings.ToLower(rawURL), ".m3u8")
|
||||
if isPlaylist {
|
||||
// m3u8:改写分片/密钥地址为签名代理后返回,hls.js 无感续播
|
||||
data, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
|
||||
rewritten := svc.Reader.RewritePlaylist(bookID, rawURL, string(data))
|
||||
c.Data(http.StatusOK, "application/vnd.apple.mpegurl", []byte(rewritten))
|
||||
// 书源声明了 imageDecode 时,图片字节需要服务端二次解密。
|
||||
// 这类响应必须整段读取后处理,不再支持 Range(解密后长度会变)。
|
||||
if decodeJS := svc.Reader.ImageDecodeRule(c.Request.Context(), book); decodeJS != "" && !isMediaPlaylistRequest(resp) {
|
||||
sniffed, readErr := io.ReadAll(io.LimitReader(resp.Body, readerImageDecodeMaxBytes+1))
|
||||
if readErr != nil || len(sniffed) > readerImageDecodeMaxBytes {
|
||||
c.JSON(http.StatusBadGateway, gin.H{"error": "图片过大或读取失败"})
|
||||
return
|
||||
}
|
||||
decoded, decErr := svc.Reader.DecodeImageBytes(c.Request.Context(), book, rawURL, false, sniffed)
|
||||
if decErr != nil {
|
||||
c.JSON(http.StatusBadGateway, gin.H{"error": "图片解密失败: " + decErr.Error()})
|
||||
return
|
||||
}
|
||||
contentType := ct
|
||||
if contentType == "" {
|
||||
contentType = http.DetectContentType(decoded)
|
||||
}
|
||||
c.Data(resp.StatusCode, contentType, decoded)
|
||||
return
|
||||
}
|
||||
// m3u8 判定不能只看 Content-Type 或 URL 后缀:上游经常把播放列表标成
|
||||
// application/octet-stream,或者地址是 /index.m3u8?token=...(后缀判断不命中)。
|
||||
// 先读一小段用 #EXTM3U 标记确认;不是播放列表就把这段拼回响应体继续流式透传。
|
||||
//
|
||||
// 只嗅探 200 的 GET:206 的体是二进制分片、HEAD 没有体,都不需要判断。
|
||||
if c.Request.Method == http.MethodGet && resp.StatusCode == http.StatusOK {
|
||||
head, readErr := io.ReadAll(io.LimitReader(resp.Body, 64<<10))
|
||||
if readErr == nil && reader.BodyIsPlaylist(head, ct) {
|
||||
rest, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
|
||||
rewritten := svc.Reader.RewritePlaylist(bookID, rawURL, string(head)+string(rest))
|
||||
c.Data(http.StatusOK, "application/vnd.apple.mpegurl", []byte(rewritten))
|
||||
return
|
||||
}
|
||||
// 不是播放列表:透传时 Content-Length 可能因为已经读过一部分而失配,
|
||||
// 直接省略交给 net/http 按 chunked 处理,避免少写/多写导致截断。
|
||||
for _, h := range []string{"Content-Type", "Content-Range", "Accept-Ranges"} {
|
||||
if v := resp.Header.Get(h); v != "" {
|
||||
c.Header(h, v)
|
||||
}
|
||||
}
|
||||
c.Status(resp.StatusCode)
|
||||
if len(head) > 0 {
|
||||
_, _ = c.Writer.Write(head)
|
||||
}
|
||||
_, _ = io.Copy(c.Writer, resp.Body)
|
||||
return
|
||||
}
|
||||
// 流式透传(含 206 Partial Content,支持音频拖动进度)
|
||||
@@ -852,6 +935,22 @@ func readerMediaProxyHandler(svc *service.Container) gin.HandlerFunc {
|
||||
}
|
||||
}
|
||||
|
||||
// readerImageDecodeMaxBytes 需要服务端解密的单张图片上限。
|
||||
const readerImageDecodeMaxBytes = 32 << 20
|
||||
|
||||
// isMediaPlaylistRequest 判断响应是否可能已是 HLS 播放列表。
|
||||
// 播放列表太小、不该走图片解密分支(书源的 imageDecode 只会用于图片)。
|
||||
func isMediaPlaylistRequest(resp *http.Response) bool {
|
||||
ct := strings.ToLower(resp.Header.Get("Content-Type"))
|
||||
return strings.Contains(ct, "mpegurl")
|
||||
}
|
||||
|
||||
// readerBodyIsPlaylist 判断上游响应体是否是 HLS 播放列表。
|
||||
// 以 #EXTM3U 标记为准:Content-Type 只是辅助(上游常标成 octet-stream)。
|
||||
func readerBodyIsPlaylist(head []byte, contentType string) bool {
|
||||
return reader.BodyIsPlaylist(head, contentType)
|
||||
}
|
||||
|
||||
// readerLocalAssetHandler 本地书籍内嵌资源(EPUB 图片等):
|
||||
// 鉴权走 HMAC 签名(<img src> 带不上 JWT),与 /reader/media 同一套做法。
|
||||
func readerLocalAssetHandler(svc *service.Container) gin.HandlerFunc {
|
||||
@@ -940,3 +1039,32 @@ func readerBookContentHandler(svc *service.Container) gin.HandlerFunc {
|
||||
c.JSON(http.StatusOK, content)
|
||||
}
|
||||
}
|
||||
|
||||
// readerBookContentBatchHandler 批量取正文(对应 legado CacheBook / getContentBatch)。
|
||||
//
|
||||
// 请求 {chapter_indexes:[...], apply_replace:bool}:单章失败不中断整批,
|
||||
// 返回每章内容与 hit/miss/failed 统计,供前端窗口预取与后续离线缓存。
|
||||
func readerBookContentBatchHandler(svc *service.Container) gin.HandlerFunc {
|
||||
var body struct {
|
||||
ChapterIndexes []int `json:"chapter_indexes" binding:"required"`
|
||||
// ApplyReplace 是否在服务端应用用户替换净化规则(默认 false,与单章接口一致)。
|
||||
ApplyReplace bool `json:"apply_replace"`
|
||||
}
|
||||
return func(c *gin.Context) {
|
||||
if err := c.ShouldBindJSON(&body); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "chapter_indexes 不能为空"})
|
||||
return
|
||||
}
|
||||
if len(body.ChapterIndexes) > 64 {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "一次最多请求 64 章"})
|
||||
return
|
||||
}
|
||||
userID := c.GetString(middleware.CtxUserID)
|
||||
result, err := svc.Reader.GetContentBatch(c.Request.Context(), userID, c.Param("id"), body.ChapterIndexes, body.ApplyReplace)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, result)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,200 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/glebarez/sqlite"
|
||||
"go.uber.org/zap"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/config"
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
"github.com/truewhile/MeBox/internal/repository"
|
||||
"github.com/truewhile/MeBox/internal/service"
|
||||
)
|
||||
|
||||
// 首页轮播/剧集卡片交给播放器的是「整部剧」的伪装 ID,它自身没有媒体源:远程 Emby
|
||||
// 对它的取流请求会回 500(Unable to cast 'Series' to 'IHasMediaSources')。这里锁定
|
||||
// 播放接口的行为——改投第一集,且客户端原有的查询参数(token / profile / proxy)
|
||||
// 必须原样带过去,否则播放器跟过去之后会掉鉴权。
|
||||
func TestStreamRedirectsRemoteContainerToPlayableEpisode(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
switch r.URL.Path {
|
||||
case "/emby/Users/uid-1/Items/series-100":
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"Id": "series-100", "Name": "某部新番", "Type": "Series", "IsFolder": true,
|
||||
})
|
||||
case "/emby/Users/uid-1/Items":
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"Items": []map[string]any{
|
||||
map[string]any{
|
||||
"Id": "ep-1", "Name": "第1集", "Type": "Episode", "SeriesId": "series-100",
|
||||
"ParentIndexNumber": 1, "IndexNumber": 1,
|
||||
"MediaSources": []any{map[string]any{"Id": "ep-1-src"}},
|
||||
},
|
||||
},
|
||||
})
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}))
|
||||
defer upstream.Close()
|
||||
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.AutoMigrate(
|
||||
&model.User{}, &model.UserPermission{}, &model.Setting{}, &model.Library{}, &model.Media{},
|
||||
&model.StrmAccount{}, &model.EmbyMount{},
|
||||
); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
repos := repository.New(db)
|
||||
cfg := &config.Config{}
|
||||
cfg.Secrets.JWTSecret = "stream-redirect-secret"
|
||||
log := zap.NewNop()
|
||||
|
||||
remote := service.NewEmbyRemoteService(cfg, log, repos, service.NewCryptoService("", log))
|
||||
svc := &service.Container{Cfg: cfg, Repo: repos, EmbyRemote: remote}
|
||||
|
||||
rawConfig, _ := json.Marshal(map[string]string{
|
||||
"url": upstream.URL,
|
||||
"api_key": "test-api-key",
|
||||
"remote_user_id": "uid-1",
|
||||
})
|
||||
if err := repos.StrmAccount.Create(t.Context(), &model.StrmAccount{
|
||||
Base: model.Base{ID: "acct-1"},
|
||||
Name: "远程 Emby",
|
||||
Provider: model.StrmProviderEmbyRemote,
|
||||
Config: string(rawConfig),
|
||||
Enabled: true,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := repos.EmbyMount.Create(t.Context(), &model.EmbyMount{
|
||||
Base: model.Base{ID: "mount-1"},
|
||||
AccountID: "acct-1",
|
||||
RemoteViewID: "view-1",
|
||||
RemoteViewName: "新番连载",
|
||||
CollectionType: "tvshows",
|
||||
Enabled: true,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
router := gin.New()
|
||||
router.GET("/api/stream/:id", streamHandler(svc))
|
||||
|
||||
req := httptest.NewRequest(
|
||||
http.MethodGet,
|
||||
"/api/stream/"+service.EncodeEmbyRemoteID("mount-1", "series-100")+"?token=tok-1&profile=p-1",
|
||||
nil,
|
||||
)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusFound {
|
||||
t.Fatalf("status = %d body=%s, want 302", w.Code, w.Body.String())
|
||||
}
|
||||
want := "/api/stream/" + service.EncodeEmbyRemoteID("mount-1", "ep-1") + "?token=tok-1&profile=p-1"
|
||||
if got := w.Header().Get("Location"); got != want {
|
||||
t.Fatalf("Location = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// 本身可播的远程条目(单集/电影)不能被改写,必须照原样进入原来的代理/直连分支。
|
||||
func TestStreamKeepsPlayableRemoteItem(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
|
||||
var streamCalls int
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
switch r.URL.Path {
|
||||
case "/emby/Users/uid-1/Items/ep-1":
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"Id": "ep-1", "Name": "第1集", "Type": "Episode", "SeriesId": "series-100",
|
||||
"ParentIndexNumber": 1, "IndexNumber": 1,
|
||||
"MediaSources": []any{map[string]any{"Id": "ep-1-src"}},
|
||||
})
|
||||
case "/emby/Videos/ep-1/stream":
|
||||
streamCalls++
|
||||
w.Header().Set("Content-Type", "video/mp4")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte("bytes"))
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}))
|
||||
defer upstream.Close()
|
||||
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.AutoMigrate(
|
||||
&model.User{}, &model.UserPermission{}, &model.Setting{}, &model.Library{}, &model.Media{},
|
||||
&model.StrmAccount{}, &model.EmbyMount{},
|
||||
); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
repos := repository.New(db)
|
||||
cfg := &config.Config{}
|
||||
cfg.Secrets.JWTSecret = "stream-keep-secret"
|
||||
log := zap.NewNop()
|
||||
|
||||
remote := service.NewEmbyRemoteService(cfg, log, repos, service.NewCryptoService("", log))
|
||||
emby := service.NewEmbyService(cfg, log, repos).SetEmbyRemote(remote)
|
||||
svc := &service.Container{Cfg: cfg, Repo: repos, EmbyRemote: remote, Emby: emby}
|
||||
|
||||
rawConfig, _ := json.Marshal(map[string]string{
|
||||
"url": upstream.URL,
|
||||
"api_key": "test-api-key",
|
||||
"remote_user_id": "uid-1",
|
||||
})
|
||||
if err := repos.StrmAccount.Create(t.Context(), &model.StrmAccount{
|
||||
Base: model.Base{ID: "acct-1"},
|
||||
Name: "远程 Emby",
|
||||
Provider: model.StrmProviderEmbyRemote,
|
||||
Config: string(rawConfig),
|
||||
Enabled: true,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// proxy_play=true:直接走反向代理分支,便于断言确实取到了流。
|
||||
if err := repos.EmbyMount.Create(t.Context(), &model.EmbyMount{
|
||||
Base: model.Base{ID: "mount-1"},
|
||||
AccountID: "acct-1",
|
||||
RemoteViewID: "view-1",
|
||||
RemoteViewName: "新番连载",
|
||||
CollectionType: "tvshows",
|
||||
ProxyPlay: true,
|
||||
Enabled: true,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
router := gin.New()
|
||||
router.GET("/api/stream/:id", streamHandler(svc))
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/stream/"+service.EncodeEmbyRemoteID("mount-1", "ep-1"), nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d body=%s, want 200", w.Code, w.Body.String())
|
||||
}
|
||||
if got := w.Body.String(); got != "bytes" {
|
||||
t.Fatalf("body = %q, want the proxied stream", got)
|
||||
}
|
||||
if streamCalls != 1 {
|
||||
t.Fatalf("upstream stream calls = %d, want 1", streamCalls)
|
||||
}
|
||||
}
|
||||
+32
-4
@@ -17,6 +17,13 @@ import (
|
||||
// defaultUserAgent 是默认浏览器 User-Agent(用于 HTTP 请求头)。
|
||||
const defaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
|
||||
|
||||
// maxDecompressedBody 解压后的响应体上限。
|
||||
//
|
||||
// 调用方只限制了压缩前的字节数(例如 reader 的 8MiB),而高压缩比响应可以
|
||||
// 放大几个数量级:一个几 MB 的 gzip 就能在解压时把内存吃光。这里统一兜底,
|
||||
// 超过上限时按解压失败处理(原样返回压缩字节,由调用方报错)。
|
||||
const maxDecompressedBody = 32 << 20
|
||||
|
||||
// DecompressBody 兜底解压响应体(gzip / deflate)。
|
||||
//
|
||||
// 正常情况下用不到:只要不显式设置 Accept-Encoding,net/http 会自己带上 gzip
|
||||
@@ -37,17 +44,20 @@ func DecompressBody(resp *http.Response, data []byte) []byte {
|
||||
return data
|
||||
}
|
||||
defer r.Close()
|
||||
if out, err := io.ReadAll(r); err == nil {
|
||||
// 不吞掉多个 member:默认 Multistream(true) 会让串接的 member 继续解,
|
||||
// 是压缩炸弹的常见放大手段。
|
||||
r.Multistream(false)
|
||||
if out, ok := readLimited(r); ok {
|
||||
return out
|
||||
}
|
||||
case "deflate":
|
||||
// deflate 有两种实际写法:zlib 包装与裸 DEFLATE,依次尝试。
|
||||
if out, err := io.ReadAll(flate.NewReader(bytes.NewReader(data))); err == nil {
|
||||
if out, ok := readLimited(flate.NewReader(bytes.NewReader(data))); ok {
|
||||
return out
|
||||
}
|
||||
if zr, err := zlib.NewReader(bytes.NewReader(data)); err == nil {
|
||||
defer zr.Close()
|
||||
if out, err := io.ReadAll(zr); err == nil {
|
||||
if out, ok := readLimited(zr); ok {
|
||||
return out
|
||||
}
|
||||
}
|
||||
@@ -55,6 +65,15 @@ func DecompressBody(resp *http.Response, data []byte) []byte {
|
||||
return data
|
||||
}
|
||||
|
||||
// readLimited 读取解压流,超过上限返回 ok=false。
|
||||
func readLimited(r io.Reader) ([]byte, bool) {
|
||||
out, err := io.ReadAll(io.LimitReader(r, maxDecompressedBody+1))
|
||||
if err != nil || len(out) > maxDecompressedBody {
|
||||
return nil, false
|
||||
}
|
||||
return out, true
|
||||
}
|
||||
|
||||
// StripAcceptEncoding 移除显式设置的 Accept-Encoding,交回 net/http 管理。
|
||||
//
|
||||
// 只有「调用方没设置」时 net/http 才会自动解压,因此任何来源(预设头、书源
|
||||
@@ -104,10 +123,19 @@ func NewSiteHTTPClient(timeoutSeconds int, useProxy bool) *http.Client {
|
||||
//
|
||||
// 交给 net/http 管理后:请求仍会带 `Accept-Encoding: gzip`(浏览器常见取值),
|
||||
// 且响应被自动解压;另外也避免服务端挑选我们无法解码的 br。
|
||||
// 注意:Accept 刻意用 `*/*` 而不是浏览器页面导航的
|
||||
// `text/html,application/xhtml+xml,...`。
|
||||
//
|
||||
// 很多书源接口是 DRF(Django REST framework)一类会做内容协商的后端:当 Accept
|
||||
// 首选 text/html 时,它按浏览器语义返回「可浏览的 HTML 页面」(HTTP 200),
|
||||
// 而不是 JSON。引擎再把这份 HTML 交给 JSONPath 解析,结果永远是 0 条——表现为
|
||||
// 「同一个源在手机 App 里能搜到,在 MeBox 里搜不到」(拷贝系列源就是这个坑)。
|
||||
// 手机端阅读 App 走 OkHttp,不显式设置 Accept,等价于 `*/*`;这里对齐该行为。
|
||||
// 需要页面型 Accept 的书源可以在自身 header 里显式声明覆盖。
|
||||
func HTTPHeaderPresets() map[string]string {
|
||||
return map[string]string{
|
||||
"User-Agent": defaultUserAgent,
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
|
||||
"Accept": "*/*",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||||
"Connection": "keep-alive",
|
||||
"Upgrade-Insecure-Requests": "1",
|
||||
|
||||
@@ -12,6 +12,18 @@ import (
|
||||
// 背景:预设头里曾显式写 `Accept-Encoding: gzip, deflate, br`,导致 net/http
|
||||
// 不再自动解压(它只在调用方没设置该头时才解压),压缩字节直接进入规则层。
|
||||
|
||||
// TestHeaderPresetsAcceptIsWildcard 预设 Accept 不得首选 text/html。
|
||||
//
|
||||
// 背景:DRF(Django REST framework)等会做内容协商的后端,在 Accept 首选
|
||||
// text/html 时返回「可浏览 HTML 页面」而不是 JSON,书源 JSONPath 因此永远
|
||||
// 解析出 0 条(拷贝轻小说源在 MeBox 搜不到、在阅读 App 能搜到的根因)。
|
||||
func TestHeaderPresetsAcceptIsWildcard(t *testing.T) {
|
||||
got := HTTPHeaderPresets()["Accept"]
|
||||
if got != "*/*" {
|
||||
t.Fatalf("Accept 预设应为 */*(对齐 legado/OkHttp),实际 %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHeaderPresetsDoNotSetAcceptEncoding 预设头不得设置 Accept-Encoding。
|
||||
// 一旦设置,net/http 的透明解压就失效,压缩响应会以原始字节交给上层。
|
||||
func TestHeaderPresetsDoNotSetAcceptEncoding(t *testing.T) {
|
||||
|
||||
@@ -26,13 +26,17 @@ type LibraryRoot struct {
|
||||
// Media 是单个可播放项。剧集链接到 SeriesID;电影 SeriesID == ""。
|
||||
type Media struct {
|
||||
Base
|
||||
LibraryID string `gorm:"index;size:36;index:idx_media_library_release,priority:1" json:"library_id"`
|
||||
LibraryRootID string `gorm:"index;size:36" json:"library_root_id,omitempty"`
|
||||
SeriesID string `gorm:"index;size:128" json:"series_id,omitempty"`
|
||||
Title string `gorm:"size:255;not null" json:"title"`
|
||||
OriginalName string `gorm:"size:255" json:"original_name,omitempty"`
|
||||
EpisodeTitle string `gorm:"size:255" json:"episode_title,omitempty"`
|
||||
Path string `gorm:"uniqueIndex;size:1024;not null" json:"path"`
|
||||
LibraryID string `gorm:"index;size:36;index:idx_media_library_release,priority:1" json:"library_id"`
|
||||
LibraryRootID string `gorm:"index;size:36" json:"library_root_id,omitempty"`
|
||||
SeriesID string `gorm:"index;size:128" json:"series_id,omitempty"`
|
||||
Title string `gorm:"size:255;not null" json:"title"`
|
||||
OriginalName string `gorm:"size:255" json:"original_name,omitempty"`
|
||||
EpisodeTitle string `gorm:"size:255" json:"episode_title,omitempty"`
|
||||
// Path 的唯一性由 database 包按方言显式建索引,这里刻意不用 uniqueIndex:
|
||||
// PostgreSQL 的 btree 单条索引项上限 2704 字节,而 varchar(1024) 在中文路径
|
||||
// 下可达 3072 字节,直接对整列建唯一索引会让长路径的扫描插入直接失败
|
||||
// (SQLite 没有这个限制)。见 database.ensureMediaPathIndex。
|
||||
Path string `gorm:"size:1024;not null" json:"path"`
|
||||
RelativePath string `gorm:"size:1024" json:"relative_path,omitempty"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
DurationSec int `json:"duration_sec"`
|
||||
|
||||
@@ -65,6 +65,7 @@ func AllModels() []interface{} {
|
||||
&ReaderSourceState{},
|
||||
&ReaderBook{},
|
||||
&ReaderChapter{},
|
||||
&ReaderContentCache{},
|
||||
&ReaderReplaceRule{},
|
||||
&ReaderProfile{},
|
||||
&ReaderBookGroups{},
|
||||
|
||||
@@ -8,12 +8,19 @@ import (
|
||||
)
|
||||
|
||||
// ReaderBookSource 书源:原始 JSON 全量存储 + 常用字段冗余列出便于筛选排序。
|
||||
//
|
||||
// 书源按用户独立:同一份书源不同用户可以各存一份(各自启停、排序、修改规则与变量),
|
||||
// 互不可见。唯一键是 (user_id, source_url);跨用户的「同一本书」靠
|
||||
// ReaderBook.Origin(书源 URL)而不是书源行 ID 关联。
|
||||
type ReaderBookSource struct {
|
||||
Base
|
||||
// UserID 归属用户。历史数据迁移时会回填;值为空表示「旧版全局书源」,
|
||||
// 只在没有任何用户引用时才会出现。
|
||||
UserID string `gorm:"type:varchar(36);uniqueIndex:uniq_reader_source_user_url;index" json:"user_id"`
|
||||
Name string `gorm:"type:varchar(255);index" json:"name"`
|
||||
GroupName string `gorm:"type:varchar(255);index" json:"group"`
|
||||
Type int `gorm:"default:0" json:"type"` // 0文本 1音频 2图片 3文件 4视频
|
||||
SourceURL string `gorm:"type:varchar(512);index" json:"source_url"`
|
||||
SourceURL string `gorm:"type:varchar(512);uniqueIndex:uniq_reader_source_user_url;index" json:"source_url"`
|
||||
RawJSON string `gorm:"type:text" json:"-"`
|
||||
Enabled bool `gorm:"default:true" json:"enabled"`
|
||||
EnabledExplore bool `gorm:"default:true" json:"enabled_explore"`
|
||||
@@ -98,6 +105,45 @@ type ReaderChapter struct {
|
||||
Tag string `gorm:"type:varchar(255)" json:"tag"`
|
||||
}
|
||||
|
||||
// ReaderContentCache 正文持久缓存的索引行(内容本体在磁盘上,见 reader_content_cache.go)。
|
||||
//
|
||||
// 与 legado BookHelp 的章节正文缓存对应:缓存的是「书源侧产物」(书源 replaceRegex
|
||||
// 之后、用户替换规则与代理改写之前),因此可以跨用户共享;用户维度的处理在读出后
|
||||
// 逐请求应用,规则改动即时生效。
|
||||
//
|
||||
// 章节身份不落库为外键,而是 BookKey(书源 + 书本地址)与 ChapterKey(绝对化章节
|
||||
// 地址或标题)的哈希:目录刷新(ReplaceChapters 物理重建、行 ID 会变)与书源更新
|
||||
// 之后仍然能按同一身份命中或迁移。
|
||||
type ReaderContentCache struct {
|
||||
Base
|
||||
// OriginHash 书源地址哈希(磁盘目录的第一层,清理时定位文件用)。
|
||||
OriginHash string `gorm:"type:varchar(64)" json:"origin_hash"`
|
||||
// BookKey 书源身份哈希(sha256(origin + "\0" + bookURL) 前 16 字节 hex)。
|
||||
BookKey string `gorm:"type:varchar(64);index:idx_reader_content_book" json:"book_key"`
|
||||
// ChapterKey 章节身份哈希(绝对化 URL 优先,退化为 title)。
|
||||
ChapterKey string `gorm:"type:varchar(64);index:idx_reader_content_chapter" json:"chapter_key"`
|
||||
// ChapterIdentity 章节身份原文(便于诊断与 remap 时的标题兜底匹配)。
|
||||
ChapterIdentity string `gorm:"type:varchar(512)" json:"chapter_identity"`
|
||||
// ChapterIndex 保存时的章节序号(remap 时更新)。
|
||||
ChapterIndex int `json:"chapter_index"`
|
||||
// ContentType text / audio / image。
|
||||
ContentType string `gorm:"type:varchar(16)" json:"content_type"`
|
||||
// SourceHash 书源内容指纹(RawJSON 哈希):书源更新后自然失效。
|
||||
SourceHash string `gorm:"type:varchar(64)" json:"source_hash"`
|
||||
// FormatVersion 缓存载荷格式版本:解析管线语义变化时递增,旧条目自然失效。
|
||||
FormatVersion int `json:"format_version"`
|
||||
// SizeBytes 载荷字节数(容量统计用)。
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
// AssetCount 音频轨/图片张数(清单类内容的完整性统计)。
|
||||
AssetCount int `json:"asset_count"`
|
||||
// ExpiresAt 过期时间(unix 秒);0 表示不过期。
|
||||
ExpiresAt int64 `json:"expires_at"`
|
||||
// LastAccessAt 最近命中时间(unix 秒),LRU 淘汰依据。
|
||||
LastAccessAt int64 `json:"last_access_at"`
|
||||
// Hits 命中次数(诊断用)。
|
||||
Hits int `json:"hits"`
|
||||
}
|
||||
|
||||
// ReaderReplaceRule 替换净化规则(对应 legado ReplaceRule)。
|
||||
type ReaderReplaceRule struct {
|
||||
Base
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
package repository
|
||||
|
||||
import "gorm.io/gorm"
|
||||
|
||||
// LikeOperator 返回当前方言下「与 SQLite LIKE 行为一致」的模式匹配运算符。
|
||||
//
|
||||
// SQLite 的 LIKE 对 ASCII 不区分大小写(默认编译选项),PostgreSQL 的 LIKE 区分
|
||||
// 大小写、只有 ILIKE 才不区分。同一个搜索词在两种方言下必须给出同样的结果集,
|
||||
// 否则换库之后「inception」就再也搜不到「Inception」。
|
||||
//
|
||||
// 供构造原始 SQL 片段的调用方(含 service 层)使用。
|
||||
func LikeOperator(db *gorm.DB) string {
|
||||
return likeOperator(db)
|
||||
}
|
||||
|
||||
func likeOperator(db *gorm.DB) string {
|
||||
if db != nil && db.Dialector != nil && db.Dialector.Name() == "postgres" {
|
||||
return "ILIKE"
|
||||
}
|
||||
return "LIKE"
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gorm.io/driver/postgres"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// TestSegmentPrewarmQueryBindsBooleanFound 守住 PostgreSQL 下片段预取查询的合法性。
|
||||
//
|
||||
// media_segment_fetches.found 是布尔列,而 PostgreSQL 没有 boolean = integer
|
||||
// 运算符:写成 f.found = 0 / = 1 会让计划任务里的片段预取固定报
|
||||
//
|
||||
// operator does not exist: boolean = integer (SQLSTATE 42883)
|
||||
//
|
||||
// SQLite 一直能跑,所以这个错只在 PostgreSQL 上暴露(表现为日志里周期性出现
|
||||
// 该错误、片头片尾数据永远补不上)。这里断言语句里只能出现占位符绑定。
|
||||
func TestSegmentPrewarmQueryBindsBooleanFound(t *testing.T) {
|
||||
query := segmentPrewarmCandidatesQuery
|
||||
lower := strings.ToLower(query)
|
||||
for _, forbidden := range []string{"found = 0", "found = 1", "found=0", "found=1", "found = true", "found = false"} {
|
||||
if strings.Contains(lower, forbidden) {
|
||||
t.Fatalf("预取查询把布尔列与字面量比较(%q),PostgreSQL 会直接报错:\n%s", forbidden, query)
|
||||
}
|
||||
}
|
||||
if got := strings.Count(query, "f.found = ?"); got != 2 {
|
||||
t.Fatalf("expected 2 placeholder-bound found comparisons, got %d:\n%s", got, query)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLikeOperatorMatchesSQLiteCaseInsensitivity 守住搜索大小写行为的方言差异。
|
||||
//
|
||||
// SQLite 的 LIKE 对 ASCII 不区分大小写,PostgreSQL 的 LIKE 区分大小写:同一个
|
||||
// 搜索词在两种方言下必须给出同样的结果集,PostgreSQL 必须走 ILIKE。
|
||||
func TestLikeOperatorMatchesSQLiteCaseInsensitivity(t *testing.T) {
|
||||
pg, err := gorm.Open(postgres.New(postgres.Config{
|
||||
DSN: "postgres://mebox:mebox@127.0.0.1:5432/mebox?sslmode=disable",
|
||||
}), &gorm.Config{
|
||||
DryRun: true,
|
||||
DisableAutomaticPing: true,
|
||||
SkipDefaultTransaction: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("open dry-run postgres: %v", err)
|
||||
}
|
||||
if got := LikeOperator(pg); got != "ILIKE" {
|
||||
t.Fatalf("postgres LikeOperator = %q, want ILIKE", got)
|
||||
}
|
||||
if got := LikeOperator(nil); got != "LIKE" {
|
||||
t.Fatalf("nil LikeOperator = %q, want LIKE", got)
|
||||
}
|
||||
}
|
||||
@@ -33,6 +33,20 @@ func (r *HistoryRepository) UpsertProgress(ctx context.Context, h *model.Playbac
|
||||
return r.upsert(ctx, h, true)
|
||||
}
|
||||
|
||||
// historyColumn 生成「当前表」限定的列引用。
|
||||
//
|
||||
// PostgreSQL 的 ON CONFLICT DO UPDATE ... WHERE 里,未限定的列名会同时可解析到
|
||||
// 目标表和本次插入的行,于是报
|
||||
//
|
||||
// column reference "session_started_at_ms" is ambiguous (SQLSTATE 42702)
|
||||
//
|
||||
// 结果是 POST /api/history 一律 500,观看进度与续播完全存不下来。SQLite 不做这项
|
||||
// 检查,所以这个错只在 PostgreSQL 上暴露。clause.CurrentTable 由 GORM 在构建语句
|
||||
// 时替换成真实表名,两种方言都能正确渲染。
|
||||
func historyColumn(name string) clause.Column {
|
||||
return clause.Column{Table: clause.CurrentTable, Name: name}
|
||||
}
|
||||
|
||||
func (r *HistoryRepository) upsert(ctx context.Context, h *model.PlaybackHistory, versioned bool) error {
|
||||
updates := playbackHistoryAssignments(h)
|
||||
if versioned {
|
||||
@@ -49,16 +63,16 @@ func (r *HistoryRepository) upsert(ctx context.Context, h *model.PlaybackHistory
|
||||
onConflict.Where = clause.Where{Exprs: []clause.Expression{
|
||||
clause.Or(
|
||||
clause.Lt{
|
||||
Column: clause.Column{Name: "session_started_at_ms"},
|
||||
Column: historyColumn("session_started_at_ms"),
|
||||
Value: h.SessionStartedAtMs,
|
||||
},
|
||||
clause.And(
|
||||
clause.Eq{
|
||||
Column: clause.Column{Name: "session_started_at_ms"},
|
||||
Column: historyColumn("session_started_at_ms"),
|
||||
Value: h.SessionStartedAtMs,
|
||||
},
|
||||
clause.Lte{
|
||||
Column: clause.Column{Name: "sequence"},
|
||||
Column: historyColumn("sequence"),
|
||||
Value: h.Sequence,
|
||||
},
|
||||
),
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
)
|
||||
|
||||
// TestHistoryProgressUpsertQualifiesConflictWhereColumns 守住 PostgreSQL 下进度上报
|
||||
// 的 upsert 语句合法性。
|
||||
//
|
||||
// ON CONFLICT DO UPDATE 的 WHERE 里如果不给列名加表限定,PostgreSQL 会因为这些列
|
||||
// 同时出现在本次插入的行里而拒绝该语句:
|
||||
//
|
||||
// column reference "session_started_at_ms" is ambiguous (SQLSTATE 42702)
|
||||
//
|
||||
// 后果是 POST /api/history 全部 500,观看进度与续播完全存不下来。SQLite 不做这项
|
||||
// 检查(旧写法在 SQLite 下一直正常),所以只能在 PostgreSQL 方言下断言。
|
||||
func TestHistoryProgressUpsertQualifiesConflictWhereColumns(t *testing.T) {
|
||||
repos, rec := newPostgresDryRunRepos(t)
|
||||
ctx := context.Background()
|
||||
|
||||
if err := repos.History.UpsertProgress(ctx, &model.PlaybackHistory{
|
||||
UserID: "u-1",
|
||||
MediaID: "m-1",
|
||||
PositionMs: 1_000,
|
||||
DurationMs: 2_000,
|
||||
SessionID: "session-1",
|
||||
SessionStartedAtMs: 111,
|
||||
Sequence: 2,
|
||||
}); err != nil {
|
||||
t.Fatalf("build upsert: %v", err)
|
||||
}
|
||||
|
||||
sql := rec.all()
|
||||
if strings.TrimSpace(sql) == "" {
|
||||
t.Fatal("no SQL captured from UpsertProgress")
|
||||
}
|
||||
if !strings.Contains(sql, "ON CONFLICT") {
|
||||
t.Fatalf("expected an ON CONFLICT upsert, got:\n%s", sql)
|
||||
}
|
||||
|
||||
// 只看 ON CONFLICT 之后的部分:SET 的赋值目标和 CASE 里的 duration_ms 引用
|
||||
// 属于另一套语法规则,不在这里断言。
|
||||
conflict := sql[strings.Index(sql, "ON CONFLICT"):]
|
||||
whereIdx := strings.Index(conflict, " WHERE ")
|
||||
if whereIdx < 0 {
|
||||
t.Fatalf("ON CONFLICT clause has no DO UPDATE ... WHERE guard:\n%s", sql)
|
||||
}
|
||||
where := conflict[whereIdx:]
|
||||
|
||||
for _, want := range []string{
|
||||
`"playback_histories"."session_started_at_ms"`,
|
||||
`"playback_histories"."sequence"`,
|
||||
} {
|
||||
if !strings.Contains(where, want) {
|
||||
t.Fatalf("DO UPDATE WHERE is missing the table-qualified %s; it is ambiguous on PostgreSQL:\n%s", want, sql)
|
||||
}
|
||||
}
|
||||
if strings.Contains(where, `("session_started_at_ms"`) || strings.Contains(where, ` AND "sequence"`) {
|
||||
t.Fatalf("DO UPDATE WHERE still references unqualified columns:\n%s", sql)
|
||||
}
|
||||
}
|
||||
@@ -115,7 +115,12 @@ func (r *MediaRepository) upsertWithDB(ctx context.Context, db *gorm.DB, m *mode
|
||||
return m.ID, nil
|
||||
}
|
||||
|
||||
updates := mediaUpsertUpdates(existing, *m)
|
||||
// 孤儿行回收:原库已被删除时(DeleteLibrary 硬删库行;扫描与删除并发时扫描也会
|
||||
// 把行写回已删库),这条行会钉死在不存在的 library_id 上。media.path 是全局唯一
|
||||
// 索引,重扫既插不进同路径新行,又因 LibraryID 非空而拒绝迁移归属,于是新库/重建库
|
||||
// 永远查不到这些条目(表现为"扫描明明有文件,库却是空的")。这里显式识别并允许迁移。
|
||||
reclaimLibrary := r.mediaRowLibraryIsOrphaned(ctx, db, existing.LibraryID, m.LibraryID, m.Path)
|
||||
updates := mediaUpsertUpdates(existing, *m, reclaimLibrary)
|
||||
if adopted {
|
||||
updates["path"] = m.Path
|
||||
if existing.DeletedAt.Valid {
|
||||
@@ -139,6 +144,42 @@ func (r *MediaRepository) upsertWithDB(ctx context.Context, db *gorm.DB, m *mode
|
||||
return existing.ID, nil
|
||||
}
|
||||
|
||||
// mediaRowLibraryIsOrphaned reports whether a media row is still pinned to a
|
||||
// library that no longer exists.
|
||||
//
|
||||
// 删除媒体库会硬删库行(见 MediaService.DeleteLibrary),但扫描与删除并发时,
|
||||
// 已在途的扫描仍可能把这批文件写回已删库的 library_id;历史数据里也可能留下
|
||||
// 指向早已消失的库的行。因为 media.path 是全局唯一索引,这些行一旦存在:
|
||||
// 1. 重扫无法插入同路径的新行(唯一约束冲突);
|
||||
// 2. addMediaPlacementUpdates 又因为 existing.LibraryID 非空而拒绝迁移归属。
|
||||
//
|
||||
// 结果就是新库 / 重建的同名库永远查不到这些条目,扫描报告有文件而库显示为空。
|
||||
// 这里把"原库已不存在"这种情况单独判定出来,交给扫描把归属迁移到当前库。
|
||||
func (r *MediaRepository) mediaRowLibraryIsOrphaned(ctx context.Context, db *gorm.DB, existingLibraryID, incomingLibraryID, incomingPath string) bool {
|
||||
existingLibraryID = strings.TrimSpace(existingLibraryID)
|
||||
incomingLibraryID = strings.TrimSpace(incomingLibraryID)
|
||||
// 归属未变或原行为空(后者走既有的回填分支)时无需查库。
|
||||
if existingLibraryID == "" || existingLibraryID == incomingLibraryID {
|
||||
return false
|
||||
}
|
||||
// 云盘媒体的归属迁移由 addMediaPlacementUpdates 无条件处理,无需再查库。
|
||||
if strings.HasPrefix(strings.ToLower(strings.TrimSpace(incomingPath)), "cloud://") {
|
||||
return false
|
||||
}
|
||||
if db == nil || !db.Migrator().HasTable(&model.Library{}) {
|
||||
return false
|
||||
}
|
||||
var count int64
|
||||
// 刻意不加 Unscoped:软删除的库同样不再出现在库里,挂在它下面的媒体行
|
||||
// 也属于"查不到"的孤儿,一并允许回收。查询失败时保守处理:宁可保留旧
|
||||
// 归属,也不要把仍有效的库误判成孤儿。
|
||||
if err := db.WithContext(ctx).Model(&model.Library{}).
|
||||
Where("id = ?", existingLibraryID).Count(&count).Error; err != nil {
|
||||
return false
|
||||
}
|
||||
return count == 0
|
||||
}
|
||||
|
||||
func (r *MediaRepository) findOrCreateMediaByPath(ctx context.Context, db *gorm.DB, m *model.Media, aliasPaths []string) (model.Media, bool, bool, error) {
|
||||
var existing model.Media
|
||||
err := db.WithContext(ctx).Unscoped().Where("path = ?", m.Path).First(&existing).Error
|
||||
@@ -194,14 +235,14 @@ func (r *MediaRepository) findMediaByAlias(ctx context.Context, db *gorm.DB, cur
|
||||
return existing, err
|
||||
}
|
||||
|
||||
func mediaUpsertUpdates(existing, incoming model.Media) map[string]any {
|
||||
func mediaUpsertUpdates(existing, incoming model.Media, reclaimLibrary bool) map[string]any {
|
||||
updates := map[string]any{}
|
||||
addMediaFileScanUpdates(updates, existing, incoming)
|
||||
addMediaTitleUpdates(updates, existing, incoming)
|
||||
addMediaExternalIDUpdates(updates, existing, incoming)
|
||||
addMatchedMediaMetadataUpdates(updates, existing, incoming)
|
||||
addMediaArtworkUpdates(updates, existing, incoming)
|
||||
addMediaPlacementUpdates(updates, existing, incoming)
|
||||
addMediaPlacementUpdates(updates, existing, incoming, reclaimLibrary)
|
||||
addMediaSTRMUpdate(updates, existing, incoming)
|
||||
return updates
|
||||
}
|
||||
@@ -308,13 +349,15 @@ func addMediaArtworkUpdates(updates map[string]any, existing, incoming model.Med
|
||||
}
|
||||
}
|
||||
|
||||
func addMediaPlacementUpdates(updates map[string]any, existing, incoming model.Media) {
|
||||
func addMediaPlacementUpdates(updates map[string]any, existing, incoming model.Media, reclaimLibrary bool) {
|
||||
// 云盘媒体:同一 cloud:// 文件可能先被父目录库扫描入库,之后用户按二级
|
||||
// 分类重新挂载/扫描到更精确的分类库。此时让 library_id 迁移到当前扫描库,
|
||||
// 否则媒体被钉死在旧库、新分类库里看不到(表现为"媒体部分消失")。
|
||||
// 本地媒体物理位置固定:仅在原 library_id 为空时回填,不迁移。
|
||||
// 本地媒体物理位置固定:仅在原 library_id 为空、或原库已不存在(孤儿行,
|
||||
// reclaimLibrary)时才回填/迁移,既避免把仍属于其他库的本地媒体抢走,
|
||||
// 又能让扫描回收被删除库遗留的行。
|
||||
if isCloudMediaPath := strings.HasPrefix(strings.ToLower(strings.TrimSpace(incoming.Path)), "cloud://"); incoming.LibraryID != "" && incoming.LibraryID != existing.LibraryID {
|
||||
if isCloudMediaPath || existing.LibraryID == "" {
|
||||
if isCloudMediaPath || existing.LibraryID == "" || reclaimLibrary {
|
||||
updates["library_id"] = incoming.LibraryID
|
||||
}
|
||||
}
|
||||
|
||||
@@ -106,11 +106,8 @@ func (r *MediaRepository) searchFilteredLIKE(ctx context.Context, query string,
|
||||
q := r.db.WithContext(ctx).Model(&model.Media{})
|
||||
q = applyMediaQueryFilter(q, filter)
|
||||
// SQLite 的 LIKE 对 ASCII 不区分大小写;Postgres 的 LIKE 区分大小写,
|
||||
// 需用 ILIKE 保持两端搜索行为一致。
|
||||
likeOp := "LIKE"
|
||||
if r.db.Dialector != nil && r.db.Dialector.Name() == "postgres" {
|
||||
likeOp = "ILIKE"
|
||||
}
|
||||
// 需用 ILIKE 保持两端搜索行为一致(见 dialect.go 的 likeOperator)。
|
||||
likeOp := likeOperator(r.db)
|
||||
terms := mediaSearchTerms(query)
|
||||
for _, term := range terms {
|
||||
like := "%" + escapeLike(term) + "%"
|
||||
|
||||
@@ -86,21 +86,15 @@ func (r *MediaSegmentRepository) UpsertFetch(ctx context.Context, row *model.Med
|
||||
return r.db.WithContext(ctx).Clauses(onConflict).Create(row).Error
|
||||
}
|
||||
|
||||
// ListPrewarmCandidates returns media that should be refreshed from source:
|
||||
// no ledger, expired miss (fetched_at < missBefore), or expired hit (fetched_at < hitBefore).
|
||||
// Recently played items come first so hot titles recover coverage sooner.
|
||||
func (r *MediaSegmentRepository) ListPrewarmCandidates(
|
||||
ctx context.Context,
|
||||
source string,
|
||||
missBefore, hitBefore time.Time,
|
||||
limit int,
|
||||
) ([]model.Media, error) {
|
||||
if r == nil || limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
rows := make([]model.Media, 0, limit)
|
||||
// 可查询:有 TMDb,且是剧集(有季集)或电影(无季集)。
|
||||
err := r.db.WithContext(ctx).Raw(`
|
||||
// segmentPrewarmCandidatesQuery 是片段预取候选查询。
|
||||
//
|
||||
// 抽成包级常量是为了让方言相关的回归测试能直接断言语句文本:DryRun 模式下
|
||||
// GORM 不会把 Raw 语句交给 logger,无法从执行路径上捕获生成后的 SQL。
|
||||
//
|
||||
// 注意 found 必须用占位符绑定布尔值,不能写 f.found = 0 / = 1:
|
||||
// PostgreSQL 没有 boolean = integer 运算符,那样写会固定报 SQLSTATE 42883
|
||||
// (SQLite 宽容,所以只在 PostgreSQL 上暴露)。
|
||||
const segmentPrewarmCandidatesQuery = `
|
||||
SELECT m.*
|
||||
FROM media m
|
||||
LEFT JOIN media_segment_fetches f
|
||||
@@ -119,11 +113,29 @@ WHERE m.deleted_at IS NULL
|
||||
)
|
||||
AND (
|
||||
f.id IS NULL
|
||||
OR (f.found = 0 AND f.fetched_at < ?)
|
||||
OR (f.found = 1 AND f.fetched_at < ?)
|
||||
OR (f.found = ? AND f.fetched_at < ?)
|
||||
OR (f.found = ? AND f.fetched_at < ?)
|
||||
)
|
||||
ORDER BY ph.last_played DESC
|
||||
LIMIT ?
|
||||
`, source, missBefore, hitBefore, limit).Scan(&rows).Error
|
||||
`
|
||||
|
||||
// ListPrewarmCandidates returns media that should be refreshed from source:
|
||||
// no ledger, expired miss (fetched_at < missBefore), or expired hit (fetched_at < hitBefore).
|
||||
// Recently played items come first so hot titles recover coverage sooner.
|
||||
func (r *MediaSegmentRepository) ListPrewarmCandidates(
|
||||
ctx context.Context,
|
||||
source string,
|
||||
missBefore, hitBefore time.Time,
|
||||
limit int,
|
||||
) ([]model.Media, error) {
|
||||
if r == nil || limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
rows := make([]model.Media, 0, limit)
|
||||
// 可查询:有 TMDb,且是剧集(有季集)或电影(无季集)。
|
||||
err := r.db.WithContext(ctx).
|
||||
Raw(segmentPrewarmCandidatesQuery, source, false, missBefore, true, hitBefore, limit).
|
||||
Scan(&rows).Error
|
||||
return rows, err
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"errors"
|
||||
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
)
|
||||
@@ -15,10 +16,22 @@ type ReaderRepository struct {
|
||||
db *gorm.DB
|
||||
}
|
||||
|
||||
// ListSources 书源列表(按 customOrder 排序)。
|
||||
func (r *ReaderRepository) ListSources(ctx context.Context) ([]model.ReaderBookSource, error) {
|
||||
// readerOrderBy 组装方言安全的 ORDER BY。
|
||||
//
|
||||
// 用 clause.Column 让 GORM 按当前方言给列名加引号(SQLite 反引号 / PostgreSQL 双引号),
|
||||
// 而不是把引号硬编码进排序字符串。reader_books.order、reader_chapters.index、
|
||||
// reader_replace_rules.order 都撞上了 SQL 关键字(order 在 PostgreSQL 里是保留字):
|
||||
// 之前写成 .Order("`order` ASC"),SQLite 下正常,PostgreSQL 下反引号被词法分析成
|
||||
// 运算符,直接报 `syntax error at or near "order"`,书架 / 目录 / 替换规则整页 500。
|
||||
func readerOrderBy(columns ...clause.OrderByColumn) clause.OrderBy {
|
||||
return clause.OrderBy{Columns: columns}
|
||||
}
|
||||
|
||||
// ListSources 用户的书源列表(书源按用户独立,按 customOrder 排序)。
|
||||
func (r *ReaderRepository) ListSources(ctx context.Context, userID string) ([]model.ReaderBookSource, error) {
|
||||
var out []model.ReaderBookSource
|
||||
err := r.db.WithContext(ctx).Order("custom_order ASC, updated_at DESC").Find(&out).Error
|
||||
err := r.db.WithContext(ctx).Where("user_id = ?", userID).
|
||||
Order("custom_order ASC, updated_at DESC").Find(&out).Error
|
||||
return out, err
|
||||
}
|
||||
|
||||
@@ -31,8 +44,31 @@ func (r *ReaderRepository) GetSource(ctx context.Context, id string) (*model.Rea
|
||||
return &out, nil
|
||||
}
|
||||
|
||||
// GetSourceByURL 按书源 URL 取书源(导入去重用)。
|
||||
func (r *ReaderRepository) GetSourceByURL(ctx context.Context, sourceURL string) (*model.ReaderBookSource, error) {
|
||||
// GetSourceForUser 按 ID 取书源并校验归属用户。
|
||||
func (r *ReaderRepository) GetSourceForUser(ctx context.Context, userID, id string) (*model.ReaderBookSource, error) {
|
||||
var out model.ReaderBookSource
|
||||
if err := r.db.WithContext(ctx).First(&out, "id = ? AND user_id = ?", id, userID).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &out, nil
|
||||
}
|
||||
|
||||
// GetSourceByURL 按用户 + 书源 URL 取书源(导入去重用)。
|
||||
func (r *ReaderRepository) GetSourceByURL(ctx context.Context, userID, sourceURL string) (*model.ReaderBookSource, error) {
|
||||
var out model.ReaderBookSource
|
||||
err := r.db.WithContext(ctx).First(&out, "user_id = ? AND source_url = ?", userID, sourceURL).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &out, nil
|
||||
}
|
||||
|
||||
// GetSourceAnyByURL 按书源 URL 取任意一份副本(不限定用户)。
|
||||
//
|
||||
// 媒体代理、目录刷新等路径只拿得到书籍的 origin(书源 URL)而没有用户上下文;
|
||||
// 请求目标由 URL 决定,任一副本都够用。多个用户各自改过规则/header 时,
|
||||
// 取到哪一份只影响「用谁的默认 header」,不影响书籍本身的定位。
|
||||
func (r *ReaderRepository) GetSourceAnyByURL(ctx context.Context, sourceURL string) (*model.ReaderBookSource, error) {
|
||||
var out model.ReaderBookSource
|
||||
err := r.db.WithContext(ctx).First(&out, "source_url = ?", sourceURL).Error
|
||||
if err != nil {
|
||||
@@ -41,6 +77,13 @@ func (r *ReaderRepository) GetSourceByURL(ctx context.Context, sourceURL string)
|
||||
return &out, nil
|
||||
}
|
||||
|
||||
// ListSourceRowsByURL 取某书源 URL 的全部副本(判断是否仍有其他用户引用)。
|
||||
func (r *ReaderRepository) ListSourceRowsByURL(ctx context.Context, sourceURL string) ([]model.ReaderBookSource, error) {
|
||||
var out []model.ReaderBookSource
|
||||
err := r.db.WithContext(ctx).Where("source_url = ?", sourceURL).Find(&out).Error
|
||||
return out, err
|
||||
}
|
||||
|
||||
// CreateSource 新增书源。
|
||||
func (r *ReaderRepository) CreateSource(ctx context.Context, src *model.ReaderBookSource) error {
|
||||
return r.db.WithContext(ctx).Create(src).Error
|
||||
@@ -52,20 +95,34 @@ func (r *ReaderRepository) UpdateSource(ctx context.Context, src *model.ReaderBo
|
||||
}
|
||||
|
||||
// DeleteSource 删除书源(连带清理其会话状态)。
|
||||
//
|
||||
// 用物理删除而不是软删:唯一键是 (user_id, source_url),而索引会覆盖软删行——
|
||||
// 软删后再导入同一书源会撞唯一约束(表现为「导入失败」),删掉再导入是本模块的
|
||||
// 正常操作。书源本身是可重新导入的数据,不需要软删保留。
|
||||
//
|
||||
// 会话状态表仍按 source_url 一源一条(跨用户共享),因此只有在该 URL 已无任何
|
||||
// 其他用户的书源副本时才连带清理,否则会把别人的登录态一起删掉。
|
||||
func (r *ReaderRepository) DeleteSource(ctx context.Context, id string) error {
|
||||
return r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
src := &model.ReaderBookSource{}
|
||||
if err := tx.First(src, "id = ?", id).Error; err == nil && src.SourceURL != "" {
|
||||
// 会话状态是「一源一条」,而 source_url 上有覆盖软删行的唯一索引:
|
||||
// 软删会让这一行继续占着 source_url,之后 SaveSourceState 的
|
||||
// First(默认排除软删行)查不到、Create 就会撞唯一约束,
|
||||
// 表现为「保存书源会话状态失败: UNIQUE constraint failed」,
|
||||
// cookie / 登录态从此再也存不进去。这里必须硬删。
|
||||
if err := tx.Unscoped().Delete(&model.ReaderSourceState{}, "source_url = ?", src.SourceURL).Error; err != nil {
|
||||
var others int64
|
||||
if err := tx.Model(&model.ReaderBookSource{}).
|
||||
Where("source_url = ? AND id <> ?", src.SourceURL, id).Count(&others).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if others == 0 {
|
||||
// 会话状态是「一源一条」,而 source_url 上有覆盖软删行的唯一索引:
|
||||
// 软删会让这一行继续占着 source_url,之后 SaveSourceState 的
|
||||
// First(默认排除软删行)查不到、Create 就会撞唯一约束,
|
||||
// 表现为「保存书源会话状态失败: UNIQUE constraint failed」,
|
||||
// cookie / 登录态从此再也存不进去。这里必须硬删。
|
||||
if err := tx.Unscoped().Delete(&model.ReaderSourceState{}, "source_url = ?", src.SourceURL).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return tx.Delete(&model.ReaderBookSource{}, "id = ?", id).Error
|
||||
return tx.Unscoped().Delete(&model.ReaderBookSource{}, "id = ?", id).Error
|
||||
})
|
||||
}
|
||||
|
||||
@@ -108,7 +165,11 @@ func (r *ReaderRepository) SaveSourceState(ctx context.Context, st *model.Reader
|
||||
// ListBooks 用户书架(按 order 排序)。
|
||||
func (r *ReaderRepository) ListBooks(ctx context.Context, userID string) ([]model.ReaderBook, error) {
|
||||
var out []model.ReaderBook
|
||||
err := r.db.WithContext(ctx).Where("user_id = ?", userID).Order("`order` ASC, updated_at DESC").Find(&out).Error
|
||||
err := r.db.WithContext(ctx).Where("user_id = ?", userID).
|
||||
Order(readerOrderBy(
|
||||
clause.OrderByColumn{Column: clause.Column{Name: "order"}},
|
||||
clause.OrderByColumn{Column: clause.Column{Name: "updated_at"}, Desc: true},
|
||||
)).Find(&out).Error
|
||||
return out, err
|
||||
}
|
||||
|
||||
@@ -131,6 +192,15 @@ func (r *ReaderRepository) FindBookByURL(ctx context.Context, userID, origin, bo
|
||||
return &out, nil
|
||||
}
|
||||
|
||||
// ListBooksByOriginAndURL 按书源 + 书本地址查所有用户的书架记录。
|
||||
// 目录链路的 book.putVariable 需要写回变量,而目录抓取是跨用户共享的
|
||||
// (同一本书可能被多个用户收藏),所以这里不带 userID 过滤。
|
||||
func (r *ReaderRepository) ListBooksByOriginAndURL(ctx context.Context, origin, bookURL string) ([]model.ReaderBook, error) {
|
||||
var out []model.ReaderBook
|
||||
err := r.db.WithContext(ctx).Where("origin = ? AND book_url = ?", origin, bookURL).Find(&out).Error
|
||||
return out, err
|
||||
}
|
||||
|
||||
// CreateBook / UpdateBook / DeleteBook。
|
||||
func (r *ReaderRepository) CreateBook(ctx context.Context, b *model.ReaderBook) error {
|
||||
return r.db.WithContext(ctx).Create(b).Error
|
||||
@@ -163,10 +233,127 @@ func (r *ReaderRepository) ReplaceChapters(ctx context.Context, bookID string, c
|
||||
})
|
||||
}
|
||||
|
||||
// SwitchBookOrigin 换源:清空旧源章节与更新书籍信息在同一事务内完成。
|
||||
// 分开提交时若第二步失败,会留下「仍指向旧源、但目录已清空」的中间状态。
|
||||
func (r *ReaderRepository) SwitchBookOrigin(ctx context.Context, book *model.ReaderBook) error {
|
||||
return r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
if err := tx.Unscoped().Delete(&model.ReaderChapter{}, "book_id = ?", book.ID).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Save(book).Error
|
||||
})
|
||||
}
|
||||
|
||||
// ListChapters 按序取章节。
|
||||
func (r *ReaderRepository) ListChapters(ctx context.Context, bookID string) ([]model.ReaderChapter, error) {
|
||||
var out []model.ReaderChapter
|
||||
err := r.db.WithContext(ctx).Where("book_id = ?", bookID).Order("`index` ASC").Find(&out).Error
|
||||
err := r.db.WithContext(ctx).Where("book_id = ?", bookID).
|
||||
Order(readerOrderBy(
|
||||
clause.OrderByColumn{Column: clause.Column{Name: "index"}},
|
||||
)).Find(&out).Error
|
||||
return out, err
|
||||
}
|
||||
|
||||
// ─── 正文持久缓存索引 ──────────────────────────────────────────────────────
|
||||
|
||||
// GetContentCache 按 (bookKey, chapterKey, contentType) 取缓存索引行。
|
||||
func (r *ReaderRepository) GetContentCache(ctx context.Context, bookKey, chapterKey, contentType string) (*model.ReaderContentCache, error) {
|
||||
var out model.ReaderContentCache
|
||||
err := r.db.WithContext(ctx).
|
||||
Where("book_key = ? AND chapter_key = ? AND content_type = ?", bookKey, chapterKey, contentType).
|
||||
First(&out).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &out, nil
|
||||
}
|
||||
|
||||
// GetContentCacheByChapter 按 (bookKey, chapterKey) 取缓存索引行。
|
||||
// contentType 为空表示不限类型(正文链路不需要预知类型);给出类型时精确匹配。
|
||||
func (r *ReaderRepository) GetContentCacheByChapter(ctx context.Context, bookKey, chapterKey, contentType string) (*model.ReaderContentCache, error) {
|
||||
q := r.db.WithContext(ctx).Where("book_key = ? AND chapter_key = ?", bookKey, chapterKey)
|
||||
if contentType != "" {
|
||||
q = q.Where("content_type = ?", contentType)
|
||||
}
|
||||
var out model.ReaderContentCache
|
||||
if err := q.Order("last_access_at DESC").First(&out).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &out, nil
|
||||
}
|
||||
|
||||
// UpsertContentCache 写入或更新缓存索引行(按 bookKey + chapterKey + contentType 去重)。
|
||||
func (r *ReaderRepository) UpsertContentCache(ctx context.Context, row *model.ReaderContentCache) error {
|
||||
existing, err := r.GetContentCache(ctx, row.BookKey, row.ChapterKey, row.ContentType)
|
||||
if err == nil && existing != nil {
|
||||
existing.ChapterIdentity = row.ChapterIdentity
|
||||
existing.ChapterIndex = row.ChapterIndex
|
||||
existing.SourceHash = row.SourceHash
|
||||
existing.FormatVersion = row.FormatVersion
|
||||
existing.SizeBytes = row.SizeBytes
|
||||
existing.AssetCount = row.AssetCount
|
||||
existing.ExpiresAt = row.ExpiresAt
|
||||
existing.LastAccessAt = row.LastAccessAt
|
||||
return r.db.WithContext(ctx).Save(existing).Error
|
||||
}
|
||||
return r.db.WithContext(ctx).Create(row).Error
|
||||
}
|
||||
|
||||
// TouchContentCache 记录一次命中(更新命中时间与次数)。
|
||||
func (r *ReaderRepository) TouchContentCache(ctx context.Context, id string, hits int, lastAccess int64) error {
|
||||
return r.db.WithContext(ctx).Model(&model.ReaderContentCache{}).
|
||||
Where("id = ?", id).
|
||||
Updates(map[string]any{"last_access_at": lastAccess, "hits": hits}).Error
|
||||
}
|
||||
|
||||
// ListContentCacheByBook 列出某本书的全部缓存索引(目录刷新后的 remap 用)。
|
||||
func (r *ReaderRepository) ListContentCacheByBook(ctx context.Context, bookKey string) ([]model.ReaderContentCache, error) {
|
||||
var out []model.ReaderContentCache
|
||||
err := r.db.WithContext(ctx).Where("book_key = ?", bookKey).Find(&out).Error
|
||||
return out, err
|
||||
}
|
||||
|
||||
// DeleteContentCacheByBook 删除某本书的缓存索引,返回被删除的条目(调用方据此清理磁盘文件)。
|
||||
func (r *ReaderRepository) DeleteContentCacheByBook(ctx context.Context, bookKey string) ([]model.ReaderContentCache, error) {
|
||||
rows, err := r.ListContentCacheByBook(ctx, bookKey)
|
||||
if err != nil || len(rows) == 0 {
|
||||
return nil, err
|
||||
}
|
||||
if err := r.db.WithContext(ctx).Where("book_key = ?", bookKey).Delete(&model.ReaderContentCache{}).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
// DeleteContentCacheRow 删除单条缓存索引(remap 迁移旧键时用)。
|
||||
func (r *ReaderRepository) DeleteContentCacheRow(ctx context.Context, id string) error {
|
||||
return r.db.WithContext(ctx).Delete(&model.ReaderContentCache{}, "id = ?", id).Error
|
||||
}
|
||||
|
||||
// ListContentCacheExpired 按 TTL 取过期条目。
|
||||
func (r *ReaderRepository) ListContentCacheExpired(ctx context.Context, now int64, limit int) ([]model.ReaderContentCache, error) {
|
||||
var out []model.ReaderContentCache
|
||||
err := r.db.WithContext(ctx).
|
||||
Where("expires_at > 0 AND expires_at < ?", now).
|
||||
Order("last_access_at ASC").Limit(limit).Find(&out).Error
|
||||
return out, err
|
||||
}
|
||||
|
||||
// ContentCacheStats 返回条目数与总字节数(容量淘汰用)。
|
||||
func (r *ReaderRepository) ContentCacheStats(ctx context.Context) (int64, int64, error) {
|
||||
var row struct {
|
||||
Count int64
|
||||
Bytes int64
|
||||
}
|
||||
err := r.db.WithContext(ctx).Model(&model.ReaderContentCache{}).
|
||||
Select("COUNT(*) AS count, COALESCE(SUM(size_bytes), 0) AS bytes").Scan(&row).Error
|
||||
return row.Count, row.Bytes, err
|
||||
}
|
||||
|
||||
// ListContentCacheOldest 按最近命中时间取最旧的一批(LRU 淘汰用)。
|
||||
func (r *ReaderRepository) ListContentCacheOldest(ctx context.Context, limit int) ([]model.ReaderContentCache, error) {
|
||||
var out []model.ReaderContentCache
|
||||
err := r.db.WithContext(ctx).Order("last_access_at ASC").Limit(limit).Find(&out).Error
|
||||
return out, err
|
||||
}
|
||||
|
||||
@@ -197,9 +384,17 @@ func (r *ReaderRepository) CountChaptersByBook(ctx context.Context, bookIDs []st
|
||||
}
|
||||
|
||||
// GetChapter 取指定章节。
|
||||
//
|
||||
// index 在 PostgreSQL 里是保留字,必须让 GORM 按方言加引号(SQLite 反引号 /
|
||||
// PostgreSQL 双引号)。此前写成硬编码的 "book_id = ? AND `index` = ?",
|
||||
// PostgreSQL 会把反引号词法分析成运算符并报 syntax error at or near "`"。
|
||||
func (r *ReaderRepository) GetChapter(ctx context.Context, bookID string, index int) (*model.ReaderChapter, error) {
|
||||
var out model.ReaderChapter
|
||||
if err := r.db.WithContext(ctx).First(&out, "book_id = ? AND `index` = ?", bookID, index).Error; err != nil {
|
||||
err := r.db.WithContext(ctx).
|
||||
Where(clause.Eq{Column: clause.Column{Name: "book_id"}, Value: bookID}).
|
||||
Where(clause.Eq{Column: clause.Column{Name: "index"}, Value: index}).
|
||||
First(&out).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &out, nil
|
||||
@@ -265,7 +460,11 @@ func (r *ReaderRepository) SaveBookGroups(ctx context.Context, row *model.Reader
|
||||
// ListReplaceRules 用户替换规则(按 order 排序)。
|
||||
func (r *ReaderRepository) ListReplaceRules(ctx context.Context, userID string) ([]model.ReaderReplaceRule, error) {
|
||||
var out []model.ReaderReplaceRule
|
||||
err := r.db.WithContext(ctx).Where("user_id = ?", userID).Order("`order` DESC, created_at ASC").Find(&out).Error
|
||||
err := r.db.WithContext(ctx).Where("user_id = ?", userID).
|
||||
Order(readerOrderBy(
|
||||
clause.OrderByColumn{Column: clause.Column{Name: "order"}, Desc: true},
|
||||
clause.OrderByColumn{Column: clause.Column{Name: "created_at"}},
|
||||
)).Find(&out).Error
|
||||
return out, err
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gorm.io/driver/postgres"
|
||||
"gorm.io/gorm"
|
||||
gormlogger "gorm.io/gorm/logger"
|
||||
)
|
||||
|
||||
// sqlCaptureLogger 只记录 GORM 生成的 SQL,不打印任何日志。
|
||||
type sqlCaptureLogger struct {
|
||||
mu sync.Mutex
|
||||
lines []string
|
||||
}
|
||||
|
||||
func (l *sqlCaptureLogger) LogMode(gormlogger.LogLevel) gormlogger.Interface { return l }
|
||||
func (l *sqlCaptureLogger) Info(context.Context, string, ...interface{}) {}
|
||||
func (l *sqlCaptureLogger) Warn(context.Context, string, ...interface{}) {}
|
||||
func (l *sqlCaptureLogger) Error(context.Context, string, ...interface{}) {}
|
||||
|
||||
func (l *sqlCaptureLogger) Trace(_ context.Context, _ time.Time, fc func() (string, int64), _ error) {
|
||||
sql, _ := fc()
|
||||
l.mu.Lock()
|
||||
l.lines = append(l.lines, sql)
|
||||
l.mu.Unlock()
|
||||
}
|
||||
|
||||
func (l *sqlCaptureLogger) all() string {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
return strings.Join(l.lines, "\n")
|
||||
}
|
||||
|
||||
// newPostgresDryRunRepos 返回使用 PostgreSQL 方言的 repository 容器:
|
||||
// DryRun 只构建 SQL 不执行,DisableAutomaticPing 让它无需真实数据库也能离线运行,
|
||||
// SkipDefaultTransaction 则挡掉 GORM 默认事务包装——那一步会真的去开事务连接,
|
||||
// 在 DryRun 下同样会拨号(ToSQL 内部也是这么绕开的)。于是可以直接断言生成语句
|
||||
// 在 PostgreSQL 下是否合法。
|
||||
func newPostgresDryRunRepos(t *testing.T) (*Container, *sqlCaptureLogger) {
|
||||
t.Helper()
|
||||
rec := &sqlCaptureLogger{}
|
||||
db, err := gorm.Open(postgres.New(postgres.Config{
|
||||
DSN: "postgres://mebox:mebox@127.0.0.1:5432/mebox?sslmode=disable",
|
||||
}), &gorm.Config{
|
||||
DryRun: true,
|
||||
DisableAutomaticPing: true,
|
||||
SkipDefaultTransaction: true,
|
||||
Logger: rec,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("open dry-run postgres: %v", err)
|
||||
}
|
||||
return New(db), rec
|
||||
}
|
||||
|
||||
// TestReaderListsUseDialectQuotedReservedColumns 守住阅读子系统列表查询在
|
||||
// PostgreSQL 下的合法性。reader_books.order / reader_chapters.index /
|
||||
// reader_replace_rules.order 都是关键字级列名(order 在 PostgreSQL 里是保留字)。
|
||||
//
|
||||
// 排序条件曾经写成硬编码反引号的原始字符串,例如 .Order("`order` ASC"):
|
||||
// SQLite 正常,PostgreSQL 会把反引号词法分析成运算符,直接报
|
||||
// `syntax error at or near "order"` (SQLSTATE 42601),书架 / 目录 / 替换规则整页 500。
|
||||
// 现在改用 clause.Column,由 GORM 按方言加引号(SQLite 反引号 / PostgreSQL 双引号)。
|
||||
func TestReaderListsUseDialectQuotedReservedColumns(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cases := []struct {
|
||||
name string
|
||||
run func(repos *Container) error
|
||||
wants []string
|
||||
}{
|
||||
{
|
||||
name: "books",
|
||||
run: func(repos *Container) error { _, err := repos.Reader.ListBooks(ctx, "user-1"); return err },
|
||||
wants: []string{`"order"`, `"updated_at" DESC`},
|
||||
},
|
||||
{
|
||||
name: "chapters",
|
||||
run: func(repos *Container) error { _, err := repos.Reader.ListChapters(ctx, "book-1"); return err },
|
||||
wants: []string{`"index"`},
|
||||
},
|
||||
{
|
||||
name: "replace rules",
|
||||
run: func(repos *Container) error { _, err := repos.Reader.ListReplaceRules(ctx, "user-1"); return err },
|
||||
wants: []string{`"order" DESC`, `"created_at"`},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
repos, rec := newPostgresDryRunRepos(t)
|
||||
if err := tc.run(repos); err != nil {
|
||||
t.Fatalf("build query: %v", err)
|
||||
}
|
||||
sql := rec.all()
|
||||
if strings.TrimSpace(sql) == "" {
|
||||
t.Fatal("no SQL captured from the repository call")
|
||||
}
|
||||
if strings.Contains(sql, "`") {
|
||||
t.Fatalf("SQL uses backtick quoting, which PostgreSQL rejects:\n%s", sql)
|
||||
}
|
||||
if !strings.Contains(sql, "ORDER BY") {
|
||||
t.Fatalf("SQL has no ORDER BY clause:\n%s", sql)
|
||||
}
|
||||
for _, want := range tc.wants {
|
||||
if !strings.Contains(sql, want) {
|
||||
t.Fatalf("SQL missing %q:\n%s", want, sql)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+64
-18
@@ -1,9 +1,13 @@
|
||||
// Package service — database hot backup and restore.
|
||||
//
|
||||
// BackupService uses SQLite's "VACUUM INTO" (available since 3.27.0) to
|
||||
// produce a consistent copy of the database without blocking writers.
|
||||
// Backup files are stored under {data_dir}/backups/ with a timestamp
|
||||
// suffix. The admin UI can list / download / delete / restore them.
|
||||
// Backup files are stored under {data_dir}/backups/ with a timestamp suffix.
|
||||
// The admin UI can list / download / delete / restore them.
|
||||
//
|
||||
// 两种方言的备份形态不同:
|
||||
// - SQLite:用 "VACUUM INTO"(3.27.0 起可用)产出一个一致的库文件,
|
||||
// 扩展名 .db;恢复即「替换库文件 + 重启」。
|
||||
// - PostgreSQL:没有单文件语义,用逻辑备份(见 backup_postgres.go),
|
||||
// 扩展名 .json.gz;恢复是在单个事务内重建全部表。
|
||||
package service
|
||||
|
||||
import (
|
||||
@@ -65,18 +69,34 @@ func (b *BackupService) backupFilePath(filename string) (string, error) {
|
||||
return path, nil
|
||||
}
|
||||
|
||||
// Create produces a new hot backup via "VACUUM INTO".
|
||||
// isPostgres reports whether the service is backed by PostgreSQL.
|
||||
//
|
||||
// PostgreSQL 不支持 SQLite 的 "VACUUM INTO",按文件替换恢复也会写错目标,
|
||||
// 因此备份/恢复必须按方言分支,不能共用一条路径。
|
||||
func (b *BackupService) isPostgres() bool {
|
||||
return b.db != nil && b.db.Dialector != nil && b.db.Dialector.Name() == "postgres"
|
||||
}
|
||||
|
||||
// Create produces a new hot backup. SQLite uses "VACUUM INTO"; PostgreSQL uses
|
||||
// a logical dump.
|
||||
func (b *BackupService) Create(ctx context.Context) (*BackupInfo, error) {
|
||||
dir := b.backupDir()
|
||||
if err := os.MkdirAll(dir, 0o750); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ts := time.Now().Format("20060102_150405")
|
||||
dst := filepath.Join(dir, fmt.Sprintf("mebox_%s.db", ts))
|
||||
|
||||
// VACUUM INTO creates a clean, non-WAL copy atomically.
|
||||
if err := b.db.WithContext(ctx).Exec("VACUUM INTO ?", dst).Error; err != nil {
|
||||
return nil, fmt.Errorf("VACUUM INTO failed: %w", err)
|
||||
var (
|
||||
dst string
|
||||
err error
|
||||
)
|
||||
if b.isPostgres() {
|
||||
dst, err = b.createPostgresBackup(ctx, ts)
|
||||
} else {
|
||||
dst, err = b.createSQLiteBackup(ctx, ts)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stat, err := os.Stat(dst)
|
||||
if err != nil {
|
||||
@@ -91,6 +111,15 @@ func (b *BackupService) Create(ctx context.Context) (*BackupInfo, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (b *BackupService) createSQLiteBackup(ctx context.Context, stamp string) (string, error) {
|
||||
dst := filepath.Join(b.backupDir(), fmt.Sprintf("mebox_%s.db", stamp))
|
||||
// VACUUM INTO creates a clean, non-WAL copy atomically.
|
||||
if err := b.db.WithContext(ctx).Exec("VACUUM INTO ?", dst).Error; err != nil {
|
||||
return "", fmt.Errorf("VACUUM INTO failed: %w", err)
|
||||
}
|
||||
return dst, nil
|
||||
}
|
||||
|
||||
// List returns every backup file in the backup directory, newest first.
|
||||
func (b *BackupService) List() ([]BackupInfo, error) {
|
||||
dir := b.backupDir()
|
||||
@@ -103,7 +132,7 @@ func (b *BackupService) List() ([]BackupInfo, error) {
|
||||
}
|
||||
out := make([]BackupInfo, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
if e.IsDir() || !strings.HasSuffix(e.Name(), ".db") {
|
||||
if e.IsDir() || !isBackupFilename(e.Name()) {
|
||||
continue
|
||||
}
|
||||
info, err := e.Info()
|
||||
@@ -132,9 +161,10 @@ func (b *BackupService) Delete(filename string) error {
|
||||
return os.Remove(path)
|
||||
}
|
||||
|
||||
// Restore copies a backup over the live database using VACUUM INTO in
|
||||
// reverse. WARNING: this is destructive — the live DB will be replaced.
|
||||
// Callers should shut down the server after this call.
|
||||
// Restore replaces the live database with the given backup.
|
||||
// WARNING: this is destructive. SQLite restores by swapping the database file
|
||||
// (the caller should shut down the server afterwards); PostgreSQL restores by
|
||||
// rebuilding every table inside a single transaction.
|
||||
func (b *BackupService) Restore(ctx context.Context, filename string) error {
|
||||
src, err := b.backupFilePath(filename)
|
||||
if err != nil {
|
||||
@@ -143,6 +173,19 @@ func (b *BackupService) Restore(ctx context.Context, filename string) error {
|
||||
if _, err := os.Stat(src); err != nil {
|
||||
return fmt.Errorf("backup not found: %s", filename)
|
||||
}
|
||||
if b.isPostgres() {
|
||||
if !strings.HasSuffix(filename, postgresBackupSuffix) {
|
||||
return errors.New("该备份是 SQLite 文件备份,当前运行在 PostgreSQL 上,无法恢复")
|
||||
}
|
||||
return b.restorePostgresBackup(ctx, src)
|
||||
}
|
||||
if !strings.HasSuffix(filename, ".db") {
|
||||
return errors.New("该备份是 PostgreSQL 逻辑备份,当前运行在 SQLite 上,无法恢复")
|
||||
}
|
||||
return b.restoreSQLiteBackup(filename, src)
|
||||
}
|
||||
|
||||
func (b *BackupService) restoreSQLiteBackup(filename, src string) error {
|
||||
dbPath, err := filepath.Abs(b.cfg.Database.DBPath)
|
||||
if err != nil || dbPath == "" {
|
||||
return errors.New("invalid database path")
|
||||
@@ -171,7 +214,13 @@ func (b *BackupService) Restore(ctx context.Context, filename string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// isValidBackupFilename rejects path traversal attempts and non-.db files.
|
||||
// isBackupFilename reports whether name looks like a backup produced by either
|
||||
// dialect (SQLite ".db" or PostgreSQL ".json.gz").
|
||||
func isBackupFilename(name string) bool {
|
||||
return strings.HasSuffix(name, ".db") || strings.HasSuffix(name, postgresBackupSuffix)
|
||||
}
|
||||
|
||||
// isValidBackupFilename rejects path traversal attempts and non-backup files.
|
||||
func isValidBackupFilename(name string) bool {
|
||||
if name == "" {
|
||||
return false
|
||||
@@ -179,8 +228,5 @@ func isValidBackupFilename(name string) bool {
|
||||
if strings.ContainsAny(name, "/\\") || strings.Contains(name, "..") {
|
||||
return false
|
||||
}
|
||||
if !strings.HasSuffix(name, ".db") {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
return isBackupFilename(name)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,279 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/schema"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
)
|
||||
|
||||
// PostgreSQL 逻辑备份 / 恢复。
|
||||
//
|
||||
// SQLite 的备份是「VACUUM INTO 出一个一致的库文件,恢复时替换文件」,而
|
||||
// PostgreSQL 既没有 VACUUM INTO,数据也由服务端进程管理,没有可替换的单文件,
|
||||
// 所以这里实现逻辑备份:把每张表按主键顺序分块序列化成 JSON Lines(gzip 压缩)
|
||||
// 写入 {data_dir}/backups/。
|
||||
//
|
||||
// 分块序列化而不是整表 marshal:media 表在真实媒体库上可达几十万行、每行还带
|
||||
// overview / path 等长文本,整表进内存会把小内存的 NAS 打爆。
|
||||
//
|
||||
// 恢复在单个事务内「先清空全部模型表、再逐块插入」:任何一步失败(包括备份文件
|
||||
// 读到一半损坏)都整体回滚,不会留下半清空的库。因此恢复不需要重启服务。
|
||||
const (
|
||||
postgresBackupSuffix = ".json.gz"
|
||||
postgresBackupFormat = "mebox-postgres-logical"
|
||||
postgresBackupVersion = 1
|
||||
backupDumpChunkSize = 1000
|
||||
backupRestoreBatchSize = 500
|
||||
)
|
||||
|
||||
type postgresBackupHeader struct {
|
||||
Format string `json:"format"`
|
||||
Version int `json:"version"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
// postgresBackupRow 是「列名 → JSON 值」的一行。
|
||||
//
|
||||
// 刻意逐列反射序列化,而不是直接 json.Marshal 模型结构体:模型里有大量带
|
||||
// json:"-" 的列(Base.DeletedAt、User.AllowedLibraryIDs、APIConfig.APIKey、
|
||||
// StrmAccount.Config、PlayProfile.PINHash、ReaderBookSource.RawJSON…),走结构体
|
||||
// 自带的序列化会把这些列整列丢掉——软删除行会在恢复后「复活」,加密后的第三方
|
||||
// 密钥与 PIN 会全部丢失。
|
||||
type postgresBackupRow map[string]json.RawMessage
|
||||
|
||||
type postgresBackupChunk struct {
|
||||
Table string `json:"table"`
|
||||
Rows []postgresBackupRow `json:"rows"`
|
||||
}
|
||||
|
||||
// parseModelSchema 返回模型对应的表名与 GORM schema(逐列反射所需)。
|
||||
func parseModelSchema(db *gorm.DB, m any) (table string, sch *schema.Schema, err error) {
|
||||
stmt := &gorm.Statement{DB: db}
|
||||
if err := stmt.Parse(m); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
return stmt.Schema.Table, stmt.Schema, nil
|
||||
}
|
||||
|
||||
// postgresBackupModelSchemas 返回「表名 → 模型 schema / 结构体类型」,用于把备份
|
||||
// 里的行还原成 GORM 能插入的具体类型。
|
||||
func postgresBackupModelSchemas(db *gorm.DB) (map[string]*schema.Schema, map[string]reflect.Type, error) {
|
||||
schemas := make(map[string]*schema.Schema, len(model.AllModels()))
|
||||
types := make(map[string]reflect.Type, len(model.AllModels()))
|
||||
for _, m := range model.AllModels() {
|
||||
table, sch, err := parseModelSchema(db, m)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
schemas[table] = sch
|
||||
types[table] = reflect.TypeOf(m).Elem()
|
||||
}
|
||||
return schemas, types, nil
|
||||
}
|
||||
|
||||
// createPostgresBackup 把全库导出成一个 gzip 压缩的 JSON Lines 文件。
|
||||
func (b *BackupService) createPostgresBackup(ctx context.Context, stamp string) (string, error) {
|
||||
dst := filepath.Join(b.backupDir(), "mebox_"+stamp+postgresBackupSuffix)
|
||||
file, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
complete := false
|
||||
defer func() {
|
||||
_ = file.Close()
|
||||
if !complete {
|
||||
// 半截备份文件比没有备份更危险:留着会让人以为有可用的还原点。
|
||||
_ = os.Remove(dst)
|
||||
}
|
||||
}()
|
||||
|
||||
zw := gzip.NewWriter(file)
|
||||
enc := json.NewEncoder(zw)
|
||||
if err := enc.Encode(postgresBackupHeader{
|
||||
Format: postgresBackupFormat,
|
||||
Version: postgresBackupVersion,
|
||||
CreatedAt: time.Now(),
|
||||
}); err != nil {
|
||||
return "", fmt.Errorf("写入备份头失败: %w", err)
|
||||
}
|
||||
for _, m := range model.AllModels() {
|
||||
table, sch, err := parseModelSchema(b.db, m)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !b.db.Migrator().HasTable(table) {
|
||||
continue
|
||||
}
|
||||
if err := b.dumpPostgresTable(ctx, m, sch, table, enc); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
if err := zw.Close(); err != nil {
|
||||
return "", fmt.Errorf("压缩备份失败: %w", err)
|
||||
}
|
||||
if err := file.Sync(); err != nil {
|
||||
return "", fmt.Errorf("写入备份失败: %w", err)
|
||||
}
|
||||
complete = true
|
||||
return dst, nil
|
||||
}
|
||||
|
||||
// dumpPostgresTable 按主键分块导出一张表。FindInBatches 会自己按主键升序游标
|
||||
// 翻页(WHERE pk > last),不会像 OFFSET 分页那样在大表上做 O(n²) 扫描。
|
||||
func (b *BackupService) dumpPostgresTable(ctx context.Context, m any, sch *schema.Schema, table string, enc *json.Encoder) error {
|
||||
// 软删除行也要备份:恢复后应还原成删除前的样子,而不是让它们「复活」。
|
||||
slicePtr := reflect.New(reflect.SliceOf(reflect.TypeOf(m).Elem()))
|
||||
err := b.db.WithContext(ctx).Unscoped().
|
||||
FindInBatches(slicePtr.Interface(), backupDumpChunkSize, func(_ *gorm.DB, _ int) error {
|
||||
rows, err := encodePostgresBackupRows(ctx, sch, slicePtr.Elem())
|
||||
if err != nil {
|
||||
return fmt.Errorf("编码表 %s 失败: %w", table, err)
|
||||
}
|
||||
return enc.Encode(postgresBackupChunk{Table: table, Rows: rows})
|
||||
}).Error
|
||||
if err != nil {
|
||||
return fmt.Errorf("导出表 %s 失败: %w", table, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// backupPersistableFields 返回真正对应数据库列的字段。
|
||||
//
|
||||
// GORM 的 Schema.Fields 里还带着 gorm:"-" 的字段(例如 User.AllowedLibraryList、
|
||||
// LibraryTagList),它们的 DBName 是空字符串。不过滤的话多个「列」会共用空键:
|
||||
// 编码时互相覆盖,解码时把某个字段的值写到另一个字段上,报出
|
||||
// 「cannot unmarshal number into Go value of type []string」这类与真实数据无关的错。
|
||||
func backupPersistableFields(sch *schema.Schema) []*schema.Field {
|
||||
out := make([]*schema.Field, 0, len(sch.Fields))
|
||||
for _, field := range sch.Fields {
|
||||
if field.DBName == "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, field)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// encodePostgresBackupRows 逐列反射序列化一个批次。
|
||||
func encodePostgresBackupRows(ctx context.Context, sch *schema.Schema, slice reflect.Value) ([]postgresBackupRow, error) {
|
||||
fields := backupPersistableFields(sch)
|
||||
rows := make([]postgresBackupRow, 0, slice.Len())
|
||||
for i := 0; i < slice.Len(); i++ {
|
||||
element := slice.Index(i)
|
||||
row := make(postgresBackupRow, len(fields))
|
||||
for _, field := range fields {
|
||||
value := field.ReflectValueOf(ctx, element)
|
||||
if !value.IsValid() {
|
||||
continue
|
||||
}
|
||||
raw, err := json.Marshal(value.Interface())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("列 %s: %w", field.DBName, err)
|
||||
}
|
||||
row[field.DBName] = raw
|
||||
}
|
||||
rows = append(rows, row)
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
// decodePostgresBackupRows 把「列名 → JSON 值」还原成可直接插入的具体切片。
|
||||
func decodePostgresBackupRows(ctx context.Context, sch *schema.Schema, modelType reflect.Type, rows []postgresBackupRow) (any, error) {
|
||||
fields := backupPersistableFields(sch)
|
||||
slice := reflect.MakeSlice(reflect.SliceOf(modelType), len(rows), len(rows))
|
||||
for i, row := range rows {
|
||||
element := slice.Index(i)
|
||||
for _, field := range fields {
|
||||
raw, ok := row[field.DBName]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
holder := reflect.New(field.FieldType)
|
||||
if err := json.Unmarshal(raw, holder.Interface()); err != nil {
|
||||
return nil, fmt.Errorf("列 %s: %w", field.DBName, err)
|
||||
}
|
||||
if err := field.Set(ctx, element, holder.Elem().Interface()); err != nil {
|
||||
return nil, fmt.Errorf("列 %s: %w", field.DBName, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return slice.Interface(), nil
|
||||
}
|
||||
|
||||
// restorePostgresBackup 用逻辑备份重建全库。
|
||||
func (b *BackupService) restorePostgresBackup(ctx context.Context, src string) error {
|
||||
file, err := os.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
zr, err := gzip.NewReader(file)
|
||||
if err != nil {
|
||||
return fmt.Errorf("备份文件不是有效的 gzip: %w", err)
|
||||
}
|
||||
defer zr.Close()
|
||||
|
||||
dec := json.NewDecoder(zr)
|
||||
var header postgresBackupHeader
|
||||
if err := dec.Decode(&header); err != nil {
|
||||
return fmt.Errorf("备份头损坏: %w", err)
|
||||
}
|
||||
if header.Format != postgresBackupFormat || header.Version != postgresBackupVersion {
|
||||
return fmt.Errorf("不支持的备份格式 %q (v%d)", header.Format, header.Version)
|
||||
}
|
||||
schemas, modelTypes, err := postgresBackupModelSchemas(b.db)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 恢复必须在单个事务里完成:清空成功后中途失败若不回滚,库就空了。
|
||||
return b.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
// 逆序清空,保证 library_roots → libraries 这类外键先删子表。
|
||||
models := model.AllModels()
|
||||
for i := len(models) - 1; i >= 0; i-- {
|
||||
m := models[i]
|
||||
if !tx.Migrator().HasTable(m) {
|
||||
continue
|
||||
}
|
||||
if err := tx.Session(&gorm.Session{AllowGlobalUpdate: true}).Unscoped().Delete(m).Error; err != nil {
|
||||
return fmt.Errorf("清空表失败: %w", err)
|
||||
}
|
||||
}
|
||||
for {
|
||||
var chunk postgresBackupChunk
|
||||
decodeErr := dec.Decode(&chunk)
|
||||
if errors.Is(decodeErr, io.EOF) {
|
||||
return nil
|
||||
}
|
||||
if decodeErr != nil {
|
||||
return fmt.Errorf("备份文件损坏: %w", decodeErr)
|
||||
}
|
||||
sch, ok := schemas[chunk.Table]
|
||||
if !ok {
|
||||
return fmt.Errorf("备份包含未知表 %s,可能来自不兼容的版本", chunk.Table)
|
||||
}
|
||||
if len(chunk.Rows) == 0 {
|
||||
continue
|
||||
}
|
||||
rows, err := decodePostgresBackupRows(ctx, sch, modelTypes[chunk.Table], chunk.Rows)
|
||||
if err != nil {
|
||||
return fmt.Errorf("表 %s 解码失败: %w", chunk.Table, err)
|
||||
}
|
||||
if err := tx.Unscoped().CreateInBatches(rows, backupRestoreBatchSize).Error; err != nil {
|
||||
return fmt.Errorf("恢复表 %s 失败: %w", chunk.Table, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,226 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/config"
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
)
|
||||
|
||||
func newBackupTestService(t *testing.T) (*BackupService, *config.Config) {
|
||||
t.Helper()
|
||||
cfg := &config.Config{}
|
||||
cfg.App.DataDir = t.TempDir()
|
||||
db := newServiceTestDB(t)
|
||||
svc := &BackupService{cfg: cfg, log: zap.NewNop(), db: db}
|
||||
// Create 会先建目录;直接调各方言的 create* 助手时需要它已存在。
|
||||
if err := os.MkdirAll(svc.backupDir(), 0o750); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return svc, cfg
|
||||
}
|
||||
|
||||
// TestPostgresLogicalBackupRoundTrip 验证「备份 → 改库 → 恢复」后整库回到备份时的
|
||||
// 样子,包括软删除行与 json:"-" 的列。
|
||||
//
|
||||
// 逻辑备份/恢复本身是方言无关的 GORM 代码(PostgreSQL 专有的只有文件后缀与
|
||||
// isPostgres() 分派),所以可以直接在 SQLite 上跑。
|
||||
func TestPostgresLogicalBackupRoundTrip(t *testing.T) {
|
||||
svc, _ := newBackupTestService(t)
|
||||
ctx := context.Background()
|
||||
db := svc.db
|
||||
|
||||
lib := model.Library{Base: model.Base{ID: "lib-1"}, Name: "Movies", Path: "/media/movies", Type: "movie", Enabled: true}
|
||||
if err := db.Create(&lib).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Create(&model.LibraryRoot{Base: model.Base{ID: "root-1"}, LibraryID: lib.ID, Path: "/media/movies"}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Create(&model.Media{Base: model.Base{ID: "m-keep"}, LibraryID: lib.ID, Title: "Keep", Path: "/media/movies/keep.mkv"}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Create(&model.Media{Base: model.Base{ID: "m-gone"}, LibraryID: lib.ID, Title: "Gone", Path: "/media/movies/gone.mkv"}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Delete(&model.Media{}, "id = ?", "m-gone").Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Create(&model.Setting{Key: "k", Value: "v"}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// 下面这些列都带 json:"-":走结构体自带的 JSON 序列化会被整列丢掉,
|
||||
// 恢复之后用户权限、置顶库、加密后的密钥会一起消失。
|
||||
user := model.User{
|
||||
Base: model.Base{ID: "u-1"},
|
||||
Username: "alice",
|
||||
PasswordHash: "hash",
|
||||
Role: "user",
|
||||
AllowedLibraryIDs: `["lib-1"]`,
|
||||
PinnedLibraryIDs: `["lib-1"]`,
|
||||
HideAdult: true,
|
||||
IsActive: true,
|
||||
DanmakuEnabled: true,
|
||||
PlayerVr360GuideSeen: true,
|
||||
}
|
||||
if err := db.Create(&user).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Create(&model.APIConfig{Base: model.Base{ID: "cfg-1"}, Provider: "tmdb", APIKey: "ciphertext", Enabled: true}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
dump, err := svc.createPostgresBackup(ctx, "20260101_000000")
|
||||
if err != nil {
|
||||
t.Fatalf("createPostgresBackup: %v", err)
|
||||
}
|
||||
if dump == "" {
|
||||
t.Fatal("createPostgresBackup returned an empty path")
|
||||
}
|
||||
|
||||
// 破坏当前数据:加一条备份里没有的、硬删一条备份里有的。
|
||||
if err := db.Create(&model.Media{Base: model.Base{ID: "m-extra"}, LibraryID: lib.ID, Title: "Extra", Path: "/media/movies/extra.mkv"}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Unscoped().Delete(&model.Media{}, "id = ?", "m-keep").Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := svc.restorePostgresBackup(ctx, dump); err != nil {
|
||||
t.Fatalf("restorePostgresBackup: %v", err)
|
||||
}
|
||||
|
||||
var ids []string
|
||||
if err := db.Unscoped().Model(&model.Media{}).Order("id asc").Pluck("id", &ids).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if want := []string{"m-gone", "m-keep"}; !reflect.DeepEqual(ids, want) {
|
||||
t.Fatalf("media ids after restore = %v, want %v", ids, want)
|
||||
}
|
||||
|
||||
// 软删除状态必须原样还原,而不是被「复活」。
|
||||
var gone model.Media
|
||||
if err := db.Unscoped().First(&gone, "id = ?", "m-gone").Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !gone.DeletedAt.Valid {
|
||||
t.Fatal("soft-deleted media was revived by restore")
|
||||
}
|
||||
var live int64
|
||||
if err := db.Model(&model.Media{}).Where("id = ?", "m-keep").Count(&live).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if live != 1 {
|
||||
t.Fatalf("live media count = %d, want 1", live)
|
||||
}
|
||||
|
||||
var value string
|
||||
if err := db.Model(&model.Setting{}).Where("key = ?", "k").Select("value").Scan(&value).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if value != "v" {
|
||||
t.Fatalf("setting value = %q, want v", value)
|
||||
}
|
||||
var libCount int64
|
||||
if err := db.Model(&model.Library{}).Count(&libCount).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if libCount != 1 {
|
||||
t.Fatalf("library count = %d, want 1", libCount)
|
||||
}
|
||||
var rootCount int64
|
||||
if err := db.Model(&model.LibraryRoot{}).Count(&rootCount).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rootCount != 1 {
|
||||
t.Fatalf("library root count = %d, want 1", rootCount)
|
||||
}
|
||||
|
||||
// json:"-" 的列必须同样被备份:它们是权限与凭据,丢了比丢一条媒体严重得多。
|
||||
var restoredUser model.User
|
||||
if err := db.First(&restoredUser, "id = ?", "u-1").Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if restoredUser.PasswordHash != "hash" {
|
||||
t.Fatalf("password hash = %q, want hash", restoredUser.PasswordHash)
|
||||
}
|
||||
if restoredUser.AllowedLibraryIDs != `["lib-1"]` {
|
||||
t.Fatalf(`allowed library ids = %q, want ["lib-1"]`, restoredUser.AllowedLibraryIDs)
|
||||
}
|
||||
if restoredUser.PinnedLibraryIDs != `["lib-1"]` {
|
||||
t.Fatalf(`pinned library ids = %q, want ["lib-1"]`, restoredUser.PinnedLibraryIDs)
|
||||
}
|
||||
if !restoredUser.HideAdult || !restoredUser.IsActive || !restoredUser.DanmakuEnabled || !restoredUser.PlayerVr360GuideSeen {
|
||||
t.Fatalf("boolean columns lost: %+v", restoredUser)
|
||||
}
|
||||
|
||||
var restoredConfig model.APIConfig
|
||||
if err := db.First(&restoredConfig, "id = ?", "cfg-1").Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if restoredConfig.APIKey != "ciphertext" {
|
||||
t.Fatalf("api key = %q, want ciphertext", restoredConfig.APIKey)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRestorePostgresBackupKeepsDataOnCorruptFile 验证损坏的备份不会在回滚后留下
|
||||
// 半清空的库:恢复是先清空再插入,必须整体在一个事务里,否则中途失败就把库清掉了。
|
||||
func TestRestorePostgresBackupKeepsDataOnCorruptFile(t *testing.T) {
|
||||
svc, _ := newBackupTestService(t)
|
||||
ctx := context.Background()
|
||||
if err := svc.db.Create(&model.Setting{Key: "keep", Value: "me"}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
corrupt := filepath.Join(t.TempDir(), "broken.json.gz")
|
||||
file, err := os.Create(corrupt)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
zw := gzip.NewWriter(file)
|
||||
// 头部完整、正文半截:解码在清空表之后才会失败。
|
||||
if _, err := zw.Write([]byte(
|
||||
`{"format":"mebox-postgres-logical","version":1,"created_at":"2026-01-01T00:00:00Z"}` + "\n" +
|
||||
`{"table":"media","rows":[`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := zw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := file.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := svc.restorePostgresBackup(ctx, corrupt); err == nil {
|
||||
t.Fatal("corrupt backup should fail to restore")
|
||||
}
|
||||
|
||||
var value string
|
||||
if err := svc.db.Model(&model.Setting{}).Where("key = ?", "keep").Select("value").Scan(&value).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if value != "me" {
|
||||
t.Fatalf("a failed restore wiped the database: settings keep = %q", value)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRestoreRejectsBackupFromOtherDialect 验证备份文件与当前方言不匹配时给出明确
|
||||
// 错误,而不是把 SQLite 文件写到 PostgreSQL 部署的库路径上(那是原实现的隐患)。
|
||||
func TestRestoreRejectsBackupFromOtherDialect(t *testing.T) {
|
||||
svc, _ := newBackupTestService(t)
|
||||
pgStyle := filepath.Join(svc.backupDir(), "mebox_20260101_000000.json.gz")
|
||||
if err := os.WriteFile(pgStyle, []byte("not-gzip"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// SQLite 路径下不允许恢复 PostgreSQL 逻辑备份。
|
||||
if err := svc.Restore(context.Background(), filepath.Base(pgStyle)); err == nil {
|
||||
t.Fatal("SQLite restore accepted a PostgreSQL logical backup")
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,8 @@ package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -12,6 +14,8 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/service/cloud115"
|
||||
)
|
||||
|
||||
@@ -24,6 +28,10 @@ const (
|
||||
cloud115HLSSessionTTL = 30 * time.Minute
|
||||
cloud115HLSMaxSessions = 256
|
||||
cloud115HLSMaxManifest = 8 << 20
|
||||
// cloud115HLSMaxEntries 单个会话最多记住多少个上游地址。key 由上游地址
|
||||
// 派生(见 cloud115HLSKeyForURL),所以条目数只随「实际出现过的地址」增长,
|
||||
// 正常一部片子就是几千条;这个上限只是防御异常/恶意播放列表的兜底。
|
||||
cloud115HLSMaxEntries = 20000
|
||||
)
|
||||
|
||||
type cloud115HLSSession struct {
|
||||
@@ -35,7 +43,17 @@ type cloud115HLSSession struct {
|
||||
|
||||
mu sync.Mutex
|
||||
entries map[string]string
|
||||
next int
|
||||
}
|
||||
|
||||
// cloud115HLSKeyForURL 由上游地址派生稳定的分片 key。
|
||||
//
|
||||
// 早期实现用自增序号(e1、e2…)在每次重写播放列表时重新编号:同一分片每次
|
||||
// 重写都会拿到新 key,entries 随重写次数线性膨胀(实测一个 762 段的列表反复
|
||||
// 重写后 key 已经涨到 e2288…e3049),客户端拿到的分片编号也会无谓漂移。
|
||||
// 改成地址哈希后,同一分片在任何一次重写里都是同一个 key。
|
||||
func cloud115HLSKeyForURL(upstream string) string {
|
||||
sum := sha256.Sum256([]byte(upstream))
|
||||
return hex.EncodeToString(sum[:12])
|
||||
}
|
||||
|
||||
// Cloud115HLSProxy 把 115 云端 HLS 转成 MeBox 同源 HLS。
|
||||
@@ -302,12 +320,19 @@ func (p *Cloud115HLSProxy) rewriteManifest(session *cloud115HLSSession, text, ba
|
||||
}
|
||||
|
||||
func (p *Cloud115HLSProxy) proxyURL(session *cloud115HLSSession, upstream, rawQuery string) string {
|
||||
key := cloud115HLSKeyForURL(upstream)
|
||||
session.mu.Lock()
|
||||
session.next++
|
||||
key := "e" + strconv.Itoa(session.next)
|
||||
session.entries[key] = upstream
|
||||
_, known := session.entries[key]
|
||||
if !known && len(session.entries) < cloud115HLSMaxEntries {
|
||||
session.entries[key] = upstream
|
||||
known = true
|
||||
}
|
||||
mediaID := session.MediaID
|
||||
session.mu.Unlock()
|
||||
if !known && p != nil && p.service != nil && p.service.log != nil {
|
||||
p.service.log.Warn("cloud115 hls: session entry cap reached, segment will not resolve",
|
||||
zap.String("session", session.ID), zap.Int("cap", cloud115HLSMaxEntries))
|
||||
}
|
||||
|
||||
query := childProxyQuery(rawQuery, mediaID)
|
||||
return "/api/cloud115/hls/" + url.PathEscape(session.ID) + "/" + url.PathEscape(key) + "?" + query
|
||||
|
||||
@@ -25,7 +25,8 @@ func TestCloud115HLSProxyRewriteManifest(t *testing.T) {
|
||||
if strings.Contains(rewritten, "cpats01.115.com") {
|
||||
t.Fatalf("upstream URL leaked into rewritten manifest: %s", rewritten)
|
||||
}
|
||||
if !strings.Contains(rewritten, "/api/cloud115/hls/sess/e1?") {
|
||||
wantKey := cloud115HLSKeyForURL("https://cpats01.115.com/a.m3u8?u=1&se=2")
|
||||
if !strings.Contains(rewritten, "/api/cloud115/hls/sess/"+wantKey+"?") {
|
||||
t.Fatalf("proxy URL missing: %s", rewritten)
|
||||
}
|
||||
if !strings.Contains(rewritten, "media_id=media-1") || !strings.Contains(rewritten, "token=t") {
|
||||
@@ -36,6 +37,32 @@ func TestCloud115HLSProxyRewriteManifest(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloud115HLSProxyRewriteIsStableAcrossRewrites(t *testing.T) {
|
||||
proxy := &Cloud115HLSProxy{sessions: map[string]*cloud115HLSSession{}}
|
||||
session := &cloud115HLSSession{
|
||||
ID: "sess",
|
||||
MediaID: "media-1",
|
||||
entries: map[string]string{},
|
||||
}
|
||||
manifest := "#EXTM3U\n#EXT-X-TARGETDURATION:10\n" +
|
||||
"https://cpats01.115.com/seg0.ts?x=0\nhttps://cpats01.115.com/seg1.ts?x=1\n"
|
||||
|
||||
first := proxy.rewriteManifest(session, manifest, "https://cpats01.115.com/v.m3u8", "media_id=media-1")
|
||||
entriesAfterFirst := len(session.entries)
|
||||
second := proxy.rewriteManifest(session, manifest, "https://cpats01.115.com/v.m3u8", "media_id=media-1")
|
||||
|
||||
// 同一分片在每次重写里都必须是同一个 key,客户端拿到的地址才不会无谓漂移。
|
||||
if first != second {
|
||||
t.Fatalf("rewrite is not stable:\nfirst = %s\nsecond = %s", first, second)
|
||||
}
|
||||
if len(session.entries) != entriesAfterFirst {
|
||||
t.Fatalf("entries grew on rewrite: %d -> %d", entriesAfterFirst, len(session.entries))
|
||||
}
|
||||
if entriesAfterFirst != 2 {
|
||||
t.Fatalf("session entries = %d, want 2", entriesAfterFirst)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloud115HLSProxyRewriteKeyURI(t *testing.T) {
|
||||
proxy := &Cloud115HLSProxy{sessions: map[string]*cloud115HLSSession{}}
|
||||
session := &cloud115HLSSession{
|
||||
@@ -151,7 +178,6 @@ func TestCloud115HLSProxyServeChildRewritesVariant(t *testing.T) {
|
||||
MediaID: "media-1",
|
||||
ExpiresAt: time.Now().Add(time.Hour),
|
||||
entries: map[string]string{"e1": "https://cpats01.115.com/v.m3u8"},
|
||||
next: 1,
|
||||
}
|
||||
proxy.sessions[session.ID] = session
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/cloud115/hls/sess/e1?media_id=media-1&token=t", nil)
|
||||
@@ -162,7 +188,8 @@ func TestCloud115HLSProxyServeChildRewritesVariant(t *testing.T) {
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200", rec.Code)
|
||||
}
|
||||
if !strings.Contains(rec.Body.String(), "/api/cloud115/hls/sess/e2?") {
|
||||
wantKey := cloud115HLSKeyForURL("https://cpats01.115.com/seg.ts?x=1")
|
||||
if !strings.Contains(rec.Body.String(), "/api/cloud115/hls/sess/"+wantKey+"?") {
|
||||
t.Fatalf("rewritten variant missing proxy segment: %s", rec.Body.String())
|
||||
}
|
||||
if strings.Contains(rec.Body.String(), "cpats01.115.com") {
|
||||
@@ -192,7 +219,6 @@ func TestCloud115HLSProxyServeChildStreamsRange(t *testing.T) {
|
||||
MediaID: "media-1",
|
||||
ExpiresAt: time.Now().Add(time.Hour),
|
||||
entries: map[string]string{"e1": "https://cpats01.115.com/seg.ts?x=1"},
|
||||
next: 1,
|
||||
}
|
||||
proxy.sessions[session.ID] = session
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/cloud115/hls/sess/e1?media_id=media-1&token=t", nil)
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/config"
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
"github.com/truewhile/MeBox/internal/repository"
|
||||
)
|
||||
|
||||
// newDanmakuRemoteResolverTest 起一个假的远程 Emby(账号 + 挂载齐备),返回接好
|
||||
// 弹幕解析器的 builder。
|
||||
func newDanmakuRemoteResolverTest(t *testing.T, handler http.Handler) *serviceContainerBuilder {
|
||||
t.Helper()
|
||||
server := httptest.NewServer(handler)
|
||||
t.Cleanup(server.Close)
|
||||
|
||||
db := newServiceTestDB(t, &model.StrmAccount{}, &model.EmbyMount{})
|
||||
repos := repository.New(db)
|
||||
cfg := &config.Config{}
|
||||
remote := NewEmbyRemoteService(cfg, zap.NewNop(), repos, NewCryptoService("", zap.NewNop()))
|
||||
remote.SetRuntimeCache(NewRuntimeCacheService(cfg, zap.NewNop()))
|
||||
|
||||
rawConfig, _ := json.Marshal(map[string]string{
|
||||
"url": server.URL,
|
||||
"api_key": "test-api-key",
|
||||
"remote_user_id": "uid-1",
|
||||
})
|
||||
if err := repos.StrmAccount.Create(t.Context(), &model.StrmAccount{
|
||||
Base: model.Base{ID: "acct-1"},
|
||||
Name: "远程 Emby",
|
||||
Provider: model.StrmProviderEmbyRemote,
|
||||
Config: string(rawConfig),
|
||||
Enabled: true,
|
||||
}); err != nil {
|
||||
t.Fatalf("create account: %v", err)
|
||||
}
|
||||
if err := repos.EmbyMount.Create(t.Context(), &model.EmbyMount{
|
||||
Base: model.Base{ID: "mount-1"},
|
||||
AccountID: "acct-1",
|
||||
RemoteViewID: "view-1",
|
||||
RemoteViewName: "新番连载",
|
||||
CollectionType: "tvshows",
|
||||
Enabled: true,
|
||||
}); err != nil {
|
||||
t.Fatalf("create mount: %v", err)
|
||||
}
|
||||
return &serviceContainerBuilder{
|
||||
cfg: cfg,
|
||||
log: zap.NewNop(),
|
||||
repos: repos,
|
||||
c: &Container{Cfg: cfg, Log: zap.NewNop(), Repo: repos, EmbyRemote: remote},
|
||||
}
|
||||
}
|
||||
|
||||
func danmakuResolverTestEpisode(id string) map[string]any {
|
||||
return map[string]any{
|
||||
"Id": id,
|
||||
"Name": "第 1 集",
|
||||
"Type": "Episode",
|
||||
"Path": "/media/anime/PSYREN/" + id + ".mkv",
|
||||
"MediaSources": []any{
|
||||
map[string]any{"Id": id + "-src", "Container": "mkv", "Size": int64(1 << 20)},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// 目录类条目(整部剧/季)自身没有媒体源,拿它的 ID 去请求远程 /Videos/{id}/stream 只会
|
||||
// 回 500,哈希层于是每次都白白跳过。这条用例保证弹幕解析器跟随 /api/stream 的规则先
|
||||
// 落到第一集:哈希算的是真正要播的那条流。
|
||||
func TestResolveDanmakuRemoteMediaUsesPlayableChildForContainer(t *testing.T) {
|
||||
var requested []string
|
||||
b := newDanmakuRemoteResolverTest(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
requested = append(requested, r.URL.Path)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
switch r.URL.Path {
|
||||
case "/emby/Users/uid-1/Items/series-100":
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"Id": "series-100", "Name": "PSYREN", "Type": "Series", "IsFolder": true,
|
||||
})
|
||||
case "/emby/Users/uid-1/Items":
|
||||
if got := r.URL.Query().Get("ParentId"); got != "series-100" {
|
||||
t.Errorf("ParentId = %q, want series-100", got)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"Items": []map[string]any{
|
||||
{"Id": "season-1", "Type": "Season", "IndexNumber": 1},
|
||||
danmakuResolverTestEpisode("ep-1"),
|
||||
},
|
||||
})
|
||||
case "/emby/Users/uid-1/Items/ep-1":
|
||||
_ = json.NewEncoder(w).Encode(danmakuResolverTestEpisode("ep-1"))
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}))
|
||||
|
||||
media, streamURL, err := b.resolveDanmakuRemoteMedia(
|
||||
context.Background(), EncodeEmbyRemoteID("mount-1", "series-100"))
|
||||
if err != nil {
|
||||
t.Fatalf("resolve failed: %v", err)
|
||||
}
|
||||
if media == nil {
|
||||
t.Fatal("resolve returned no media metadata")
|
||||
}
|
||||
if !strings.Contains(streamURL, "/Videos/ep-1/stream") {
|
||||
t.Fatalf("stream url = %q, want the playable child ep-1", streamURL)
|
||||
}
|
||||
if strings.Contains(streamURL, "series-100") {
|
||||
t.Fatalf("stream url = %q, must not point at the container id (remote Emby answers 500)", streamURL)
|
||||
}
|
||||
if !strings.Contains(strings.Join(requested, " "), "/Items/ep-1") {
|
||||
t.Fatalf("metadata must come from the playable child, requests = %v", requested)
|
||||
}
|
||||
}
|
||||
|
||||
// 本身可播的条目(单集/电影)不能被改写:直接取它自己的流地址。
|
||||
func TestResolveDanmakuRemoteMediaKeepsPlayableItem(t *testing.T) {
|
||||
b := newDanmakuRemoteResolverTest(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if r.URL.Path != "/emby/Users/uid-1/Items/ep-2" {
|
||||
t.Errorf("unexpected upstream path %q", r.URL.Path)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(danmakuResolverTestEpisode("ep-2"))
|
||||
}))
|
||||
|
||||
media, streamURL, err := b.resolveDanmakuRemoteMedia(
|
||||
context.Background(), EncodeEmbyRemoteID("mount-1", "ep-2"))
|
||||
if err != nil {
|
||||
t.Fatalf("resolve failed: %v", err)
|
||||
}
|
||||
if media == nil {
|
||||
t.Fatal("resolve returned no media metadata")
|
||||
}
|
||||
if !strings.Contains(streamURL, "/Videos/ep-2/stream") {
|
||||
t.Fatalf("stream url = %q, want /Videos/ep-2/stream", streamURL)
|
||||
}
|
||||
}
|
||||
|
||||
// 不是远程伪装 ID 时直接报错,不去猜一个可以拉取的地址。
|
||||
func TestResolveDanmakuRemoteMediaRejectsPlainID(t *testing.T) {
|
||||
b := newDanmakuRemoteResolverTest(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
t.Error("plain local ids must not reach the remote Emby")
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}))
|
||||
|
||||
if _, _, err := b.resolveDanmakuRemoteMedia(context.Background(), "local-media-1"); err == nil {
|
||||
t.Fatal("expected an error for a non remote-emby id")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
)
|
||||
|
||||
// TestLikelyEpisodicPathSQLNormalisesSeparator 守住剧集路径判定的转义差异。
|
||||
//
|
||||
// 此前 embyLikelyEpisodicPathSQL 会额外生成一份反斜杠模式
|
||||
// (strings.ReplaceAll(pattern, "/", `\`))。PostgreSQL 的 LIKE 默认就把反斜杠
|
||||
// 当转义符,模式结尾的 `\%` 被解析成「字面百分号」而不是「反斜杠 + 通配符」,
|
||||
// 那些模式在 PostgreSQL 下永不命中;SQLite 没有默认转义符所以一直正常。
|
||||
// 现在统一在 SQL 里把分隔符归一化成 '/',模式本身不再含反斜杠。
|
||||
func TestLikelyEpisodicPathSQLNormalisesSeparator(t *testing.T) {
|
||||
clause, args := embyLikelyEpisodicPathSQL()
|
||||
if !strings.Contains(clause, "REPLACE(media.path") {
|
||||
t.Fatalf("clause no longer normalises the path separator:\n%s", clause)
|
||||
}
|
||||
if got := strings.Count(clause, "LIKE ?"); got != len(args) {
|
||||
t.Fatalf("clause has %d placeholders but %d args:\n%s", got, len(args), clause)
|
||||
}
|
||||
for _, arg := range args {
|
||||
pattern, ok := arg.(string)
|
||||
if !ok {
|
||||
t.Fatalf("unexpected pattern type %T", arg)
|
||||
}
|
||||
if strings.Contains(pattern, `\`) {
|
||||
t.Fatalf("pattern %q still contains a backslash; PostgreSQL's LIKE would treat it as an escape character", pattern)
|
||||
}
|
||||
if !strings.Contains(pattern, "/") {
|
||||
t.Fatalf("pattern %q lost its separator", pattern)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestFilterLikelyEpisodicPathsHandlesWindowsStylePaths 验证反斜杠路径(Windows
|
||||
// 宿主机入库的媒体)仍然被判成剧集并从电影库查询里排除。
|
||||
func TestFilterLikelyEpisodicPathsHandlesWindowsStylePaths(t *testing.T) {
|
||||
db := newServiceTestDB(t)
|
||||
episodic := model.Media{
|
||||
Base: model.Base{ID: "ep-1"},
|
||||
Path: `D:\Media\国产剧\我的剧\Season 1\我的剧 - S01E01.mkv`,
|
||||
SeasonNum: 1,
|
||||
EpisodeNum: 1,
|
||||
}
|
||||
movie := model.Media{
|
||||
Base: model.Base{ID: "mv-1"},
|
||||
Path: `D:\Media\电影\某电影\某电影.mkv`,
|
||||
}
|
||||
if err := db.Create(&episodic).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Create(&movie).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
q := filterLikelyEpisodicPathsFromMovieQuery(
|
||||
db.Model(&model.Media{}).Where("season_num > 0 OR episode_num > 0"),
|
||||
)
|
||||
var rows []model.Media
|
||||
if err := q.Find(&rows).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(rows) != 0 {
|
||||
t.Fatalf("windows-style episodic path was not excluded, got %d rows", len(rows))
|
||||
}
|
||||
|
||||
// 反证:电影类型的行必须保留。
|
||||
var kept []model.Media
|
||||
keepQ := filterLikelyEpisodicPathsFromMovieQuery(
|
||||
db.Model(&model.Media{}).Where("path LIKE ?", `D:\Media\电影\%`),
|
||||
)
|
||||
if err := keepQ.Find(&kept).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(kept) != 1 || kept[0].ID != "mv-1" {
|
||||
t.Fatalf("plain movie row was dropped from the movie query: %+v", kept)
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
"github.com/truewhile/MeBox/internal/repository"
|
||||
)
|
||||
|
||||
func (e *EmbyService) mediaItems(ctx context.Context, p ItemsParams) (map[string]any, error) {
|
||||
@@ -21,7 +22,11 @@ func (e *EmbyService) mediaItems(ctx context.Context, p ItemsParams) (map[string
|
||||
q = q.Where("library_id IN ? OR series_id = ?", e.mergedLibraryIDs(ctx, p.ParentID), p.ParentID)
|
||||
}
|
||||
if p.SearchTerm != "" {
|
||||
q = q.Where("title LIKE ? OR original_name LIKE ?", "%"+p.SearchTerm+"%", "%"+p.SearchTerm+"%")
|
||||
// ILIKE(PostgreSQL)/ LIKE(SQLite)由方言决定,避免换库后搜索词
|
||||
// 大小写不匹配就搜不到结果。
|
||||
likeOp := repository.LikeOperator(e.repo.DB)
|
||||
term := "%" + p.SearchTerm + "%"
|
||||
q = q.Where("title "+likeOp+" ? OR original_name "+likeOp+" ?", term, term)
|
||||
}
|
||||
if containsEmbyFilter(p.Filters, "IsFavorite") {
|
||||
if strings.TrimSpace(p.UserID) == "" {
|
||||
@@ -264,7 +269,11 @@ func (e *EmbyService) seriesItemsForLibrary(ctx context.Context, libraryID strin
|
||||
q = q.Where("library_id IN ?", e.mergedLibraryIDs(ctx, libraryID))
|
||||
}
|
||||
if p.SearchTerm != "" {
|
||||
q = q.Where("title LIKE ? OR original_name LIKE ?", "%"+p.SearchTerm+"%", "%"+p.SearchTerm+"%")
|
||||
// ILIKE(PostgreSQL)/ LIKE(SQLite)由方言决定,避免换库后搜索词
|
||||
// 大小写不匹配就搜不到结果。
|
||||
likeOp := repository.LikeOperator(e.repo.DB)
|
||||
term := "%" + p.SearchTerm + "%"
|
||||
q = q.Where("title "+likeOp+" ? OR original_name "+likeOp+" ?", term, term)
|
||||
}
|
||||
if containsEmbyFilter(p.Filters, "IsFavorite") {
|
||||
if strings.TrimSpace(p.UserID) == "" {
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
"github.com/truewhile/MeBox/internal/repository"
|
||||
)
|
||||
|
||||
// movieLibraryHasEpisodicContent 报告电影类型库里是否混入了「剧集结构」内容
|
||||
@@ -51,7 +52,11 @@ func (e *EmbyService) movieLibraryItems(ctx context.Context, p ItemsParams) (map
|
||||
q = e.applyUserMediaVisibility(ctx, q, p.UserID)
|
||||
q = q.Where("library_id IN ?", libIDs)
|
||||
if p.SearchTerm != "" {
|
||||
q = q.Where("title LIKE ? OR original_name LIKE ?", "%"+p.SearchTerm+"%", "%"+p.SearchTerm+"%")
|
||||
// 方言相关的 ILIKE/LIKE:PostgreSQL 的 LIKE 区分大小写,
|
||||
// 直接用 LIKE 会让客户端搜索的大小写与入库标题不一致时搜不到。
|
||||
likeOp := repository.LikeOperator(e.repo.DB)
|
||||
term := "%" + p.SearchTerm + "%"
|
||||
q = q.Where("title "+likeOp+" ? OR original_name "+likeOp+" ?", term, term)
|
||||
}
|
||||
if containsEmbyFilter(p.Filters, "IsFavorite") {
|
||||
if strings.TrimSpace(p.UserID) == "" {
|
||||
@@ -266,6 +271,14 @@ func filterLikelyEpisodicPathsFromMovieQuery(q *gorm.DB) *gorm.DB {
|
||||
return q.Where("NOT ((media.season_num > 0 OR media.episode_num > 0) AND ("+clause+"))", args...)
|
||||
}
|
||||
|
||||
// embyEpisodicPathExpr 把路径分隔符归一化成 '/' 后再小写化,供剧集路径判定使用。
|
||||
//
|
||||
// 反斜杠归一化必须放在 SQL 里做,而不是再生成一份反斜杠模式:PostgreSQL 的
|
||||
// LIKE 默认就把反斜杠当转义符,模式结尾的 `\%` 会被解析成「字面百分号」而不是
|
||||
// 「反斜杠 + 通配符」,那些模式在 PostgreSQL 下永不命中(SQLite 没有默认转义符,
|
||||
// 所以只有换库之后才会暴露)。归一化列值后两种方言结果一致。
|
||||
const embyEpisodicPathExpr = `LOWER(REPLACE(media.path, '\', '/'))`
|
||||
|
||||
func embyLikelyEpisodicPathSQL() (string, []any) {
|
||||
patterns := []string{
|
||||
"%/season %/%", "%/season.%/%", "%/season-%/%", "%/season_%/%",
|
||||
@@ -274,15 +287,11 @@ func embyLikelyEpisodicPathSQL() (string, []any) {
|
||||
"%/电视剧/%", "%/剧集/%", "%/连续剧/%", "%/短剧/%", "%/国产剧/%", "%/国剧/%", "%/大陆剧/%", "%/华语剧/%", "%/国产电视剧/%", "%/大陆电视剧/%", "%/华语电视剧/%", "%/欧美剧/%", "%/欧美电视剧/%", "%/美剧/%", "%/英剧/%", "%/日韩剧/%", "%/日韩电视剧/%", "%/日剧/%", "%/韩剧/%", "%/港剧/%", "%/台剧/%", "%/港台剧/%", "%/泰剧/%",
|
||||
"%/日番/%", "%/国漫/%", "%/番剧/%", "%/动漫/%", "%/特别篇/%", "%/特別篇/%", "%/番外/%", "%/特典/%",
|
||||
}
|
||||
clauses := make([]string, 0, len(patterns)*2)
|
||||
args := make([]any, 0, len(patterns)*2)
|
||||
clauses := make([]string, 0, len(patterns))
|
||||
args := make([]any, 0, len(patterns))
|
||||
for _, pattern := range patterns {
|
||||
clauses = append(clauses, "LOWER(media.path) LIKE ?")
|
||||
clauses = append(clauses, embyEpisodicPathExpr+" LIKE ?")
|
||||
args = append(args, pattern)
|
||||
if strings.Contains(pattern, "/") {
|
||||
clauses = append(clauses, "LOWER(media.path) LIKE ?")
|
||||
args = append(args, strings.ReplaceAll(pattern, "/", `\`))
|
||||
}
|
||||
}
|
||||
return strings.Join(clauses, " OR "), args
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
@@ -237,6 +238,88 @@ func (r *EmbyRemoteService) ConfiguredRemoteHosts(ctx context.Context) []string
|
||||
return hosts
|
||||
}
|
||||
|
||||
// findAccountByHost 返回某条线路主机名匹配 host 的启用远程 Emby 账号(无则 nil)。
|
||||
func (r *EmbyRemoteService) findAccountByHost(ctx context.Context, host string) *model.StrmAccount {
|
||||
host = normalizeHostKey(host)
|
||||
if host == "" || r == nil || r.repo == nil || r.repo.StrmAccount == nil {
|
||||
return nil
|
||||
}
|
||||
accounts, err := r.ListAccounts(ctx)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
for i := range accounts {
|
||||
lines, _, err := r.LinesOf(&accounts[i])
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, line := range lines {
|
||||
u, err := url.Parse(line.URL)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if normalizeHostKey(u.Hostname()) == host {
|
||||
return &accounts[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// normalizeHostKey 去掉端口并小写化,便于主机名字符串比较。
|
||||
func normalizeHostKey(host string) string {
|
||||
host = strings.ToLower(strings.TrimSpace(host))
|
||||
if host == "" {
|
||||
return ""
|
||||
}
|
||||
if h, _, err := net.SplitHostPort(host); err == nil {
|
||||
return strings.ToLower(strings.TrimSpace(h))
|
||||
}
|
||||
return host
|
||||
}
|
||||
|
||||
// RemoteEmbyImageTokenForHost 返回图片代理回源某个远程 Emby 主机所需的当前
|
||||
// api_key。host 不属于任何已配置账号时返回 ok=false。
|
||||
func (r *EmbyRemoteService) RemoteEmbyImageTokenForHost(ctx context.Context, host string) (string, bool) {
|
||||
if r == nil || r.repo == nil {
|
||||
return "", false
|
||||
}
|
||||
acct := r.findAccountByHost(ctx, host)
|
||||
if acct == nil {
|
||||
return "", false
|
||||
}
|
||||
cfg, err := r.configOf(acct)
|
||||
if err != nil || strings.TrimSpace(cfg.Token) == "" {
|
||||
return "", false
|
||||
}
|
||||
return cfg.Token, true
|
||||
}
|
||||
|
||||
// RefreshRemoteEmbyImageTokenForHost 强制用用户名/密码重新登录拥有该主机的账号
|
||||
// 并持久化新 token,供图片代理在上游 401 时自愈。仅配置了 api_key、没有登录
|
||||
// 凭据的账号无法自动刷新,返回错误由调用方按原失败处理。
|
||||
func (r *EmbyRemoteService) RefreshRemoteEmbyImageTokenForHost(ctx context.Context, host string) (string, error) {
|
||||
if r == nil || r.repo == nil {
|
||||
return "", errors.New("远程 Emby 服务不可用")
|
||||
}
|
||||
acct := r.findAccountByHost(ctx, host)
|
||||
if acct == nil {
|
||||
return "", fmt.Errorf("未找到主机 %s 对应的远程 Emby 账号", host)
|
||||
}
|
||||
cfg, err := r.configOf(acct)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if strings.TrimSpace(cfg.Username) == "" || strings.TrimSpace(cfg.Password) == "" {
|
||||
return "", errors.New("远程 Emby 账号未配置用户名/密码,无法自动刷新 api_key")
|
||||
}
|
||||
cfg.Token = "" // 强制走登录流程,忽略已失效的 api_key
|
||||
if err := r.ensureToken(ctx, acct, cfg); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return cfg.Token, nil
|
||||
}
|
||||
|
||||
// AccountByID 按 ID 查找远程 Emby 挂载账号(不存在或类型不符返回 nil)。
|
||||
func (r *EmbyRemoteService) AccountByID(ctx context.Context, id string) *model.StrmAccount {
|
||||
if strings.TrimSpace(id) == "" {
|
||||
@@ -699,17 +782,23 @@ func (r *EmbyRemoteService) doGetOnLine(ctx context.Context, acct *model.StrmAcc
|
||||
return err
|
||||
}
|
||||
if status == http.StatusUnauthorized && attempt == 0 {
|
||||
// 401:只清当前线路的内存 token 并立即重认证;不在此时删除
|
||||
// DB 里的 api_key——①外层还会按线路故障转移(其他线路可能
|
||||
// 存有自己的 token);②纯 api_key 账号删除后无法再认证,一次
|
||||
// 线路误报就会把账号“砖化”。重认证成功后 persistToken 会用
|
||||
// 新 token 覆盖 api_key。
|
||||
// 401:只清当前线路的内存 token 并立即重认证;不先删 DB 里的
|
||||
// api_key——纯 api_key 账号删除后无法再认证,一次线路误报就会
|
||||
// 把账号“砖化”。重认证成功后把新 token 写回:既让重试用上正确
|
||||
// 凭据,也避免每个后续请求都重新登录一次。
|
||||
previous := cfg.Token
|
||||
cfg.Token = ""
|
||||
if err := r.ensureTokenOnLine(ctx, acct, cfg); err != nil {
|
||||
return fmt.Errorf("认证重试失败: %w", err)
|
||||
}
|
||||
if q != nil {
|
||||
q.Set("api_key", cfg.Token)
|
||||
}
|
||||
master.Token = cfg.Token
|
||||
master.RemoteUserID = cfg.RemoteUserID
|
||||
if cfg.Token != "" && cfg.Token != previous {
|
||||
_ = r.persistToken(ctx, acct, cfg)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if status >= 300 {
|
||||
@@ -1498,7 +1587,22 @@ func (r *EmbyRemoteService) proxyVideoStreamOnLine(ctx context.Context, w http.R
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode >= 400 {
|
||||
data, _ := io.ReadAll(io.LimitReader(resp.Body, 256))
|
||||
return redactSensitiveError(fmt.Errorf("远程 Emby 视频流失败(%d): %s", resp.StatusCode, strings.TrimSpace(string(data))))
|
||||
detail := strings.TrimSpace(string(data))
|
||||
// 远程 Emby 对「整部剧/目录」这类自身没有媒体源的条目会直接 500,正文是
|
||||
// 一句 .NET 异常(如 Unable to cast object of type 'Series' to type
|
||||
// 'IHasMediaSources')。原样丢给用户没人看得懂,也没有排查线索——细节进日志,
|
||||
// 给用户一句能照着做的提示。
|
||||
if r.log != nil {
|
||||
r.log.Warn("remote emby video stream rejected",
|
||||
zap.String("remote_id", remoteID),
|
||||
zap.Int("status", resp.StatusCode),
|
||||
zap.String("detail", detail),
|
||||
)
|
||||
}
|
||||
if resp.StatusCode >= 500 {
|
||||
return redactSensitiveError(fmt.Errorf("远程 Emby 无法播放该条目(HTTP %d):请在详情页选择具体集数后重试", resp.StatusCode))
|
||||
}
|
||||
return redactSensitiveError(fmt.Errorf("远程 Emby 视频流失败(%d): %s", resp.StatusCode, detail))
|
||||
}
|
||||
for _, header := range []string{"Content-Type", "Content-Length", "Content-Range", "Accept-Ranges", "ETag", "Cache-Control"} {
|
||||
if value := resp.Header.Get(header); value != "" {
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"testing"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/config"
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
"github.com/truewhile/MeBox/internal/repository"
|
||||
)
|
||||
|
||||
func TestRemoteEmbyImageTokenForHostUnknownHost(t *testing.T) {
|
||||
svc, _, _ := newImageTagTestService(t)
|
||||
|
||||
if token, ok := svc.RemoteEmbyImageTokenForHost(t.Context(), "nope.example"); ok || token != "" {
|
||||
t.Fatalf("got (%q,%v), want no token for a host not owned by any account", token, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteEmbyImageTokenForHostMatchesConfiguredLine(t *testing.T) {
|
||||
svc, _, _ := newImageTagTestService(t)
|
||||
|
||||
token, ok := svc.RemoteEmbyImageTokenForHost(t.Context(), "emby.test")
|
||||
if !ok || token != "fake-token" {
|
||||
t.Fatalf("got (%q,%v), want the account token for the configured line host", token, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRefreshRemoteEmbyImageTokenForHostRequiresCredentials(t *testing.T) {
|
||||
svc, _, _ := newImageTagTestService(t)
|
||||
|
||||
// The account carries only an api_key — there is nothing to re-authenticate with.
|
||||
if _, err := svc.RefreshRemoteEmbyImageTokenForHost(t.Context(), "emby.test"); err == nil {
|
||||
t.Fatal("expected an error when the account has no username/password to re-authenticate with")
|
||||
}
|
||||
}
|
||||
|
||||
// TestRefreshRemoteEmbyImageTokenForHostRelogins 覆盖「token 被撤销后自动恢复」:
|
||||
// 用用户名/密码重新登录拿到新 token,并持久化,后续请求不再重复登录。
|
||||
func TestRefreshRemoteEmbyImageTokenForHostRelogins(t *testing.T) {
|
||||
const fresh = "fresh-token"
|
||||
logins := 0
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/emby/Users/AuthenticateByName" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
logins++
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"AccessToken":"` + fresh + `","User":{"Id":"remote-user"}}`))
|
||||
}))
|
||||
defer upstream.Close()
|
||||
|
||||
db := newServiceTestDB(t, &model.StrmAccount{}, &model.EmbyMount{})
|
||||
repos := repository.New(db)
|
||||
svc := NewEmbyRemoteService(&config.Config{}, zap.NewNop(), repos, NewCryptoService("", zap.NewNop()))
|
||||
acct := &model.StrmAccount{
|
||||
Base: model.Base{ID: "acct-refresh"},
|
||||
Name: "refresh-emby",
|
||||
Provider: model.StrmProviderEmbyRemote,
|
||||
Enabled: true,
|
||||
Config: `{"url":"` + upstream.URL + `","api_key":"stale-token","username":"u","password":"p"}`,
|
||||
}
|
||||
if err := repos.StrmAccount.Create(t.Context(), acct); err != nil {
|
||||
t.Fatalf("create account: %v", err)
|
||||
}
|
||||
|
||||
u, err := url.Parse(upstream.URL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
host := u.Hostname()
|
||||
|
||||
if token, ok := svc.RemoteEmbyImageTokenForHost(t.Context(), host); !ok || token != "stale-token" {
|
||||
t.Fatalf("token before refresh = (%q,%v), want the stale stored token", token, ok)
|
||||
}
|
||||
|
||||
token, err := svc.RefreshRemoteEmbyImageTokenForHost(t.Context(), host)
|
||||
if err != nil {
|
||||
t.Fatalf("refresh: %v", err)
|
||||
}
|
||||
if token != fresh {
|
||||
t.Fatalf("refreshed token = %q, want %q", token, fresh)
|
||||
}
|
||||
if logins != 1 {
|
||||
t.Fatalf("logins = %d, want 1", logins)
|
||||
}
|
||||
|
||||
// 新 token 必须落库,否则每个请求都要重新登录一次。
|
||||
if token, ok := svc.RemoteEmbyImageTokenForHost(t.Context(), host); !ok || token != fresh {
|
||||
t.Fatalf("token after refresh = (%q,%v), want the persisted %q", token, ok, fresh)
|
||||
}
|
||||
if logins != 1 {
|
||||
t.Fatalf("logins = %d, want 1 after the refresh was persisted", logins)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/config"
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
"github.com/truewhile/MeBox/internal/repository"
|
||||
)
|
||||
|
||||
// newPlayableIDTestService 起一个假的远程 Emby,并建好账号。
|
||||
func newPlayableIDTestService(t *testing.T, handler http.Handler) (*EmbyRemoteService, *model.StrmAccount) {
|
||||
t.Helper()
|
||||
server := httptest.NewServer(handler)
|
||||
t.Cleanup(server.Close)
|
||||
|
||||
db := newServiceTestDB(t, &model.StrmAccount{}, &model.EmbyMount{})
|
||||
repos := repository.New(db)
|
||||
cfg := &config.Config{}
|
||||
remote := NewEmbyRemoteService(cfg, zap.NewNop(), repos, NewCryptoService("", zap.NewNop()))
|
||||
remote.SetRuntimeCache(NewRuntimeCacheService(cfg, zap.NewNop()))
|
||||
|
||||
rawConfig, _ := json.Marshal(map[string]string{
|
||||
"url": server.URL,
|
||||
"api_key": "test-api-key",
|
||||
"remote_user_id": "uid-1",
|
||||
})
|
||||
acct := &model.StrmAccount{
|
||||
Base: model.Base{ID: "acct-1"},
|
||||
Name: "远程 Emby",
|
||||
Provider: model.StrmProviderEmbyRemote,
|
||||
Config: string(rawConfig),
|
||||
Enabled: true,
|
||||
}
|
||||
if err := repos.StrmAccount.Create(t.Context(), acct); err != nil {
|
||||
t.Fatalf("create account: %v", err)
|
||||
}
|
||||
return remote, acct
|
||||
}
|
||||
|
||||
func playableIDTestEpisode(id string, index, sources int) map[string]any {
|
||||
item := map[string]any{
|
||||
"Id": id,
|
||||
"Name": "第" + strconv.Itoa(index) + "集",
|
||||
"Type": "Episode",
|
||||
"SeriesId": "series-100",
|
||||
"ParentIndexNumber": 1,
|
||||
"IndexNumber": index,
|
||||
}
|
||||
if sources > 0 {
|
||||
item["MediaSources"] = []any{map[string]any{"Id": id + "-source"}}
|
||||
}
|
||||
return item
|
||||
}
|
||||
|
||||
// 整部剧的 ID 自身没有媒体源,远程 Emby 对它的取流请求会回 500
|
||||
// (Unable to cast 'Series' to 'IHasMediaSources')。这条用例保证我们会换成第一集,
|
||||
// 而不是把这种 ID 直接交给远程。
|
||||
func TestPlayableRemoteIDResolvesContainerToFirstEpisode(t *testing.T) {
|
||||
var itemsCalls int64
|
||||
remote, acct := newPlayableIDTestService(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
switch r.URL.Path {
|
||||
case "/emby/Users/uid-1/Items/series-100":
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"Id": "series-100", "Name": "剧一", "Type": "Series", "IsFolder": true,
|
||||
})
|
||||
case "/emby/Users/uid-1/Items/ep-2":
|
||||
_ = json.NewEncoder(w).Encode(playableIDTestEpisode("ep-2", 2, 1))
|
||||
case "/emby/Users/uid-1/Items":
|
||||
atomic.AddInt64(&itemsCalls, 1)
|
||||
if got := r.URL.Query().Get("ParentId"); got != "series-100" {
|
||||
t.Errorf("ParentId = %q, want series-100", got)
|
||||
}
|
||||
// 故意乱序,并让第一条没有媒体源:只有真按季/集排序、且检查过媒体源,
|
||||
// 才会选中 ep-1。
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"Items": []map[string]any{
|
||||
{"Id": "season-1", "Type": "Season", "IndexNumber": 1},
|
||||
playableIDTestEpisode("ep-3", 3, 1),
|
||||
playableIDTestEpisode("ep-1", 1, 1),
|
||||
playableIDTestEpisode("ep-2", 2, 1),
|
||||
},
|
||||
})
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}))
|
||||
|
||||
ctx := context.Background()
|
||||
if got := remote.PlayableRemoteID(ctx, acct, "series-100"); got != "ep-1" {
|
||||
t.Fatalf("series id resolved to %q, want ep-1", got)
|
||||
}
|
||||
// 播放器对同一个地址会反复发 Range 请求,解析结果必须命中缓存,不再打远程。
|
||||
if got := remote.PlayableRemoteID(ctx, acct, "series-100"); got != "ep-1" {
|
||||
t.Fatalf("cached resolve = %q, want ep-1", got)
|
||||
}
|
||||
if n := atomic.LoadInt64(&itemsCalls); n != 1 {
|
||||
t.Fatalf("episode query issued %d times, want 1 (cached)", n)
|
||||
}
|
||||
}
|
||||
|
||||
// 本身带媒体源的条目(电影/单集)必须原样返回,不能被改写。
|
||||
func TestPlayableRemoteIDKeepsPlayableItem(t *testing.T) {
|
||||
var queries int64
|
||||
remote, acct := newPlayableIDTestService(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
atomic.AddInt64(&queries, 1)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if r.URL.Path != "/emby/Users/uid-1/Items/ep-2" {
|
||||
t.Errorf("unexpected upstream path %s", r.URL.Path)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(playableIDTestEpisode("ep-2", 2, 1))
|
||||
}))
|
||||
|
||||
if got := remote.PlayableRemoteID(context.Background(), acct, "ep-2"); got != "ep-2" {
|
||||
t.Fatalf("playable item = %q, want ep-2", got)
|
||||
}
|
||||
if n := atomic.LoadInt64(&queries); n != 1 {
|
||||
t.Fatalf("upstream queries = %d, want 1", n)
|
||||
}
|
||||
}
|
||||
|
||||
// 远程不可用/查不到条目时保持原 ID:让原有流程去报错,别把播放地址改成空。
|
||||
func TestPlayableRemoteIDKeepsOriginalOnFailure(t *testing.T) {
|
||||
remote, acct := newPlayableIDTestService(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}))
|
||||
|
||||
if got := remote.PlayableRemoteID(context.Background(), acct, "series-100"); got != "series-100" {
|
||||
t.Fatalf("failure path = %q, want the original id", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 容器下没有任何带媒体源的子条目时同样返回原 ID(下一层会给友好报错)。
|
||||
func TestPlayableRemoteIDKeepsContainerWithoutPlayableChild(t *testing.T) {
|
||||
remote, acct := newPlayableIDTestService(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if r.URL.Path == "/emby/Users/uid-1/Items" {
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"Items": []map[string]any{
|
||||
{"Id": "season-1", "Type": "Season", "IndexNumber": 1},
|
||||
},
|
||||
})
|
||||
return
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"Id": "series-100", "Type": "Series", "IsFolder": true})
|
||||
}))
|
||||
|
||||
if got := remote.PlayableRemoteID(context.Background(), acct, "series-100"); got != "series-100" {
|
||||
t.Fatalf("container without playable child = %q, want the original id", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// 远程条目自身没有媒体源时(例如把「整部剧」的 ID 当播放目标),远程 Emby 会回
|
||||
// 500 + 一句 .NET 异常。异常原文不能直接丢给用户,但要能从日志里查得到。
|
||||
func TestProxyVideoStreamOnLineHidesUpstreamException(t *testing.T) {
|
||||
const upstreamBody = "Unable to cast object of type 'MediaBrowser.Controller.Entities.TV.Series' to type 'MediaBrowser.Controller.Entities.IHasMediaSources'."
|
||||
|
||||
var gotPath, gotStatic, gotRange, gotToken string
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
||||
gotPath = req.URL.Path
|
||||
gotStatic = req.URL.Query().Get("Static")
|
||||
gotRange = req.Header.Get("Range")
|
||||
gotToken = req.Header.Get("X-Emby-Token")
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
_, _ = w.Write([]byte(upstreamBody))
|
||||
}))
|
||||
defer upstream.Close()
|
||||
|
||||
svc := &EmbyRemoteService{stream: upstream.Client(), log: zap.NewNop()}
|
||||
cfg := &EmbyRemoteConfig{BaseURL: upstream.URL, Token: "tok-1"}
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/stream/embyremote~acct~222679", nil)
|
||||
req.Header.Set("Range", "bytes=0-1023")
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
err := svc.proxyVideoStreamOnLine(context.Background(), rec, req, cfg, "222679")
|
||||
if err == nil {
|
||||
t.Fatal("expected an error for upstream 500")
|
||||
}
|
||||
if gotPath != "/emby/Videos/222679/stream" {
|
||||
t.Fatalf("upstream path = %q", gotPath)
|
||||
}
|
||||
if gotStatic != "true" {
|
||||
t.Fatalf("Static = %q, want true", gotStatic)
|
||||
}
|
||||
if gotRange != "bytes=0-1023" {
|
||||
t.Fatalf("Range = %q, want forwarded range", gotRange)
|
||||
}
|
||||
if gotToken != "tok-1" {
|
||||
t.Fatalf("X-Emby-Token = %q", gotToken)
|
||||
}
|
||||
if rec.Body.Len() != 0 {
|
||||
t.Fatalf("nothing should be written to the client on upstream failure, got %q", rec.Body.String())
|
||||
}
|
||||
|
||||
msg := err.Error()
|
||||
if strings.Contains(msg, "Unable to cast") || strings.Contains(msg, "MediaBrowser") {
|
||||
t.Fatalf("raw upstream exception must not leak to clients: %q", msg)
|
||||
}
|
||||
if !strings.Contains(msg, "无法播放该条目") {
|
||||
t.Fatalf("error should tell the user what to do, got %q", msg)
|
||||
}
|
||||
}
|
||||
|
||||
// 4xx(例如远程条目被删除/无权限)保持原有措辞,带着上游说明便于排查。
|
||||
func TestProxyVideoStreamOnLineKeepsClientErrorDetail(t *testing.T) {
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
_, _ = w.Write([]byte("Item not found"))
|
||||
}))
|
||||
defer upstream.Close()
|
||||
|
||||
svc := &EmbyRemoteService{stream: upstream.Client(), log: zap.NewNop()}
|
||||
cfg := &EmbyRemoteConfig{BaseURL: upstream.URL, Token: "tok-1"}
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/stream/embyremote~acct~404", nil)
|
||||
err := svc.proxyVideoStreamOnLine(context.Background(), httptest.NewRecorder(), req, cfg, "404")
|
||||
if err == nil {
|
||||
t.Fatal("expected an error for upstream 404")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "404") || !strings.Contains(err.Error(), "Item not found") {
|
||||
t.Fatalf("client error detail should be kept, got %q", err.Error())
|
||||
}
|
||||
}
|
||||
@@ -820,6 +820,123 @@ func (r *EmbyRemoteService) remoteItemSeriesID(ctx context.Context, acct *model.
|
||||
return remoteItemString(out, "SeriesId")
|
||||
}
|
||||
|
||||
// PlayableRemoteID 把「自身没有媒体源」的远程条目换成可直接播放的条目 ID。
|
||||
//
|
||||
// 远程 Emby 只为带媒体源的条目提供 /Videos/{id}/stream:拿整部剧/季/目录的 ID 去
|
||||
// 请求,它会回 500 加一句 .NET 异常(Unable to cast object of type 'Series' to
|
||||
// type 'IHasMediaSources')。首页轮播、剧集库卡片、外部播放器与第三方客户端都可能
|
||||
// 把这种 ID 交过来,统一在这里回落到第一集,保证拿到的 ID 真能取到流。
|
||||
//
|
||||
// 条目本身可播、或解析失败时返回原 ID(调用方拿到后用原逻辑处理即可,无需区分错误)。
|
||||
func (r *EmbyRemoteService) PlayableRemoteID(ctx context.Context, acct *model.StrmAccount, remoteID string) string {
|
||||
remoteID = strings.TrimSpace(remoteID)
|
||||
if r == nil || acct == nil || remoteID == "" {
|
||||
return remoteID
|
||||
}
|
||||
cfg, err := r.remoteConfigWithToken(ctx, acct)
|
||||
if err != nil {
|
||||
return remoteID
|
||||
}
|
||||
cacheKey := r.remoteCacheKey("playable-remote-id", acct.ID, remoteID)
|
||||
if r.cache != nil {
|
||||
var cached string
|
||||
// 播放器会对同一个地址发很多次 Range 请求,判定结果必须走缓存,否则每次拖动
|
||||
// 进度条都要多打两次远程 Emby。缓存比常规媒体缓存长:条目可播性是静态属性。
|
||||
if r.cache.GetJSON(ctx, cacheKey, &cached) {
|
||||
if cached == "" {
|
||||
return remoteID
|
||||
}
|
||||
return cached
|
||||
}
|
||||
}
|
||||
resolved := r.resolvePlayableRemoteID(ctx, acct, cfg, remoteID)
|
||||
// 解析失败不写缓存:远程抖动时下次照样重试,而不是把「播不了」记住十分钟。
|
||||
if r.cache != nil && resolved != "" {
|
||||
r.cache.SetJSON(ctx, cacheKey, resolved, playableRemoteIDCacheTTL)
|
||||
}
|
||||
if resolved == "" {
|
||||
return remoteID
|
||||
}
|
||||
return resolved
|
||||
}
|
||||
|
||||
// playableRemoteIDCacheTTL 是「条目可播性」判定的缓存时长。
|
||||
const playableRemoteIDCacheTTL = 10 * time.Minute
|
||||
|
||||
// resolvePlayableRemoteID 返回 remoteID 对应的可播条目:本身带媒体源时原样返回,
|
||||
// 容器条目(整部剧/季/目录)返回排在最前的、确实带媒体源的子条目。
|
||||
func (r *EmbyRemoteService) resolvePlayableRemoteID(ctx context.Context, acct *model.StrmAccount, cfg *EmbyRemoteConfig, remoteID string) string {
|
||||
var item map[string]any
|
||||
if err := r.doGet(ctx, acct, cfg, "/Users/"+url.PathEscape(r.remoteUserID(cfg))+"/Items/"+url.PathEscape(remoteID), nil, &item); err != nil {
|
||||
return ""
|
||||
}
|
||||
if remoteItemHasMediaSources(item) || !remoteItemIsContainer(item) {
|
||||
return remoteID
|
||||
}
|
||||
|
||||
q := url.Values{}
|
||||
q.Set("ParentId", remoteID)
|
||||
q.Set("Recursive", "true")
|
||||
q.Set("IncludeItemTypes", "Episode,Movie,Video")
|
||||
q.Set("Fields", "MediaSources")
|
||||
q.Set("SortBy", "ParentIndexNumber,IndexNumber,SortName")
|
||||
q.Set("Limit", "50")
|
||||
var body struct {
|
||||
Items []map[string]any `json:"Items"`
|
||||
}
|
||||
if err := r.doGet(ctx, acct, cfg, "/Users/"+url.PathEscape(r.remoteUserID(cfg))+"/Items", q, &body); err != nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
// 远程 SortBy 只覆盖它认识的字段(不同 Emby/Jellyfin 版本支持度不一),
|
||||
// 这里再按季/集排一次,确保拿到的是第一集而不是服务端返回顺序里的第一条。
|
||||
children := body.Items
|
||||
sort.SliceStable(children, func(i, j int) bool {
|
||||
if a, b := remoteItemInt(children[i], "ParentIndexNumber"), remoteItemInt(children[j], "ParentIndexNumber"); a != b {
|
||||
return a < b
|
||||
}
|
||||
if a, b := remoteItemInt(children[i], "IndexNumber"), remoteItemInt(children[j], "IndexNumber"); a != b {
|
||||
return a < b
|
||||
}
|
||||
return remoteItemString(children[i], "SortName") < remoteItemString(children[j], "SortName")
|
||||
})
|
||||
for _, child := range children {
|
||||
if id := remoteItemString(child, "Id"); id != "" && remoteItemHasMediaSources(child) {
|
||||
return id
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// remoteItemHasMediaSources 该条目是否自带媒体源(即能直接取流)。
|
||||
func remoteItemHasMediaSources(item map[string]any) bool {
|
||||
if item == nil {
|
||||
return false
|
||||
}
|
||||
switch sources := item["MediaSources"].(type) {
|
||||
case []any:
|
||||
return len(sources) > 0
|
||||
case []map[string]any:
|
||||
return len(sources) > 0
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// remoteItemIsContainer 该条目是否是「装别人」的容器:整部剧/季/目录/合集。
|
||||
func remoteItemIsContainer(item map[string]any) bool {
|
||||
switch remoteItemString(item, "Type") {
|
||||
case "Series", "Season", "Folder", "BoxSet", "CollectionFolder", "AggregateFolder", "Playlist":
|
||||
return true
|
||||
}
|
||||
// 类型缺失时退回 IsFolder(Emby 的目录类条目都会带)。
|
||||
if _, ok := item["Type"]; !ok {
|
||||
if isFolder, okBool := item["IsFolder"].(bool); okBool {
|
||||
return isFolder
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ─── 远程 item JSON 取值辅助 ────────────────────────────────────────────────
|
||||
|
||||
func anyString(v any) string {
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"net/http"
|
||||
@@ -65,6 +66,16 @@ type ImageProxy struct {
|
||||
allowedHostsMu sync.Mutex
|
||||
allowedHostsCache map[string]bool
|
||||
allowedHostsAt time.Time
|
||||
|
||||
// remoteEmbyTokenFn / refreshRemoteEmbyTokenFn let the proxy carry a fresh
|
||||
// credential for configured remote-Emby mounts. Image URLs are minted from
|
||||
// whatever token the account had when the payload was built, so a URL can
|
||||
// outlive its api_key; these hooks re-supply the current one and force a
|
||||
// re-login when the mount rejects it.
|
||||
remoteEmbyTokenFn func(ctx context.Context, host string) (string, bool)
|
||||
refreshRemoteEmbyTokenFn func(ctx context.Context, host string) (string, error)
|
||||
remoteEmbyTokenMu sync.Mutex
|
||||
remoteEmbyTokenCache map[string]remoteEmbyImageToken
|
||||
}
|
||||
|
||||
const (
|
||||
@@ -100,17 +111,20 @@ func NewImageProxy(cfg *config.Config, log *zap.Logger) *ImageProxy {
|
||||
// 仅 URL 解析层的 isPrivateHost 可被十进制/十六进制 IP、解析到
|
||||
// 私网的域名与 DNS rebinding 绕过;在拨号层对最终连接 IP 做二次
|
||||
// 校验(含重定向后的每条连接)堵住该旁路。
|
||||
// 用户明确配置的远程挂载源(如内网 Emby)豁免该私网限制。
|
||||
dialer := &net.Dialer{
|
||||
//
|
||||
// 用户明确配置的远程挂载源(如内网 Emby)与显式配置的服务(如
|
||||
// MetaTube)豁免该私网限制。豁免必须在 DialContext 这一层判断:
|
||||
// Control 回调收到的 address 已经是**解析后的 IP**,而放行名单里
|
||||
// 存的是主机名(mebox-metatube-1、NAS 名),在 Control 里比较永远
|
||||
// 不相等——那段分支对主机名条目是死代码,现场表现就是「名单里有
|
||||
// 这个主机,回源依然被拒」。
|
||||
guardDialer := &net.Dialer{
|
||||
Timeout: 15 * time.Second,
|
||||
Control: func(_, address string, _ syscall.RawConn) error {
|
||||
host, _, err := net.SplitHostPort(address)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if proxy.isAllowedRemoteHost(host) {
|
||||
return nil
|
||||
}
|
||||
ip := net.ParseIP(host)
|
||||
if ip == nil {
|
||||
return errors.New("image proxy: refusing non-IP dial target")
|
||||
@@ -121,7 +135,19 @@ func NewImageProxy(cfg *config.Config, log *zap.Logger) *ImageProxy {
|
||||
return nil
|
||||
},
|
||||
}
|
||||
transport.DialContext = dialer.DialContext
|
||||
// 放行目标的专用拨号器:不再做私网校验,但仍然只连到请求里的那个
|
||||
// 主机(豁免的是私网限制,不是「任意目标」)。
|
||||
allowlistedDialer := &net.Dialer{Timeout: 15 * time.Second}
|
||||
transport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) {
|
||||
host, _, err := net.SplitHostPort(address)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if proxy.isAllowedRemoteHost(host) {
|
||||
return allowlistedDialer.DialContext(ctx, network, address)
|
||||
}
|
||||
return guardDialer.DialContext(ctx, network, address)
|
||||
}
|
||||
}
|
||||
|
||||
proxy.client = &http.Client{Timeout: 30 * time.Second, Transport: transport}
|
||||
|
||||
@@ -0,0 +1,193 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/repository"
|
||||
)
|
||||
|
||||
// 显式配置的服务地址也要进入图片代理的私网放行名单。
|
||||
//
|
||||
// 背景(现场 2026-10-16):MetaTube 刮削后端被配置成 docker 服务名
|
||||
// `http://mebox-metatube-1:8081`,刮削结果里的 poster_url 就是这个地址。
|
||||
// 图片代理的连接层 SSRF 校验只放行远程 Emby 挂载主机,于是每一次封面回源都
|
||||
// 被拒(日志 warn: requests to private/internal hosts are not allowed),
|
||||
// 前端长期只拿到占位图。
|
||||
//
|
||||
// 这里的取舍与 isPrivateHost 的注释一致:私网校验防的是「用户可控 URL 打到
|
||||
// 内网」,而管理员显式配置的服务地址属于受信输入,和远程 Emby 挂载同级。
|
||||
func configuredServiceImageHosts(values ...string) []string {
|
||||
seen := make(map[string]struct{}, len(values))
|
||||
out := make([]string, 0, len(values))
|
||||
for _, raw := range values {
|
||||
host := imageHostFromBaseURL(raw)
|
||||
if host == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[host]; ok {
|
||||
continue
|
||||
}
|
||||
seen[host] = struct{}{}
|
||||
out = append(out, host)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// internalServiceImageHosts 只保留指向内网/局域网的服务地址。
|
||||
//
|
||||
// 公网域名(image.tmdb.org、api.themoviedb.org 等)必须留在名单之外:放行后
|
||||
// 图片代理会优先走直连客户端,而 GFW 环境恰恰需要先用 HTTP(S)_PROXY。把它们
|
||||
// 加进来会让每张海报都白付一次直连失败。
|
||||
func internalServiceImageHosts(values ...string) []string {
|
||||
hosts := configuredServiceImageHosts(values...)
|
||||
out := make([]string, 0, len(hosts))
|
||||
for _, host := range hosts {
|
||||
if isInternalServiceHost(host) {
|
||||
out = append(out, host)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// isInternalServiceHost 判断主机名是否只在内网可达:私网/回环/链路本地 IP、
|
||||
// 单标签服务名(docker compose 服务名如 mebox-metatube-1),或常见的内部域名后缀。
|
||||
func isInternalServiceHost(host string) bool {
|
||||
host = strings.ToLower(strings.TrimSpace(host))
|
||||
if host == "" {
|
||||
return false
|
||||
}
|
||||
if ip := net.ParseIP(host); ip != nil {
|
||||
return isPrivateIP(ip)
|
||||
}
|
||||
// 单标签名(不含点)不是公网可解析的 FQDN,只可能是容器名/主机名。
|
||||
if !strings.Contains(host, ".") {
|
||||
return true
|
||||
}
|
||||
for _, suffix := range []string{".local", ".internal", ".lan", ".home.arpa", ".localhost"} {
|
||||
if strings.HasSuffix(host, suffix) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// imageHostFromBaseURL 从配置里的 base URL 取出小写主机名(不含端口)。
|
||||
// 容忍缺少 scheme 的写法(`mebox-metatube-1:8081`),并拒绝非 http/https。
|
||||
func imageHostFromBaseURL(raw string) string {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return ""
|
||||
}
|
||||
if !strings.Contains(raw, "://") {
|
||||
raw = "http://" + raw
|
||||
}
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
switch strings.ToLower(u.Scheme) {
|
||||
case "http", "https":
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
return strings.ToLower(strings.TrimSpace(u.Hostname()))
|
||||
}
|
||||
|
||||
// mergeImageProxyAllowedHosts 合并多组放行主机(远程 Emby 挂载 + 显式配置的
|
||||
// 服务地址),去重时忽略大小写与端口。
|
||||
func mergeImageProxyAllowedHosts(groups ...[]string) []string {
|
||||
seen := make(map[string]struct{})
|
||||
out := make([]string, 0)
|
||||
for _, group := range groups {
|
||||
for _, raw := range group {
|
||||
host := strings.ToLower(strings.TrimSpace(raw))
|
||||
if host == "" {
|
||||
continue
|
||||
}
|
||||
if h, _, err := net.SplitHostPort(host); err == nil {
|
||||
host = strings.ToLower(strings.TrimSpace(h))
|
||||
}
|
||||
if host == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[host]; ok {
|
||||
continue
|
||||
}
|
||||
seen[host] = struct{}{}
|
||||
out = append(out, host)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// imageMintingSettingKeys 是「其取值会被拼进图片 URL」的设置项。
|
||||
//
|
||||
// 只收录这类设置:放行名单每多一条就是多一处 SSRF 例外,因此 app.server_url、
|
||||
// strm.base_url 这些只出现在播放地址里的设置不在此列(它们由 streamURL 使用,
|
||||
// 与 /api/img 无关)。
|
||||
var imageMintingSettingKeys = []string{
|
||||
"adult.scraper.metatube_server",
|
||||
"adult.scraper.builtin_javdb_url",
|
||||
"adult.scraper.builtin_javbus_url",
|
||||
}
|
||||
|
||||
// imageMintingAPIProviders 是「其 base_url 会被拼进图片 URL」的 api_configs 行。
|
||||
//
|
||||
// 同样刻意收窄:api_configs 里每个 provider 都有一行出厂默认值,其中 metatube
|
||||
// 默认指向 http://127.0.0.1:7700。把整张表都放行会让每次安装都无条件放开回环
|
||||
// 地址,而只有真正用 MetaTube 刮削的部署才需要这个例外。
|
||||
var imageMintingAPIProviders = []string{"metatube"}
|
||||
|
||||
// configuredServiceImageHostsFromRepo 从持久化配置里收集「会被写进图片 URL」的
|
||||
// 内网服务地址。只在构造阶段调用一次,所以直接读库、忽略错误——读不到只是少
|
||||
// 一条放行,不影响启动。
|
||||
func configuredServiceImageHostsFromRepo(ctx context.Context, repos *repository.Container) []string {
|
||||
if repos == nil {
|
||||
return nil
|
||||
}
|
||||
values := make([]string, 0, 4)
|
||||
|
||||
if repos.Setting != nil {
|
||||
for _, key := range imageMintingSettingKeys {
|
||||
if val, err := repos.Setting.Get(ctx, key); err == nil && strings.TrimSpace(val) != "" {
|
||||
values = append(values, val)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if repos.ApiConfig != nil {
|
||||
// 用 List 而不是 FindByProvider:后者走 First(),缺行时 gorm 会打一条
|
||||
// "record not found" 日志,而「没配置 MetaTube」是完全正常的状态。
|
||||
if rows, err := repos.ApiConfig.List(ctx); err == nil {
|
||||
for _, row := range rows {
|
||||
if !imageMintingAPIProvider(row.Provider) {
|
||||
continue
|
||||
}
|
||||
if strings.TrimSpace(row.BaseURL) != "" {
|
||||
values = append(values, row.BaseURL)
|
||||
}
|
||||
// Extra 是逗号/分号分隔的备用源列表(见 api_config.go 的 adult 默认值)。
|
||||
values = append(values, adultConfiguredBases(row.Extra)...)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return internalServiceImageHosts(values...)
|
||||
}
|
||||
|
||||
func imageMintingAPIProvider(provider string) bool {
|
||||
provider = strings.ToLower(strings.TrimSpace(provider))
|
||||
for _, allowed := range imageMintingAPIProviders {
|
||||
if provider == allowed {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -0,0 +1,249 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"github.com/glebarez/sqlite"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
"github.com/truewhile/MeBox/internal/repository"
|
||||
)
|
||||
|
||||
// 现场(2026-10-16):13 条媒体记录的 poster_url 指向
|
||||
// http://mebox-metatube-1:8081/v1/images/primary/... ,而图片代理只放行
|
||||
// 远程 Emby 挂载主机。MetaTube 是用户显式配置的刮削后端(settings 里的
|
||||
// adult.scraper.metatube_server),它跑在 docker 私网里,于是每一次封面
|
||||
// 回源都被连接层 SSRF 校验拒掉,日志刷 warn,前端长期只拿到占位图。
|
||||
//
|
||||
// 结论:显式配置的服务地址必须和远程 Emby 一样被加入放行名单。
|
||||
func TestConfiguredServiceImageHostsExtractsHostnames(t *testing.T) {
|
||||
got := configuredServiceImageHosts(
|
||||
"http://mebox-metatube-1:8081",
|
||||
"https://api.themoviedb.org/3",
|
||||
"http://192.168.1.50:8096/emby",
|
||||
)
|
||||
want := []string{"192.168.1.50", "api.themoviedb.org", "mebox-metatube-1"}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("hosts = %v, want %v", got, want)
|
||||
}
|
||||
for i := range want {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("hosts = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfiguredServiceImageHostsNormalizesAndDedupes(t *testing.T) {
|
||||
got := configuredServiceImageHosts(
|
||||
" HTTP://MeBox-MetaTube-1:8081 ",
|
||||
"http://mebox-metatube-1:8081/other/path",
|
||||
"mebox-metatube-1:8081",
|
||||
)
|
||||
if len(got) != 1 || got[0] != "mebox-metatube-1" {
|
||||
t.Fatalf("hosts = %v, want exactly [mebox-metatube-1]", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfiguredServiceImageHostsSkipsUnusableValues(t *testing.T) {
|
||||
got := configuredServiceImageHosts(
|
||||
"",
|
||||
" ",
|
||||
"not a url at all",
|
||||
"http://",
|
||||
"://missing-scheme",
|
||||
"/relative/path",
|
||||
"ftp://files.example.com",
|
||||
)
|
||||
if len(got) != 0 {
|
||||
t.Fatalf("hosts = %v, want none", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfiguredServiceImageHostsPreservesIPv6(t *testing.T) {
|
||||
got := configuredServiceImageHosts("http://[fd00::5]:8081")
|
||||
if len(got) != 1 || got[0] != "fd00::5" {
|
||||
t.Fatalf("hosts = %v, want [fd00::5]", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImageProxyAllowlistCombinesMountsAndConfiguredServices(t *testing.T) {
|
||||
// 回归:合并后的名单必须同时覆盖远程 Emby 挂载与显式配置的服务,
|
||||
// 否则两组图片里必有一组被 SSRF 校验挡住。
|
||||
proxy := newUpstreamImageProxy(t)
|
||||
proxy.SetAllowedRemoteHostsProvider(func() []string {
|
||||
return mergeImageProxyAllowedHosts(
|
||||
[]string{"192.168.1.50"},
|
||||
configuredServiceImageHosts("http://mebox-metatube-1:8081"),
|
||||
)
|
||||
})
|
||||
|
||||
for _, host := range []string{"192.168.1.50", "mebox-metatube-1"} {
|
||||
if !proxy.isAllowedRemoteHost(host) {
|
||||
t.Fatalf("host %q must bypass the private-IP guard", host)
|
||||
}
|
||||
}
|
||||
if proxy.isAllowedRemoteHost("evil.example.com") {
|
||||
t.Fatal("unrelated hosts must stay blocked")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeImageProxyAllowedHostsDedupesCaseInsensitively(t *testing.T) {
|
||||
got := mergeImageProxyAllowedHosts(
|
||||
[]string{"192.168.1.50:8096", "MeBox-MetaTube-1"},
|
||||
[]string{"mebox-metatube-1", "", " ", "192.168.1.50"},
|
||||
)
|
||||
sort.Strings(got)
|
||||
want := []string{"192.168.1.50", "mebox-metatube-1"}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("merged = %v, want %v", got, want)
|
||||
}
|
||||
for i := range want {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("merged = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 公网上游必须留在名单之外:放行后图片代理会优先走直连,而 GFW 环境恰恰要靠
|
||||
// HTTP(S)_PROXY 才能取到 image.tmdb.org 这类海报。只有内网地址才需要豁免。
|
||||
func TestInternalServiceImageHostsKeepsOnlyInternalAddresses(t *testing.T) {
|
||||
got := internalServiceImageHosts(
|
||||
"http://mebox-metatube-1:8081",
|
||||
"http://127.0.0.1:7700",
|
||||
"http://192.168.1.50:8096",
|
||||
"https://api.themoviedb.org/3",
|
||||
"https://image.tmdb.org",
|
||||
"https://api.bgm.tv",
|
||||
"https://javdb.com",
|
||||
"http://nas.local:7700",
|
||||
"http://media.internal",
|
||||
)
|
||||
want := []string{
|
||||
"127.0.0.1",
|
||||
"192.168.1.50",
|
||||
"mebox-metatube-1",
|
||||
"media.internal",
|
||||
"nas.local",
|
||||
}
|
||||
sort.Strings(want)
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("hosts = %v, want %v", got, want)
|
||||
}
|
||||
for i := range want {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("hosts = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsInternalServiceHost(t *testing.T) {
|
||||
internal := []string{
|
||||
"mebox-metatube-1", "127.0.0.1", "10.1.2.3", "172.18.0.5",
|
||||
"192.168.1.50", "fd00::5", "nas.local", "media.internal",
|
||||
"box.lan", "db.home.arpa",
|
||||
}
|
||||
for _, host := range internal {
|
||||
if !isInternalServiceHost(host) {
|
||||
t.Errorf("isInternalServiceHost(%q) = false, want true", host)
|
||||
}
|
||||
}
|
||||
external := []string{
|
||||
"", "api.themoviedb.org", "image.tmdb.org", "www.javbus.com",
|
||||
"pics.dmm.co.jp", "8.8.8.8", "example.com",
|
||||
}
|
||||
for _, host := range external {
|
||||
if isInternalServiceHost(host) {
|
||||
t.Errorf("isInternalServiceHost(%q) = true, want false", host)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 端到端装配:从真实的 settings / api_configs 里读出内网服务地址。
|
||||
// 现场数据是 settings.adult.scraper.metatube_server = http://mebox-metatube-1:8081,
|
||||
// 而 api_configs.metatube.base_url 还是出厂默认的 http://127.0.0.1:7700。
|
||||
func TestConfiguredServiceImageHostsFromRepoReadsSettingsAndAPIConfigs(t *testing.T) {
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to open sqlite: %v", err)
|
||||
}
|
||||
if err := db.AutoMigrate(&model.Setting{}, &model.APIConfig{}); err != nil {
|
||||
t.Fatalf("migrate: %v", err)
|
||||
}
|
||||
repos := repository.New(db)
|
||||
ctx := context.Background()
|
||||
|
||||
if err := repos.Setting.Set(ctx, "adult.scraper.metatube_server", "http://mebox-metatube-1:8081"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := repos.Setting.Set(ctx, "adult.scraper.builtin_javdb_url", "https://javdb.com"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, row := range []model.APIConfig{
|
||||
{Provider: "metatube", BaseURL: "http://127.0.0.1:7700", Enabled: true},
|
||||
{Provider: "tmdb", BaseURL: "https://api.themoviedb.org/3", Enabled: true},
|
||||
{Provider: "openai", BaseURL: "http://192.168.1.77:11434/v1", Enabled: true},
|
||||
} {
|
||||
if err := repos.ApiConfig.Create(ctx, &row); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
got := configuredServiceImageHostsFromRepo(ctx, repos)
|
||||
// javdb.com / api.themoviedb.org 是公网地址,必须留在名单之外(否则图片
|
||||
// 代理会改走直连,GFW 环境下反而取不到图);openai 的 base_url 虽然指向
|
||||
// 内网,但它从不产出图片 URL,同样不应成为 SSRF 例外。
|
||||
want := []string{"127.0.0.1", "mebox-metatube-1"}
|
||||
sort.Strings(want)
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("hosts = %v, want %v", got, want)
|
||||
}
|
||||
for i := range want {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("hosts = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 放行名单是 SSRF 例外,必须收窄:只有「会产出图片 URL」的配置才入选。
|
||||
func TestConfiguredServiceImageHostsFromRepoIgnoresUnrelatedInternalServices(t *testing.T) {
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to open sqlite: %v", err)
|
||||
}
|
||||
if err := db.AutoMigrate(&model.Setting{}, &model.APIConfig{}); err != nil {
|
||||
t.Fatalf("migrate: %v", err)
|
||||
}
|
||||
repos := repository.New(db)
|
||||
ctx := context.Background()
|
||||
|
||||
// 只配置了内网的 OpenAI 兼容网关与内网下载器,没有任何图片来源。
|
||||
if err := repos.Setting.Set(ctx, "app.server_url", "http://10.0.0.2:1314"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := repos.Setting.Set(ctx, "strm.base_url", "http://10.0.0.2:1314"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, row := range []model.APIConfig{
|
||||
{Provider: "openai", BaseURL: "http://192.168.1.77:11434/v1", Enabled: true},
|
||||
{Provider: "tmdb", BaseURL: "https://api.themoviedb.org/3", Enabled: true},
|
||||
} {
|
||||
if err := repos.ApiConfig.Create(ctx, &row); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
if got := configuredServiceImageHostsFromRepo(ctx, repos); len(got) != 0 {
|
||||
t.Fatalf("hosts = %v, want none: unrelated internal services must not become SSRF exceptions", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfiguredServiceImageHostsFromRepoToleratesMissingRepos(t *testing.T) {
|
||||
if got := configuredServiceImageHostsFromRepo(context.Background(), nil); len(got) != 0 {
|
||||
t.Fatalf("hosts = %v, want none", got)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/config"
|
||||
)
|
||||
|
||||
// hostAndPortOf 从测试服务器的 URL 里取出主机名与端口。
|
||||
func hostAndPortOf(t *testing.T, raw string) (string, string) {
|
||||
t.Helper()
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("parse upstream url: %v", err)
|
||||
}
|
||||
return u.Hostname(), u.Port()
|
||||
}
|
||||
|
||||
// 现场(2026-10-16,生产容器内实测):放行名单里明明有 mebox-metatube-1,
|
||||
// 回源仍然失败于
|
||||
//
|
||||
// image proxy: requests to private/internal hosts are not allowed
|
||||
//
|
||||
// 原因:连接层校验(dialer 的 Control 回调)拿到的 address 是**解析后的 IP**,
|
||||
// 而名单里存的是主机名(ConfiguredRemoteHosts 与
|
||||
// configuredServiceImageHostsFromRepo 都返回 u.Hostname()),两者永远不相等:
|
||||
// 那段 isAllowedRemoteHost 分支对主机名条目是死代码。
|
||||
//
|
||||
// 远程 Emby 挂载用 IP 配置时看不出问题,用主机名(NAS 名、docker 服务名)配置
|
||||
// 时同样中招。这个用例锁住「按主机名放行」这条路径。
|
||||
func TestImageProxyDialsAllowlistedHostname(t *testing.T) {
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "image/jpeg")
|
||||
_, _ = w.Write(testJPEG)
|
||||
}))
|
||||
defer upstream.Close()
|
||||
|
||||
proxy := NewImageProxy(
|
||||
&config.Config{Cache: config.CacheConfig{CacheDir: filepath.Join(t.TempDir(), "cache")}},
|
||||
zap.NewNop(),
|
||||
)
|
||||
// 名单里放「主机名」而不是 IP —— 这正是生产里 mebox-metatube-1 的形态。
|
||||
proxy.SetAllowedRemoteHostsProvider(func() []string { return []string{"localhost"} })
|
||||
|
||||
// 直接用该代理自己的 transport 拨号,才能命中连接层校验。
|
||||
_, port := hostAndPortOf(t, upstream.URL)
|
||||
resp, err := proxy.client.Get("http://localhost:" + port + "/poster.jpg")
|
||||
if err != nil {
|
||||
t.Fatalf("allowlisted hostname must be dialable, got: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
// 反向约束:未放行的主机名解析到私网地址时仍必须被拒。
|
||||
// 这是连接层校验存在的理由(DNS rebinding),不能为了修上面那条而拆掉。
|
||||
func TestImageProxyStillBlocksUnlistedHostnameResolvingToPrivateIP(t *testing.T) {
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write(testJPEG)
|
||||
}))
|
||||
defer upstream.Close()
|
||||
|
||||
proxy := NewImageProxy(
|
||||
&config.Config{Cache: config.CacheConfig{CacheDir: filepath.Join(t.TempDir(), "cache")}},
|
||||
zap.NewNop(),
|
||||
)
|
||||
// 名单为空:localhost 解析到回环地址,必须被拦。
|
||||
proxy.SetAllowedRemoteHostsProvider(func() []string { return nil })
|
||||
|
||||
_, port := hostAndPortOf(t, upstream.URL)
|
||||
resp, err := proxy.client.Get("http://localhost:" + port + "/poster.jpg")
|
||||
if err == nil {
|
||||
resp.Body.Close()
|
||||
t.Fatal("a non-allowlisted hostname resolving to a private IP must be refused")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
// 在真实 docker 网络里验证图片代理的私网放行名单修复。
|
||||
//
|
||||
// 现场(修复前):MetaTube 刮削结果里的 poster_url 指向
|
||||
// http://mebox-metatube-1:8081/... ,而放行名单只含远程 Emby 挂载主机,
|
||||
// 于是每一次封面回源都被连接层 SSRF 校验拒掉,前端长期只拿到占位图。
|
||||
//
|
||||
// 本文件用与 service_builder.initImageProxy 相同的装配方式复现那条路径。
|
||||
// 真实回源那一步只在能解析到 MetaTube 服务名的环境里跑(生产容器网络);
|
||||
// 开发机上会 Skip,因为那属于环境缺失,不是回归。
|
||||
//
|
||||
// 在生产机上运行的姿势(无需 Go 工具链):
|
||||
//
|
||||
// GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go test -c -o /tmp/verify.test ./internal/service
|
||||
// scp /tmp/verify.test root@HOST:/tmp/ && ssh root@HOST \
|
||||
// 'docker cp /tmp/verify.test mebox:/tmp/ && docker exec mebox /tmp/verify.test \
|
||||
// -test.run TestImageProxyFetchesFromMetaTubeService -test.v'
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/glebarez/sqlite"
|
||||
"go.uber.org/zap"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/config"
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
"github.com/truewhile/MeBox/internal/repository"
|
||||
)
|
||||
|
||||
// 与生产环境一致的两个取值(取自 10.0.0.2 上真实的 settings / api_configs)。
|
||||
const (
|
||||
metaTubeServer = "http://mebox-metatube-1:8081"
|
||||
metaTubeLegacy = "http://127.0.0.1:7700"
|
||||
sampleImageURL = "http://mebox-metatube-1:8081/v1/images/primary/JAV321/dsam-15"
|
||||
)
|
||||
|
||||
// newMetaTubeVerificationRepos 建一个带现场取值的临时库(不碰生产数据)。
|
||||
func newMetaTubeVerificationRepos(t *testing.T) (*repository.Container, *ImageProxy) {
|
||||
t.Helper()
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatalf("open sqlite: %v", err)
|
||||
}
|
||||
if err := db.AutoMigrate(&model.Setting{}, &model.APIConfig{}); err != nil {
|
||||
t.Fatalf("migrate: %v", err)
|
||||
}
|
||||
repos := repository.New(db)
|
||||
ctx := context.Background()
|
||||
if err := repos.Setting.Set(ctx, "adult.scraper.metatube_server", metaTubeServer); err != nil {
|
||||
t.Fatalf("seed setting: %v", err)
|
||||
}
|
||||
if err := repos.ApiConfig.Create(ctx, &model.APIConfig{
|
||||
Provider: "metatube", BaseURL: metaTubeLegacy, Enabled: true,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed metatube api config: %v", err)
|
||||
}
|
||||
// 一个公网上游,用来确认它没有混进放行名单。
|
||||
if err := repos.ApiConfig.Create(ctx, &model.APIConfig{
|
||||
Provider: "tmdb", BaseURL: "https://api.themoviedb.org/3", Enabled: true,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed tmdb api config: %v", err)
|
||||
}
|
||||
|
||||
cfg := &config.Config{Cache: config.CacheConfig{CacheDir: filepath.Join(t.TempDir(), "cache")}}
|
||||
cfg.Secrets.JWTSecret = "verify-secret"
|
||||
proxy := NewImageProxy(cfg, zap.NewNop())
|
||||
// 与 service_builder.initImageProxy 相同的装配方式(无远程 Emby 挂载:
|
||||
// 本次只验证「显式配置的服务地址」这条路径)。
|
||||
proxy.SetAllowedRemoteHostsProvider(func() []string {
|
||||
return mergeImageProxyAllowedHosts(
|
||||
nil,
|
||||
configuredServiceImageHostsFromRepo(ctx, repos),
|
||||
)
|
||||
})
|
||||
return repos, proxy
|
||||
}
|
||||
|
||||
// TestImageProxyAllowlistIncludesConfiguredMetaTube 确认内网服务地址被放行、
|
||||
// 公网上游没有被放行。
|
||||
func TestImageProxyAllowlistIncludesConfiguredMetaTube(t *testing.T) {
|
||||
repos, proxy := newMetaTubeVerificationRepos(t)
|
||||
t.Logf("allowlist from config: %v",
|
||||
configuredServiceImageHostsFromRepo(context.Background(), repos))
|
||||
|
||||
for _, host := range []string{"mebox-metatube-1", "127.0.0.1"} {
|
||||
if !proxy.isAllowedRemoteHost(host) {
|
||||
t.Fatalf("host %q must be allowlisted (it is an explicitly configured service)", host)
|
||||
}
|
||||
}
|
||||
if proxy.isAllowedRemoteHost("api.themoviedb.org") {
|
||||
t.Fatal("public upstreams must stay off the allowlist so image fetches keep using the proxy")
|
||||
}
|
||||
}
|
||||
|
||||
// TestImageProxyReachesMetaTubeService 证明放行名单真的解除了私网拦截。
|
||||
//
|
||||
// 现场(修复前)回源必然失败于
|
||||
//
|
||||
// dial tcp 172.18.0.5:8081: image proxy: requests to private/internal hosts are not allowed
|
||||
//
|
||||
// 也就是连接根本没发生。修复后连接成功,失败只可能来自上游自己的 HTTP 状态
|
||||
// (MetaTube 冷缓存时对未刮削过的番号返回 500)。
|
||||
//
|
||||
// 因此这个用例断言的不是「拿到 200」,而是**「不再是被私网校验拒绝」**:
|
||||
// 后者才是本次修复的对象,前者取决于 MetaTube 缓存里有没有那条记录。
|
||||
func TestImageProxyReachesMetaTubeService(t *testing.T) {
|
||||
_, proxy := newMetaTubeVerificationRepos(t)
|
||||
|
||||
data, ctype, err := proxy.Fetch(context.Background(), sampleImageURL)
|
||||
if err != nil {
|
||||
if isMetaTubeUnreachable(err) {
|
||||
t.Skipf("mebox-metatube-1 is not reachable from this environment: %v", err)
|
||||
}
|
||||
if isPrivateHostRejection(err) {
|
||||
t.Fatalf("the allowlist did not take effect: %v", err)
|
||||
}
|
||||
// 连接已经建立,只是上游自己返回了错误状态(MetaTube 冷缓存)。
|
||||
t.Logf("reached metaTube; upstream returned its own error: %v", err)
|
||||
return
|
||||
}
|
||||
t.Logf("fetched %d bytes (content-type=%s)", len(data), ctype)
|
||||
if len(data) == 0 {
|
||||
t.Fatal("fetched an empty image body")
|
||||
}
|
||||
}
|
||||
|
||||
// isPrivateHostRejection 报告错误是否来自连接层的私网校验。
|
||||
// 这正是本次修复必须消除的那一类失败。
|
||||
func isPrivateHostRejection(err error) bool {
|
||||
return err != nil && strings.Contains(err.Error(), "requests to private/internal hosts are not allowed")
|
||||
}
|
||||
|
||||
// isMetaTubeUnreachable 区分「环境里没有这个服务」与「放行名单没生效」。
|
||||
func isMetaTubeUnreachable(err error) bool {
|
||||
var dnsErr *net.DNSError
|
||||
if errors.As(err, &dnsErr) {
|
||||
return true
|
||||
}
|
||||
msg := strings.ToLower(err.Error())
|
||||
for _, needle := range []string{"no such host", "server misbehaving", "i/o timeout", "connection refused"} {
|
||||
if strings.Contains(msg, needle) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -14,6 +14,10 @@ import (
|
||||
var errImageProxyRequestSetup = errors.New("image proxy request setup failed")
|
||||
var errImageProxyNonImageContent = errors.New("upstream returned non-image content")
|
||||
|
||||
// errImageProxyUnauthorized 表示上游以 401/403 拒绝了回源凭据。它与普通失败
|
||||
// 分开,是为了让调用方对挂载 Emby 做一次重认证再重试,而不是直接下发占位图。
|
||||
var errImageProxyUnauthorized = errors.New("upstream rejected image credentials")
|
||||
|
||||
// prefetchCardResizeOptions 对应前端 ARTWORK.posterCard(见 web/src/api/client.ts):
|
||||
// 卡片是海报墙最常请求的档位,刮削阶段预生成它能让首个列表请求直接命中缓存。
|
||||
// 若前端调整该预设,这里只是白生成一份用不到的档位(约几十 KB),不影响正确性。
|
||||
@@ -183,16 +187,27 @@ func (p *ImageProxy) removeUnusableImageCache(cachePath, failPath string) {
|
||||
}
|
||||
|
||||
func (p *ImageProxy) fetchAndCacheRemoteImage(ctx context.Context, raw, host, cachePath, failPath string) (remoteImageFetchResult, error) {
|
||||
var lastErr error
|
||||
for _, candidate := range p.remoteImageFetchClients(host) {
|
||||
result, err := p.fetchRemoteImageOnce(ctx, raw, host, candidate, cachePath, failPath)
|
||||
if err == nil {
|
||||
return result, nil
|
||||
result, lastErr := p.fetchRemoteImageAcrossClients(ctx, raw, host, cachePath, failPath)
|
||||
if lastErr == nil {
|
||||
return result, nil
|
||||
}
|
||||
if errors.Is(lastErr, errImageProxyRequestSetup) {
|
||||
return remoteImageFetchResult{}, lastErr
|
||||
}
|
||||
// 已挂载的远程 Emby 认 X-Emby-Token 请求头,但轮换前生成的图片 URL 里还留着
|
||||
// 旧 api_key。上游拒绝时重新登录拥有该主机的账号一次再重试,而不是一直下发
|
||||
// 占位图、等人工去改账号配置。
|
||||
if errors.Is(lastErr, errImageProxyUnauthorized) {
|
||||
token, refreshErr := p.refreshRemoteEmbyTokenForHost(ctx, host)
|
||||
if refreshErr != nil {
|
||||
logImageFetchError(p.log, "imageproxy: remote emby re-auth failed", host, "reauth", refreshErr)
|
||||
} else if token != "" {
|
||||
retried, retryErr := p.fetchRemoteImageAcrossClients(ctx, raw, host, cachePath, failPath)
|
||||
if retryErr == nil {
|
||||
return retried, nil
|
||||
}
|
||||
lastErr = retryErr
|
||||
}
|
||||
if errors.Is(err, errImageProxyRequestSetup) {
|
||||
return remoteImageFetchResult{}, err
|
||||
}
|
||||
lastErr = err
|
||||
}
|
||||
if p.canUseExternalImageFallback() && isDoubanImageHost(host) {
|
||||
data, ctype, _, err := fetchRemoteImageWithCurl(ctx, raw, host)
|
||||
@@ -210,6 +225,24 @@ func (p *ImageProxy) fetchAndCacheRemoteImage(ctx context.Context, raw, host, ca
|
||||
return remoteImageFetchResult{}, redactSensitiveError(lastErr)
|
||||
}
|
||||
|
||||
// fetchRemoteImageAcrossClients 按直连/代理顺序依次尝试,返回首个成功结果。
|
||||
// 请求构造失败立即中止(重试无意义);401/403 会继续尝试另一个客户端,并把
|
||||
// errImageProxyUnauthorized 作为最终错误交给调用方去刷新挂载 token。
|
||||
func (p *ImageProxy) fetchRemoteImageAcrossClients(ctx context.Context, raw, host, cachePath, failPath string) (remoteImageFetchResult, error) {
|
||||
var lastErr error
|
||||
for _, candidate := range p.remoteImageFetchClients(host) {
|
||||
result, err := p.fetchRemoteImageOnce(ctx, raw, host, candidate, cachePath, failPath)
|
||||
if err == nil {
|
||||
return result, nil
|
||||
}
|
||||
if errors.Is(err, errImageProxyRequestSetup) {
|
||||
return remoteImageFetchResult{}, err
|
||||
}
|
||||
lastErr = err
|
||||
}
|
||||
return remoteImageFetchResult{}, lastErr
|
||||
}
|
||||
|
||||
// fetchAndCacheRemoteImageShared coalesces concurrent requests for the same
|
||||
// upstream image. A poster can appear in the hero, a shelf and the detail page
|
||||
// at the same time; without this guard every resize variant may fetch the same
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// remoteEmbyImageTokenTTL 是 host → api_key 的缓存时长。海报墙首屏一口气请求
|
||||
// 几十张图,逐个回读数据库不值得;token 轮换时 401 重认证会立即覆盖这份缓存。
|
||||
const remoteEmbyImageTokenTTL = 60 * time.Second
|
||||
|
||||
type remoteEmbyImageToken struct {
|
||||
token string
|
||||
at time.Time
|
||||
}
|
||||
|
||||
// SetRemoteEmbyAuthProvider 注入「按主机名取当前远程 Emby api_key」的回调。
|
||||
// 图片代理回源已挂载的远程 Emby 时用它替换 URL 里可能已过期的凭据。
|
||||
func (p *ImageProxy) SetRemoteEmbyAuthProvider(fn func(ctx context.Context, host string) (string, bool)) {
|
||||
if p == nil {
|
||||
return
|
||||
}
|
||||
p.remoteEmbyTokenFn = fn
|
||||
}
|
||||
|
||||
// SetRemoteEmbyAuthRefresher 注入「强制重新登录并返回新 api_key」的回调。上游以
|
||||
// 401/403 拒绝旧 token 时调用一次,再重试拉图,从而不需要人工改账号配置。
|
||||
func (p *ImageProxy) SetRemoteEmbyAuthRefresher(fn func(ctx context.Context, host string) (string, error)) {
|
||||
if p == nil {
|
||||
return
|
||||
}
|
||||
p.refreshRemoteEmbyTokenFn = fn
|
||||
}
|
||||
|
||||
// remoteEmbyTokenForHost 返回挂载 Emby 的当前 token,短期缓存避免每张图都查库。
|
||||
// 「不是挂载主机」的结果同样缓存,否则普通图床(TMDb/Douban 等)的每张海报都会
|
||||
// 白白查一次账号表。
|
||||
func (p *ImageProxy) remoteEmbyTokenForHost(ctx context.Context, host string) string {
|
||||
if p == nil || p.remoteEmbyTokenFn == nil || strings.TrimSpace(host) == "" {
|
||||
return ""
|
||||
}
|
||||
p.remoteEmbyTokenMu.Lock()
|
||||
if cached, ok := p.remoteEmbyTokenCache[host]; ok && time.Since(cached.at) < remoteEmbyImageTokenTTL {
|
||||
p.remoteEmbyTokenMu.Unlock()
|
||||
return cached.token
|
||||
}
|
||||
p.remoteEmbyTokenMu.Unlock()
|
||||
|
||||
token, ok := p.remoteEmbyTokenFn(ctx, host)
|
||||
if !ok {
|
||||
token = ""
|
||||
}
|
||||
p.cacheRemoteEmbyToken(host, token)
|
||||
return token
|
||||
}
|
||||
|
||||
func (p *ImageProxy) cacheRemoteEmbyToken(host, token string) {
|
||||
if p == nil || strings.TrimSpace(host) == "" {
|
||||
return
|
||||
}
|
||||
p.remoteEmbyTokenMu.Lock()
|
||||
if p.remoteEmbyTokenCache == nil {
|
||||
p.remoteEmbyTokenCache = make(map[string]remoteEmbyImageToken, 8)
|
||||
}
|
||||
p.remoteEmbyTokenCache[host] = remoteEmbyImageToken{token: token, at: time.Now()}
|
||||
p.remoteEmbyTokenMu.Unlock()
|
||||
}
|
||||
|
||||
// refreshRemoteEmbyTokenForHost 强制重认证一次并刷新缓存。没有配置刷新器、或
|
||||
// 账号无法自动刷新(如仅填了 api_key)时返回空串,调用方按原错误处理。
|
||||
func (p *ImageProxy) refreshRemoteEmbyTokenForHost(ctx context.Context, host string) (string, error) {
|
||||
if p == nil || p.refreshRemoteEmbyTokenFn == nil || strings.TrimSpace(host) == "" {
|
||||
return "", nil
|
||||
}
|
||||
token, err := p.refreshRemoteEmbyTokenFn(ctx, host)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if token != "" {
|
||||
p.cacheRemoteEmbyToken(host, token)
|
||||
}
|
||||
return token, nil
|
||||
}
|
||||
|
||||
// applyRemoteEmbyAuth 用账号当前 token 覆盖回源请求上的凭据。已配置的远程 Emby
|
||||
// 以 X-Emby-Token 请求头为准,所以图片 URL 里带的是轮换前的旧 api_key 也不会再
|
||||
// 被采纳;同时把查询参数里的 api_key 一并改写,避免旧值干扰。
|
||||
func (p *ImageProxy) applyRemoteEmbyAuth(ctx context.Context, req *http.Request, host string) {
|
||||
if p == nil || p.remoteEmbyTokenFn == nil || req == nil || req.URL == nil {
|
||||
return
|
||||
}
|
||||
token := p.remoteEmbyTokenForHost(ctx, host)
|
||||
if token == "" {
|
||||
return
|
||||
}
|
||||
req.Header.Set("X-Emby-Token", token)
|
||||
q := req.URL.Query()
|
||||
q.Set("api_key", token)
|
||||
req.URL.RawQuery = q.Encode()
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestImageProxyInjectsRemoteEmbyToken 覆盖封面空白问题:图片 URL 里签发时的
|
||||
// api_key 已经失效,但账号当前 token 有效。代理回源必须用当前 token 覆盖旧值,
|
||||
// 否则挂载 Emby 的封面会一直 401、下发占位图。
|
||||
func TestImageProxyInjectsRemoteEmbyToken(t *testing.T) {
|
||||
const current = "current-token"
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("X-Emby-Token") != current {
|
||||
http.Error(w, "unauthorized", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "image/jpeg")
|
||||
_, _ = w.Write(testJPEG)
|
||||
}))
|
||||
defer upstream.Close()
|
||||
|
||||
u, err := url.Parse(upstream.URL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
proxy := newUpstreamImageProxy(t, u.Host)
|
||||
proxy.SetRemoteEmbyAuthProvider(func(context.Context, string) (string, bool) {
|
||||
return current, true
|
||||
})
|
||||
|
||||
raw := upstream.URL + "/emby/Items/abc/Images/Primary?api_key=stale-token"
|
||||
rec := httptest.NewRecorder()
|
||||
if err := proxy.Serve(t.Context(), rec, httptest.NewRequest(http.MethodGet, "/api/img", nil), raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(rec.Body.Bytes(), testJPEG) {
|
||||
t.Fatalf("body = %x, want the upstream image served with the current token", rec.Body.Bytes())
|
||||
}
|
||||
}
|
||||
|
||||
// TestImageProxyRefreshesRemoteEmbyTokenOn401 覆盖 token 在运行中被撤销的场景:
|
||||
// 账号当前 token 也被上游拒绝时,代理应重新登录一次并重试,且只刷新一次。
|
||||
func TestImageProxyRefreshesRemoteEmbyTokenOn401(t *testing.T) {
|
||||
const (
|
||||
stale = "stale-token"
|
||||
fresh = "fresh-token"
|
||||
)
|
||||
var mu sync.Mutex
|
||||
refreshes := 0
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("X-Emby-Token") != fresh {
|
||||
http.Error(w, "unauthorized", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "image/jpeg")
|
||||
_, _ = w.Write(testJPEG)
|
||||
}))
|
||||
defer upstream.Close()
|
||||
|
||||
u, err := url.Parse(upstream.URL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
proxy := newUpstreamImageProxy(t, u.Host)
|
||||
proxy.SetRemoteEmbyAuthProvider(func(context.Context, string) (string, bool) {
|
||||
return stale, true
|
||||
})
|
||||
proxy.SetRemoteEmbyAuthRefresher(func(context.Context, string) (string, error) {
|
||||
mu.Lock()
|
||||
refreshes++
|
||||
mu.Unlock()
|
||||
return fresh, nil
|
||||
})
|
||||
|
||||
raw := upstream.URL + "/emby/Items/abc/Images/Primary?api_key=" + stale
|
||||
rec := httptest.NewRecorder()
|
||||
if err := proxy.Serve(t.Context(), rec, httptest.NewRequest(http.MethodGet, "/api/img", nil), raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(rec.Body.Bytes(), testJPEG) {
|
||||
t.Fatalf("body = %x, want the upstream image served after re-auth", rec.Body.Bytes())
|
||||
}
|
||||
|
||||
mu.Lock()
|
||||
got := refreshes
|
||||
mu.Unlock()
|
||||
if got != 1 {
|
||||
t.Fatalf("refreshes = %d, want exactly 1 re-auth", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestImageProxyLeavesNonEmbyHostsUntouched 确认新的凭据注入只作用于已配置的
|
||||
// 远程 Emby 主机:普通图床(这里用 provider 返回 ok=false 模拟)不受影响。
|
||||
func TestImageProxyDoesNotInjectForUnknownHost(t *testing.T) {
|
||||
var gotHeader string
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotHeader = r.Header.Get("X-Emby-Token")
|
||||
w.Header().Set("Content-Type", "image/jpeg")
|
||||
_, _ = w.Write(testJPEG)
|
||||
}))
|
||||
defer upstream.Close()
|
||||
|
||||
u, err := url.Parse(upstream.URL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
proxy := newUpstreamImageProxy(t, u.Host)
|
||||
proxy.SetRemoteEmbyAuthProvider(func(context.Context, string) (string, bool) {
|
||||
return "", false
|
||||
})
|
||||
|
||||
raw := upstream.URL + "/img/cover.jpg"
|
||||
rec := httptest.NewRecorder()
|
||||
if err := proxy.Serve(t.Context(), rec, httptest.NewRequest(http.MethodGet, "/api/img", nil), raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if gotHeader != "" {
|
||||
t.Fatalf("X-Emby-Token = %q, want no credential header for a non-Emby host", gotHeader)
|
||||
}
|
||||
}
|
||||
@@ -71,6 +71,8 @@ func (p *ImageProxy) fetchRemoteImageOnce(ctx context.Context, raw, host string,
|
||||
return remoteImageFetchResult{}, errImageProxyRequestSetup
|
||||
}
|
||||
applyRemoteImageHeaders(req, host, raw)
|
||||
// 已挂载的远程 Emby:用账号当前 token 覆盖旧凭据,返回头形式对端优先采纳。
|
||||
p.applyRemoteEmbyAuth(ctx, req, host)
|
||||
|
||||
resp, err := candidate.client.Do(req)
|
||||
if err != nil {
|
||||
@@ -80,6 +82,9 @@ func (p *ImageProxy) fetchRemoteImageOnce(ctx context.Context, raw, host string,
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode >= 400 {
|
||||
p.log.Warn("imageproxy: upstream returned non-OK", zap.String("host", host), zap.String("client", candidate.name), zap.String("status", resp.Status))
|
||||
if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden {
|
||||
return remoteImageFetchResult{}, errImageProxyUnauthorized
|
||||
}
|
||||
return remoteImageFetchResult{}, errors.New("upstream returned " + resp.Status)
|
||||
}
|
||||
if err := p.streamImageToCache(cachePath, failPath, resp.Body); err != nil {
|
||||
|
||||
@@ -50,9 +50,12 @@ func LookupLocalAvailability(ctx context.Context, repo *repository.Container, ti
|
||||
return out
|
||||
}
|
||||
like := "%" + query + "%"
|
||||
// 方言相关的 ILIKE/LIKE:PostgreSQL 下用 LIKE 会因标题大小写与查询不一致而
|
||||
// 漏判「本地已有该剧」,进而反复提示订阅缺失。
|
||||
likeOp := repository.LikeOperator(repo.DB)
|
||||
var rows []model.Media
|
||||
if err := repo.DB.WithContext(ctx).
|
||||
Where("title LIKE ? OR original_name LIKE ? OR path LIKE ?", like, like, like).
|
||||
Where("title "+likeOp+" ? OR original_name "+likeOp+" ? OR path "+likeOp+" ?", like, like, like).
|
||||
Order("season_num asc, episode_num asc, episode_fraction asc, created_at desc").
|
||||
Limit(2000).
|
||||
Find(&rows).Error; err != nil {
|
||||
|
||||
@@ -39,7 +39,9 @@ const (
|
||||
// audioTranscodeDirName 缓存目录名,位于 cache.cache_dir 之下。
|
||||
audioTranscodeDirName = "reader-audio"
|
||||
// maxAudioTranscodeCacheBytes 转码缓存上限,超出按修改时间淘汰最旧的。
|
||||
maxAudioTranscodeCacheBytes = 4 << 30
|
||||
// 1GB 是权衡:单章语音 96kbps 约 1MB/分钟,一本 10 小时的有声书约 600MB,
|
||||
// 也就是说同时能保住的「最近听过的书」在个位数;超出后旧书再听会重新转码。
|
||||
maxAudioTranscodeCacheBytes = 1 << 30
|
||||
// audioTranscodeKeepRatio 触发淘汰后回落到上限的比例,避免每次写入都淘汰。
|
||||
audioTranscodeKeepRatio = 0.9
|
||||
// maxAudioTranscodeDuration 单章转码超时。
|
||||
@@ -269,7 +271,7 @@ func (s *ReaderService) transcodeInputHeaders(ctx context.Context, bookID, sourc
|
||||
if err != nil || book == nil || strings.TrimSpace(book.Origin) == "" {
|
||||
return headers
|
||||
}
|
||||
if src, err := s.repo.GetSourceByURL(ctx, book.Origin); err == nil && src != nil && src.Header != "" {
|
||||
if src, err := s.repo.GetSourceAnyByURL(ctx, book.Origin); err == nil && src != nil && src.Header != "" {
|
||||
var extra map[string]any
|
||||
if json.Unmarshal([]byte(src.Header), &extra) == nil {
|
||||
for k, v := range extra {
|
||||
|
||||
@@ -2,6 +2,7 @@ package reader
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -255,3 +256,50 @@ func indexOf(items []string, want string) int {
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// 转码缓存上限是 1GB:单章语音约 1MB/分钟(96kbps),
|
||||
// 也就是一本 10 小时的有声书约 600MB,同时能保住「最近听过的书」在个位数。
|
||||
// 这里把常量与淘汰后的落点一起钉住,防止以后被无意改回大值。
|
||||
func TestAudioTranscodeCacheLimitIsOneGiB(t *testing.T) {
|
||||
if maxAudioTranscodeCacheBytes != int64(1)<<30 {
|
||||
t.Fatalf("转码缓存上限应为 1GB,实际 %d 字节", maxAudioTranscodeCacheBytes)
|
||||
}
|
||||
// 淘汰目标是上限的 90%:模拟缓存略超上限时,删掉最旧的若干章回到目标以下。
|
||||
dir := t.TempDir()
|
||||
limit := int64(1000)
|
||||
perFile := int64(120)
|
||||
for i := 0; i < 9; i++ { // 1080 > 1000,超限
|
||||
name := filepath.Join(dir, fmt.Sprintf("ch%02d.mp3", i))
|
||||
if err := os.WriteFile(name, make([]byte, perFile), 0o640); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ts := time.Now().Add(-time.Duration(9-i) * time.Minute)
|
||||
if err := os.Chtimes(name, ts, ts); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
pruneAudioTranscodeCache(dir, limit)
|
||||
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var remain int64
|
||||
for _, e := range entries {
|
||||
info, err := e.Info()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
remain += info.Size()
|
||||
}
|
||||
if remain > limit {
|
||||
t.Fatalf("淘汰后应落回上限内: remain=%d limit=%d", remain, limit)
|
||||
}
|
||||
// 最新的那章必须还在(LRU 先删最旧的)。
|
||||
if _, err := os.Stat(filepath.Join(dir, "ch08.mp3")); err != nil {
|
||||
t.Fatalf("最新一章应保留: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "ch00.mp3")); !os.IsNotExist(err) {
|
||||
t.Fatalf("最旧一章应被删除,err=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,239 @@
|
||||
package reader
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// 本文件:段落评论接口(/para_review)的分页适配。
|
||||
//
|
||||
// 上游聚合站自带的评论页按 page=N 翻页(滑到底请求 page=2、page=3…),但它的列表
|
||||
// 接口早就改成了游标分页:响应里给 next_cursor,请求时要带 cursor=。接口不认识
|
||||
// page,于是每次「加载更多」都原样返回第一页——一条新评论都加不进去,has_more 又
|
||||
// 始终是 true,评论页就一遍遍触发加载更多、反复闪骨架屏。用户看到的就是「段评往
|
||||
// 下滑一闪一闪」,其实评论一条没多。
|
||||
//
|
||||
// 评论页是上游 HTML,改不了,于是在代理层做最小适配:
|
||||
// - 请求带 page=N(N>1)且有上一页给的游标时,换成 cursor= 请求;
|
||||
// - 某页返回的 comment_id 全是已经见过的,就把 has_more 改成 false 收尾,
|
||||
// 免得页面在「加载更多 → 没有新内容」之间空转。
|
||||
//
|
||||
// 只认路径以 /para_review 结尾、带 page 且不带 cursor 的请求;对不上就原样透传,
|
||||
// 不影响别的书源。
|
||||
|
||||
// commentPageState 一个评论列表(同一段落 + 同一排序)的翻页状态。
|
||||
type commentPageState struct {
|
||||
// cursor 上一页响应给的游标:下一页请求用它代替 page。
|
||||
cursor string
|
||||
// seen 已经下发给页面的 comment_id:整页重复时据此收尾。
|
||||
seen map[string]struct{}
|
||||
}
|
||||
|
||||
// commentPager 承载页面里评论接口的翻页适配器(每个承载页面一份)。
|
||||
type commentPager struct {
|
||||
mu sync.Mutex
|
||||
states map[string]*commentPageState
|
||||
}
|
||||
|
||||
func newCommentPager() *commentPager {
|
||||
return &commentPager{states: map[string]*commentPageState{}}
|
||||
}
|
||||
|
||||
// rewriteRequest 按需把 page=N 改写成 cursor=<上一页游标>。
|
||||
//
|
||||
// 返回改写后的地址与状态键(空串表示这个请求不需要适配,调用方也不必观察响应)。
|
||||
func (p *commentPager) rewriteRequest(target string) (string, string) {
|
||||
if p == nil {
|
||||
return target, ""
|
||||
}
|
||||
parsed, err := url.Parse(target)
|
||||
if err != nil || !isCommentListPath(parsed.Path) {
|
||||
return target, ""
|
||||
}
|
||||
query := parsed.Query()
|
||||
pageRaw := strings.TrimSpace(query.Get("page"))
|
||||
if pageRaw == "" || strings.TrimSpace(query.Get("cursor")) != "" {
|
||||
return target, ""
|
||||
}
|
||||
page, err := strconv.Atoi(pageRaw)
|
||||
if err != nil || page < 1 {
|
||||
return target, ""
|
||||
}
|
||||
key := commentPageKey(parsed.Path, query)
|
||||
if page == 1 {
|
||||
// 重新加载 / 换了排序:游标与去重表都从头来。
|
||||
p.mu.Lock()
|
||||
delete(p.states, key)
|
||||
p.mu.Unlock()
|
||||
return target, key
|
||||
}
|
||||
p.mu.Lock()
|
||||
cursor := ""
|
||||
if st := p.states[key]; st != nil {
|
||||
cursor = st.cursor
|
||||
}
|
||||
p.mu.Unlock()
|
||||
if cursor == "" {
|
||||
// 不是从第一页翻过来的(没有游标可用):原样透传,仍然观察响应,
|
||||
// 靠下面的「整页重复」判断让页面停下来。
|
||||
return target, key
|
||||
}
|
||||
query.Set("cursor", cursor)
|
||||
query.Del("page")
|
||||
parsed.RawQuery = query.Encode()
|
||||
return parsed.String(), key
|
||||
}
|
||||
|
||||
// observeResponse 观察评论列表响应:记下游标,并在整页重复时把 has_more 改成 false。
|
||||
func (p *commentPager) observeResponse(key, body string) string {
|
||||
if p == nil || key == "" {
|
||||
return body
|
||||
}
|
||||
trimmed := strings.TrimSpace(body)
|
||||
if !strings.HasPrefix(trimmed, "{") {
|
||||
return body
|
||||
}
|
||||
var env commentListEnvelope
|
||||
if err := json.Unmarshal([]byte(trimmed), &env); err != nil {
|
||||
return body
|
||||
}
|
||||
if env.Data.Comments == nil && env.Data.HasMore == nil {
|
||||
return body
|
||||
}
|
||||
ids := make([]string, 0, len(env.Data.Comments))
|
||||
for _, c := range env.Data.Comments {
|
||||
if id := strings.TrimSpace(c.CommentID); id != "" {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
}
|
||||
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if p.states == nil {
|
||||
p.states = map[string]*commentPageState{}
|
||||
}
|
||||
st := p.states[key]
|
||||
if st == nil {
|
||||
st = &commentPageState{seen: map[string]struct{}{}}
|
||||
p.states[key] = st
|
||||
}
|
||||
repeated := len(ids) > 0
|
||||
for _, id := range ids {
|
||||
if _, ok := st.seen[id]; !ok {
|
||||
repeated = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if repeated && env.Data.HasMore != nil && *env.Data.HasMore {
|
||||
// 整页都是老评论:接口不认 page,再翻下去也只会拿到同一页。
|
||||
// 改掉 has_more,让页面显示「没有更多了」,别再反复拉取闪骨架屏。
|
||||
if replaced, ok := setFirstJSONBoolFalse(trimmed, "has_more"); ok {
|
||||
return replaced
|
||||
}
|
||||
return body
|
||||
}
|
||||
for _, id := range ids {
|
||||
st.seen[id] = struct{}{}
|
||||
}
|
||||
if env.Data.HasMore != nil && !*env.Data.HasMore {
|
||||
// 已经是最后一页:游标链到头,清掉它,页面万一再要一页也别拿旧游标
|
||||
// 去请求(那种请求会落在下面的「整页重复」判断上收尾)。
|
||||
st.cursor = ""
|
||||
return body
|
||||
}
|
||||
// 游标只在拿到新内容时前进:请求失败/重复时页面会重试同一页,
|
||||
// 游标也跟着重试同一个。
|
||||
if cursor := rawJSONScalar(env.Data.Cursor); cursor != "" {
|
||||
st.cursor = cursor
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
// commentListEnvelope 评论列表响应里我们关心的字段。
|
||||
type commentListEnvelope struct {
|
||||
Data struct {
|
||||
Comments []struct {
|
||||
CommentID string `json:"comment_id"`
|
||||
} `json:"comments"`
|
||||
HasMore *bool `json:"has_more"`
|
||||
Cursor json.RawMessage `json:"next_cursor"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
// isCommentListPath 判断是否是段落评论列表接口(/para_review)。
|
||||
func isCommentListPath(path string) bool {
|
||||
p := strings.TrimSuffix(strings.ToLower(strings.TrimSpace(path)), "/")
|
||||
return strings.HasSuffix(p, "/para_review")
|
||||
}
|
||||
|
||||
// commentPageKey 生成翻页状态键:接口路径 + 除 page/cursor 之外的查询参数。
|
||||
// 同一段落的不同排序(sort_by)各自一套状态,互不干扰。
|
||||
func commentPageKey(path string, query url.Values) string {
|
||||
rest := make(url.Values, len(query))
|
||||
for k, v := range query {
|
||||
if k == "page" || k == "cursor" {
|
||||
continue
|
||||
}
|
||||
rest[k] = v
|
||||
}
|
||||
keys := make([]string, 0, len(rest))
|
||||
for k := range rest {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
var b strings.Builder
|
||||
b.WriteString(path)
|
||||
for _, k := range keys {
|
||||
for _, v := range rest[k] {
|
||||
b.WriteString("|" + k + "=" + v)
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// rawJSONScalar 取一个 JSON 标量的字面量(游标是毫秒时间戳,可能被上游写成数字)。
|
||||
func rawJSONScalar(raw json.RawMessage) string {
|
||||
s := strings.TrimSpace(string(raw))
|
||||
if s == "" || s == "null" {
|
||||
return ""
|
||||
}
|
||||
if strings.HasPrefix(s, `"`) {
|
||||
var out string
|
||||
if json.Unmarshal(raw, &out) == nil {
|
||||
return strings.TrimSpace(out)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// setFirstJSONBoolFalse 把 JSON 里第一个 `"key":true` 改成 `"key":false`。
|
||||
//
|
||||
// 只做这一处等长替换,不重新序列化整份响应:评论里的 user_id、时间戳动辄十几位,
|
||||
// 走一遍 float64 会变成科学计数法,页面再解析就废了。
|
||||
func setFirstJSONBoolFalse(body, key string) (string, bool) {
|
||||
token := `"` + key + `"`
|
||||
idx := strings.Index(body, token)
|
||||
if idx < 0 {
|
||||
return body, false
|
||||
}
|
||||
i := idx + len(token)
|
||||
for i < len(body) && (body[i] == ' ' || body[i] == '\t' || body[i] == '\n' || body[i] == '\r') {
|
||||
i++
|
||||
}
|
||||
if i >= len(body) || body[i] != ':' {
|
||||
return body, false
|
||||
}
|
||||
i++
|
||||
for i < len(body) && (body[i] == ' ' || body[i] == '\t' || body[i] == '\n' || body[i] == '\r') {
|
||||
i++
|
||||
}
|
||||
if !strings.HasPrefix(body[i:], "true") {
|
||||
return body, false
|
||||
}
|
||||
return body[:i] + "false" + body[i+len("true"):], true
|
||||
}
|
||||
@@ -0,0 +1,187 @@
|
||||
package reader
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/service/reader/rule"
|
||||
)
|
||||
|
||||
// 本文件:段落评论翻页适配(browser_comment_page.go)的测试。
|
||||
//
|
||||
// 覆盖两件事:
|
||||
// 1. page=N(N>1)被换成上一页响应给的 cursor;
|
||||
// 2. 整页重复(接口不认 page 时的表现)时把 has_more 改成 false,页面才会收尾。
|
||||
|
||||
const commentParaBase = "https://cmt.example.com/para_review?item_id=X¶=1&source=QQ"
|
||||
|
||||
func TestCommentPagerRewritesPageToCursor(t *testing.T) {
|
||||
p := newCommentPager()
|
||||
|
||||
// 第一页:原样请求,响应里的游标被记下
|
||||
target, key := p.rewriteRequest(commentParaBase + "&page=1")
|
||||
if key == "" {
|
||||
t.Fatal("段落评论接口应被识别")
|
||||
}
|
||||
if target != commentParaBase+"&page=1" {
|
||||
t.Fatalf("第一页不该改写: %q", target)
|
||||
}
|
||||
p.observeResponse(key, `{"code":0,"data":{"comments":[{"comment_id":"c1"}],"has_more":true,"next_cursor":"1500000000000"}}`)
|
||||
|
||||
// 第二页:改成带 cursor 的请求,page 不再出现
|
||||
target, key2 := p.rewriteRequest(commentParaBase + "&page=2")
|
||||
if key2 != key {
|
||||
t.Fatalf("同一段落的状态键应一致: %q vs %q", key2, key)
|
||||
}
|
||||
if !strings.Contains(target, "cursor=1500000000000") || strings.Contains(target, "page=") {
|
||||
t.Fatalf("第二页应换成 cursor 请求: %q", target)
|
||||
}
|
||||
|
||||
// 拿到新评论:游标前进到下一页
|
||||
p.observeResponse(key, `{"code":0,"data":{"comments":[{"comment_id":"c2"}],"has_more":true,"next_cursor":"1400000000000"}}`)
|
||||
target, _ = p.rewriteRequest(commentParaBase + "&page=3")
|
||||
if !strings.Contains(target, "cursor=1400000000000") {
|
||||
t.Fatalf("第三页应使用新的游标: %q", target)
|
||||
}
|
||||
|
||||
// 重新加载(page=1)会把整轮状态清掉:没有游标就原样透传
|
||||
p.rewriteRequest(commentParaBase + "&page=1")
|
||||
target, _ = p.rewriteRequest(commentParaBase + "&page=2")
|
||||
if strings.Contains(target, "cursor=") {
|
||||
t.Fatalf("重置后不该还带着旧游标: %q", target)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCommentPagerStopsOnRepeatedPage(t *testing.T) {
|
||||
p := newCommentPager()
|
||||
_, key := p.rewriteRequest(commentParaBase + "&page=1")
|
||||
first := `{"code":0,"data":{"comments":[{"comment_id":"c1","like_count":562,"user":{"user_id":601948729051219}}],"has_more":true,"next_cursor":"1500000000000"}}`
|
||||
if got := p.observeResponse(key, first); got != first {
|
||||
t.Fatalf("第一页不该被改写: %q", got)
|
||||
}
|
||||
|
||||
// 同一页又被返回一次(接口忽略 page):改 has_more,让页面显示「没有更多了」
|
||||
repeat := `{"code":0,"data":{"comments":[{"comment_id":"c1","like_count":562,"user":{"user_id":601948729051219}}],"has_more":true,"next_cursor":"1500000000000"}}`
|
||||
got := p.observeResponse(key, repeat)
|
||||
if !strings.Contains(got, `"has_more":false`) {
|
||||
t.Fatalf("重复页应把 has_more 改成 false: %q", got)
|
||||
}
|
||||
// 只动 has_more 一处,其它字段(尤其是大整数)保持原样
|
||||
if strings.Replace(got, `"has_more":false`, `"has_more":true`, 1) != first {
|
||||
t.Fatalf("除 has_more 外不应改动响应: %q", got)
|
||||
}
|
||||
// 重复页不再推进游标
|
||||
target, _ := p.rewriteRequest(commentParaBase + "&page=2")
|
||||
if !strings.Contains(target, "cursor=1500000000000") {
|
||||
t.Fatalf("游标不该被重复页顶掉: %q", target)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCommentPagerIgnoresOtherRequests(t *testing.T) {
|
||||
p := newCommentPager()
|
||||
cases := []string{
|
||||
"https://cmt.example.com/book/detail?page=2", // 不是评论接口
|
||||
"https://cmt.example.com/para_review?item_id=X", // 没有 page
|
||||
"https://cmt.example.com/para_review?item_id=X&cursor=1", // 已经在用游标
|
||||
"https://cmt.example.com/para_review?item_id=X&page=0", // page 非法
|
||||
"https://cmt.example.com/para_review?item_id=X&sort_by=1&", // 没有 page
|
||||
}
|
||||
for _, target := range cases {
|
||||
got, key := p.rewriteRequest(target)
|
||||
if got != target || key != "" {
|
||||
t.Fatalf("不该改写的请求被处理了: %q -> %q (key=%q)", target, got, key)
|
||||
}
|
||||
}
|
||||
// 非 JSON 响应原样返回
|
||||
if got := p.observeResponse("k", "<html></html>"); got != "<html></html>" {
|
||||
t.Fatalf("非 JSON 响应被改动: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBrowserXHRCommentPagerFollowsCursor 端到端:页面按 page 翻页,
|
||||
// 代理层换成 cursor,接口给出新评论,最后一页重复时收尾。
|
||||
func TestBrowserXHRCommentPagerFollowsCursor(t *testing.T) {
|
||||
svc, _ := newLoginTestService(t)
|
||||
|
||||
var mu sync.Mutex
|
||||
seenCursors := []string{}
|
||||
seenPages := []string{}
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/page":
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
_, _ = io.WriteString(w, `<html><body>comment page</body></html>`)
|
||||
case "/para_review":
|
||||
q := r.URL.Query()
|
||||
mu.Lock()
|
||||
seenCursors = append(seenCursors, q.Get("cursor"))
|
||||
seenPages = append(seenPages, q.Get("page"))
|
||||
mu.Unlock()
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
// 上游只认 cursor:不认识 page,靠 next_cursor 一页页往前。
|
||||
switch q.Get("cursor") {
|
||||
case "":
|
||||
_, _ = io.WriteString(w, `{"code":0,"data":{"comments":[{"comment_id":"c1"}],"has_more":true,"next_cursor":"1500000000000"}}`)
|
||||
case "1500000000000":
|
||||
_, _ = io.WriteString(w, `{"code":0,"data":{"comments":[{"comment_id":"c2"}],"has_more":true,"next_cursor":"1400000000000"}}`)
|
||||
case "1400000000000":
|
||||
_, _ = io.WriteString(w, `{"code":0,"data":{"comments":[{"comment_id":"c3"}],"has_more":false,"next_cursor":""}}`)
|
||||
default:
|
||||
// 兜底分支:上游把同一页又给了一遍(页面的 page 循环就会走到这里)
|
||||
_, _ = io.WriteString(w, `{"code":0,"data":{"comments":[{"comment_id":"c3"}],"has_more":true,"next_cursor":"1300000000000"}}`)
|
||||
}
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
entry, err := svc.registerBrowser(t.Context(), srv.URL, "src-1", readerTestUserID,
|
||||
browserCookieTarget{}, rule.BrowserTask{URL: srv.URL + "/page", Title: "段评"}, browserModeOpen)
|
||||
if err != nil {
|
||||
t.Fatalf("登记承载页面失败: %v", err)
|
||||
}
|
||||
call := func(page string) string {
|
||||
t.Helper()
|
||||
res, err := svc.ProxyBrowserXHR(t.Context(), entry.id, http.MethodGet,
|
||||
srv.URL+"/para_review?item_id=X¶=1&source=QQ&page="+page, nil, "")
|
||||
if err != nil {
|
||||
t.Fatalf("代理请求失败: %v", err)
|
||||
}
|
||||
return res.Body
|
||||
}
|
||||
|
||||
if body := call("1"); !strings.Contains(body, `"has_more":true`) || !strings.Contains(body, "c1") {
|
||||
t.Fatalf("第一页响应异常: %q", body)
|
||||
}
|
||||
if body := call("2"); !strings.Contains(body, "c2") {
|
||||
t.Fatalf("第二页应拿到下一页评论: %q", body)
|
||||
}
|
||||
if body := call("3"); !strings.Contains(body, "c3") || !strings.Contains(body, `"has_more":false`) {
|
||||
t.Fatalf("第三页应是真正的最后一页: %q", body)
|
||||
}
|
||||
// 页面不知道已经到底,还可能再要一页:上游又把最后一页给了一遍,
|
||||
// 适配层要把 has_more 改成 false,页面才会显示「没有更多了」并停下。
|
||||
last := call("4")
|
||||
if !strings.Contains(last, `"has_more":false`) {
|
||||
t.Fatalf("整页重复时应收尾(has_more=false): %q", last)
|
||||
}
|
||||
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if strings.Join(seenPages, ",") != "1,,,4" {
|
||||
t.Fatalf("page 参数只应出现在没有游标可用的请求上: %v", seenPages)
|
||||
}
|
||||
if strings.Join(seenCursors, ",") != ",1500000000000,1400000000000," {
|
||||
t.Fatalf("游标传递不符预期: %v", seenCursors)
|
||||
}
|
||||
var env map[string]any
|
||||
if err := json.Unmarshal([]byte(last), &env); err != nil {
|
||||
t.Fatalf("收尾后的响应不是合法 JSON: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -103,6 +103,9 @@ type pendingBrowser struct {
|
||||
finalURL string
|
||||
mode string
|
||||
expires time.Time
|
||||
// comments 段落评论接口的翻页适配(上游页面按 page 翻页,接口只认 cursor,
|
||||
// 见 browser_comment_page.go)。
|
||||
comments *commentPager
|
||||
|
||||
done chan struct{}
|
||||
mu sync.Mutex
|
||||
@@ -218,6 +221,7 @@ func (s *ReaderService) registerBrowser(ctx context.Context, sourceURL, sourceID
|
||||
mode: mode,
|
||||
expires: time.Now().Add(browserPageTTL),
|
||||
done: make(chan struct{}),
|
||||
comments: newCommentPager(),
|
||||
}
|
||||
|
||||
s.browserMu.Lock()
|
||||
@@ -446,7 +450,7 @@ func (s *ReaderService) requestBrowserResource(ctx context.Context, sourceURL st
|
||||
}
|
||||
// 书源级请求头(含 Referer / UA),与阅读请求保持一致
|
||||
if s.repo != nil {
|
||||
if src, findErr := s.repo.GetSourceByURL(reqCtx, sourceURL); findErr == nil && src != nil && src.Header != "" {
|
||||
if src, findErr := s.repo.GetSourceAnyByURL(reqCtx, sourceURL); findErr == nil && src != nil && src.Header != "" {
|
||||
var headers map[string]any
|
||||
if json.Unmarshal([]byte(src.Header), &headers) == nil {
|
||||
for k, v := range headers {
|
||||
@@ -511,6 +515,13 @@ func (s *ReaderService) ProxyBrowserXHR(ctx context.Context, id, method, target
|
||||
if method == "" {
|
||||
method = http.MethodGet
|
||||
}
|
||||
// 段落评论接口:评论页按 page=N 翻页,而接口只认 cursor(见
|
||||
// browser_comment_page.go)。改写发生在补书源凭据之前,Cookie/Referer
|
||||
// 仍然按目标站点选取。
|
||||
pagerKey := ""
|
||||
if entry.comments != nil {
|
||||
target, pagerKey = entry.comments.rewriteRequest(target)
|
||||
}
|
||||
reqCtx, cancel := context.WithTimeout(ctx, browserFetchTimeout)
|
||||
defer cancel()
|
||||
// 挂上 sink:页面自己发起的请求(如扫码登录的轮询/取票跳转)下发的
|
||||
@@ -546,7 +557,7 @@ func (s *ReaderService) ProxyBrowserXHR(ctx context.Context, id, method, target
|
||||
}
|
||||
}
|
||||
if s.repo != nil {
|
||||
if src, findErr := s.repo.GetSourceByURL(reqCtx, entry.sourceURL); findErr == nil && src != nil && src.Header != "" {
|
||||
if src, findErr := s.repo.GetSourceAnyByURL(reqCtx, entry.sourceURL); findErr == nil && src != nil && src.Header != "" {
|
||||
var extra map[string]any
|
||||
if json.Unmarshal([]byte(src.Header), &extra) == nil {
|
||||
for k, v := range extra {
|
||||
@@ -577,6 +588,9 @@ func (s *ReaderService) ProxyBrowserXHR(ctx context.Context, id, method, target
|
||||
out := &BrowserXHRResult{Status: resp.StatusCode, ContentType: contentType}
|
||||
if isTextualContent(contentType) {
|
||||
out.Body = string(data)
|
||||
if entry.comments != nil && pagerKey != "" {
|
||||
out.Body = entry.comments.observeResponse(pagerKey, out.Body)
|
||||
}
|
||||
} else {
|
||||
out.Body = base64.StdEncoding.EncodeToString(data)
|
||||
out.Base64 = true
|
||||
|
||||
@@ -444,7 +444,7 @@ func (s *ReaderService) OpenContentComment(ctx context.Context, userID, bookID,
|
||||
return nil, fmt.Errorf("书籍不存在或已移出书架")
|
||||
}
|
||||
sourceID := ""
|
||||
if src, findErr := s.repo.GetSourceByURL(ctx, book.Origin); findErr == nil && src != nil {
|
||||
if src, findErr := s.repo.GetSourceAnyByURL(ctx, book.Origin); findErr == nil && src != nil {
|
||||
sourceID = src.ID
|
||||
}
|
||||
if strings.TrimSpace(title) == "" {
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
package reader
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// 图片解密(coverDecodeJs / ruleContent.imageDecode)与 HLS 判定链路。
|
||||
|
||||
// 正文图片解密:书源规则把字节异或 0x55 后返回,服务端解密应还原原文。
|
||||
func TestDecodeImageBytesRunsRuleJS(t *testing.T) {
|
||||
svc, srv, book := prepareCacheTestBook(t)
|
||||
ctx := t.Context()
|
||||
|
||||
// 动态给书源加上 imageDecode 规则:XOR 0x55 还原。
|
||||
raw := strings.Replace(
|
||||
cacheTestSourceJSON(srv.URL),
|
||||
`"ruleContent": {"content": "id.content@textNodes"}`,
|
||||
`"ruleContent": {"content": "id.content@textNodes", "imageDecode": "var src = new Uint8Array(result); var out = new Uint8Array(src.length); for (var i=0;i<src.length;i++){ out[i] = src[i] ^ 0x55; } out"}`,
|
||||
1,
|
||||
)
|
||||
if _, err := svc.ImportSources(ctx, "u1", raw); err != nil {
|
||||
t.Fatalf("更新书源失败: %v", err)
|
||||
}
|
||||
rule := svc.ImageDecodeRule(ctx, book)
|
||||
if rule == "" {
|
||||
t.Fatal("imageDecode 规则未解析")
|
||||
}
|
||||
|
||||
plain := []byte("PNG-PLAIN-BYTES")
|
||||
encoded := make([]byte, len(plain))
|
||||
for i, b := range plain {
|
||||
encoded[i] = b ^ 0x55
|
||||
}
|
||||
decoded, err := svc.DecodeImageBytes(ctx, book, "https://img.example.com/a.png", false, encoded)
|
||||
if err != nil {
|
||||
t.Fatalf("解密失败: %v", err)
|
||||
}
|
||||
if string(decoded) != string(plain) {
|
||||
t.Fatalf("解密结果 = %q,期望 %q", decoded, plain)
|
||||
}
|
||||
}
|
||||
|
||||
// 没有规则时零开销直通(返回原始字节)。
|
||||
func TestDecodeImageBytesPassthroughWithoutRule(t *testing.T) {
|
||||
svc, srv, book := prepareCacheTestBook(t)
|
||||
data := []byte("raw-image-bytes")
|
||||
out, err := svc.DecodeImageBytes(t.Context(), book, srv.URL+"/a.png", false, data)
|
||||
if err != nil {
|
||||
t.Fatalf("直通路径不应报错: %v", err)
|
||||
}
|
||||
if string(out) != string(data) {
|
||||
t.Fatalf("无规则时应原样返回: %q", out)
|
||||
}
|
||||
}
|
||||
|
||||
// 解密脚本抛错时退回原始字节(保证至少还能看到图)。
|
||||
func TestDecodeImageBytesFallsBackOnBadRule(t *testing.T) {
|
||||
svc, srv, book := prepareCacheTestBook(t)
|
||||
ctx := t.Context()
|
||||
raw := strings.Replace(
|
||||
cacheTestSourceJSON(srv.URL),
|
||||
`"ruleContent": {"content": "id.content@textNodes"}`,
|
||||
`"ruleContent": {"content": "id.content@textNodes", "imageDecode": "throw new Error('bad rule')"}`,
|
||||
1,
|
||||
)
|
||||
if _, err := svc.ImportSources(ctx, "u1", raw); err != nil {
|
||||
t.Fatalf("更新书源失败: %v", err)
|
||||
}
|
||||
data := []byte("still-an-image")
|
||||
out, err := svc.DecodeImageBytes(ctx, book, srv.URL+"/a.png", false, data)
|
||||
if err != nil {
|
||||
t.Fatalf("坏规则应降级而不是报错: %v", err)
|
||||
}
|
||||
if string(out) != string(data) {
|
||||
t.Fatalf("坏规则应返回原始字节: %q", out)
|
||||
}
|
||||
}
|
||||
|
||||
// 封面解密:按书源 URL(搜索结果尚未入库)执行 coverDecodeJs。
|
||||
func TestDecodeCoverBytes(t *testing.T) {
|
||||
svc, srv, _ := prepareCacheTestBook(t)
|
||||
ctx := t.Context()
|
||||
raw := strings.Replace(
|
||||
cacheTestSourceJSON(srv.URL),
|
||||
`"bookSourceType": 0,`,
|
||||
`"bookSourceType": 0, "coverDecodeJs": "var src = new Uint8Array(result); var out = new Uint8Array(src.length); for (var i=0;i<src.length;i++){ out[i] = src[i] ^ 0x33; } out",`,
|
||||
1,
|
||||
)
|
||||
if _, err := svc.ImportSources(ctx, "u1", raw); err != nil {
|
||||
t.Fatalf("更新书源失败: %v", err)
|
||||
}
|
||||
if !svc.SourceHasCoverDecode(srv.URL) {
|
||||
t.Fatal("应识别出该源声明了封面解密")
|
||||
}
|
||||
plain := []byte("cover-bytes")
|
||||
encoded := make([]byte, len(plain))
|
||||
for i, b := range plain {
|
||||
encoded[i] = b ^ 0x33
|
||||
}
|
||||
out := svc.DecodeCoverBytes(ctx, srv.URL, srv.URL+"/cover.jpg", encoded)
|
||||
if string(out) != string(plain) {
|
||||
t.Fatalf("封面解密结果 = %q,期望 %q", out, plain)
|
||||
}
|
||||
|
||||
// 未声明解密的源:封面地址保持原样(不签代理)。
|
||||
plainURL := svc.RewriteBookCover(ctx, "", srv.URL+"/x", "https://img.example.com/c.jpg")
|
||||
if !strings.HasPrefix(plainURL, "https://img.example.com/") {
|
||||
t.Fatalf("无解密规则时不应改写封面: %q", plainURL)
|
||||
}
|
||||
// 声明了解密的源:封面走签名代理。
|
||||
signed := svc.RewriteBookCover(ctx, "", srv.URL, "https://img.example.com/c.jpg")
|
||||
if !strings.Contains(signed, "/api/reader/media?") || !strings.Contains(signed, "d=cover") {
|
||||
t.Fatalf("封面应走解密代理: %q", signed)
|
||||
}
|
||||
}
|
||||
|
||||
// HLS 播放列表嗅探:#EXTM3U 标记优先于 Content-Type。
|
||||
func TestReaderBodyIsPlaylist(t *testing.T) {
|
||||
playlist := []byte("#EXTM3U\n#EXT-X-VERSION:3\n")
|
||||
if !BodyIsPlaylist(playlist, "application/octet-stream") {
|
||||
t.Fatal("以 #EXTM3U 开头应判为播放列表")
|
||||
}
|
||||
if BodyIsPlaylist([]byte("\x89PNG\r\n"), "image/png") {
|
||||
t.Fatal("二进制图片不应判为播放列表")
|
||||
}
|
||||
if !BodyIsPlaylist([]byte("\n#EXTM3U\n"), "application/vnd.apple.mpegurl") {
|
||||
t.Fatal("带前置空行且 Content-Type 为 mpegurl 时应判为播放列表")
|
||||
}
|
||||
if BodyIsPlaylist(nil, "application/vnd.apple.mpegurl") {
|
||||
t.Fatal("空体不应判为播放列表")
|
||||
}
|
||||
}
|
||||
@@ -143,6 +143,11 @@ func parseLocalBookPayload(name string, data []byte) (*localBookPayload, error)
|
||||
return nil, fmt.Errorf("文件内容为空或无法解码")
|
||||
}
|
||||
out.ext, out.charset = ".txt", charset
|
||||
// 章节区间是按「解码后的 UTF-8 文本」算的字节下标,托管副本就必须落盘这份
|
||||
// UTF-8 文本:原始字节若是 GBK/Big5/UTF-16,按 UTF-8 区间去读会整章错位。
|
||||
// 外部原地引用不能改写源文件(见 importLocalBook 的 externalPath 分支),
|
||||
// 由读取侧按 charset 整文件解码后再切片。
|
||||
out.payload = []byte(decoded)
|
||||
for i, c := range splitTXTChapters(decoded) {
|
||||
out.chaps = append(out.chaps, model.ReaderChapter{
|
||||
Index: i, Title: c.Title, Tag: fmt.Sprintf("%d:%d", c.Start, c.End),
|
||||
@@ -275,11 +280,21 @@ func (s *ReaderService) ImportLocalBook(
|
||||
ext: parsed.ext,
|
||||
charset: parsed.charset,
|
||||
chaps: parsed.chaps,
|
||||
payload: data,
|
||||
payload: managedPayload(parsed, data),
|
||||
coverEntry: parsed.coverEntry,
|
||||
})
|
||||
}
|
||||
|
||||
// managedPayload 托管副本实际落盘的字节。
|
||||
//
|
||||
// TXT 走 out.payload(已解码的 UTF-8,与章节下标的基准一致);EPUB 原样落盘。
|
||||
func managedPayload(parsed *localBookPayload, raw []byte) []byte {
|
||||
if parsed != nil && len(parsed.payload) > 0 {
|
||||
return parsed.payload
|
||||
}
|
||||
return raw
|
||||
}
|
||||
|
||||
// ImportLocalBookFromPath 从服务器上已有的文件导入书籍(TXT / EPUB)。
|
||||
//
|
||||
// 原地引用,不复制到 data/reader/local;同一个文件重复导入按覆盖更新处理,
|
||||
@@ -590,14 +605,20 @@ func (s *ReaderService) readLocalChapter(book *model.ReaderBook, ch model.Reader
|
||||
if !ok {
|
||||
return "", fmt.Errorf("章节定位信息损坏,请重新导入该书")
|
||||
}
|
||||
if end <= start {
|
||||
return "", nil
|
||||
}
|
||||
// 外部原地引用的非 UTF-8 文件:章节区间是按导入时解码出的 UTF-8 文本算的,
|
||||
// 必须整文件解码后再切片,直接读原始字节会错位(GBK 一字 2 字节、UTF-16 更多)。
|
||||
// 托管副本在导入时已统一落盘 UTF-8,这里不会走这条分支。
|
||||
if book.LocalExternal && needsDecodeBeforeSlice(book.Charset) {
|
||||
return readLocalChapterDecoded(path, start, end)
|
||||
}
|
||||
f, err := os.Open(path) // #nosec G304 -- path 由服务端按书籍 ID 生成
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("本地书籍文件已丢失: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
if end <= start {
|
||||
return "", nil
|
||||
}
|
||||
buf := make([]byte, end-start)
|
||||
if _, err := f.ReadAt(buf, int64(start)); err != nil && err != io.EOF {
|
||||
return "", fmt.Errorf("读取章节失败: %w", err)
|
||||
@@ -605,6 +626,36 @@ func (s *ReaderService) readLocalChapter(book *model.ReaderBook, ch model.Reader
|
||||
return strings.TrimSpace(string(buf)), nil
|
||||
}
|
||||
|
||||
// needsDecodeBeforeSlice 该字符集的文件是否需要在切片前整文件解码。
|
||||
func needsDecodeBeforeSlice(charset string) bool {
|
||||
switch strings.ToLower(strings.TrimSpace(charset)) {
|
||||
case "", "utf-8", "utf8":
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// readLocalChapterDecoded 整文件解码为 UTF-8 后按字节区间取章节(外部引用模式)。
|
||||
func readLocalChapterDecoded(path string, start, end int) (string, error) {
|
||||
f, err := os.Open(path) // #nosec G304 -- path 来自书籍记录,导入时已校验允许根目录
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("本地书籍文件已丢失: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
data, err := io.ReadAll(io.LimitReader(f, LocalBookMaxBytes+1))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("读取章节失败: %w", err)
|
||||
}
|
||||
decoded, _ := decodeTextFile(data)
|
||||
if start > len(decoded) {
|
||||
return "", nil
|
||||
}
|
||||
if end > len(decoded) {
|
||||
end = len(decoded)
|
||||
}
|
||||
return strings.TrimSpace(decoded[start:end]), nil
|
||||
}
|
||||
|
||||
// DeleteLocalBookFile 移出书架时删除本地文件(网络书籍无文件,直接返回)。
|
||||
//
|
||||
// 原地引用的书指向服务器上已有的文件/目录,移出书架只解除引用,不动源文件。
|
||||
|
||||
@@ -73,7 +73,7 @@ type LoginResult struct {
|
||||
|
||||
// GetSourceLogin 返回书源登录界面描述与当前登录状态。
|
||||
func (s *ReaderService) GetSourceLogin(ctx context.Context, userID, sourceID string) (*SourceLoginInfo, error) {
|
||||
src, bs, err := s.loadSource(ctx, sourceID)
|
||||
src, bs, err := s.loadSourceForUser(ctx, userID, sourceID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -142,7 +142,7 @@ func (s *ReaderService) resolveLoginFields(sess *sourceSession, bs *BookSource,
|
||||
// fields 为前端提交的表单值,会与已保存的登录信息合并后作为 result 传入。
|
||||
// action 为空时执行 loginUrl 里的 login()(即 legado 的「确认登录」)。
|
||||
func (s *ReaderService) RunLoginAction(ctx context.Context, userID, sourceID, action string, fields map[string]string) (*LoginResult, error) {
|
||||
src, bs, err := s.loadSource(ctx, sourceID)
|
||||
src, bs, err := s.loadSourceForUser(ctx, userID, sourceID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -206,11 +206,11 @@ func (s *ReaderService) RunLoginAction(ctx context.Context, userID, sourceID, ac
|
||||
|
||||
// SetSourceVariable 覆盖保存书源变量(前端变量编辑器)。
|
||||
// 保存后书源 JS 的 getVariable() 即可读到。
|
||||
func (s *ReaderService) SetSourceVariable(ctx context.Context, sourceID, variable string) error {
|
||||
func (s *ReaderService) SetSourceVariable(ctx context.Context, userID, sourceID, variable string) error {
|
||||
if strings.TrimSpace(variable) != "" && !json.Valid([]byte(variable)) {
|
||||
return fmt.Errorf("变量必须是合法 JSON")
|
||||
}
|
||||
src, bs, err := s.loadSource(ctx, sourceID)
|
||||
src, bs, err := s.loadSourceForUser(ctx, userID, sourceID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -222,8 +222,8 @@ func (s *ReaderService) SetSourceVariable(ctx context.Context, sourceID, variabl
|
||||
}
|
||||
|
||||
// SetSourceLoginInfo 直接覆盖保存登录信息(前端表单保存,不触发登录动作)。
|
||||
func (s *ReaderService) SetSourceLoginInfo(ctx context.Context, sourceID string, fields map[string]string) error {
|
||||
src, bs, err := s.loadSource(ctx, sourceID)
|
||||
func (s *ReaderService) SetSourceLoginInfo(ctx context.Context, userID, sourceID string, fields map[string]string) error {
|
||||
src, bs, err := s.loadSourceForUser(ctx, userID, sourceID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -239,8 +239,8 @@ func (s *ReaderService) SetSourceLoginInfo(ctx context.Context, sourceID string,
|
||||
}
|
||||
|
||||
// ClearSourceLogin 清除登录态:登录信息与全部 Cookie(对应 legado logout)。
|
||||
func (s *ReaderService) ClearSourceLogin(ctx context.Context, sourceID string) error {
|
||||
src, bs, err := s.loadSource(ctx, sourceID)
|
||||
func (s *ReaderService) ClearSourceLogin(ctx context.Context, userID, sourceID string) error {
|
||||
src, bs, err := s.loadSourceForUser(ctx, userID, sourceID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -22,7 +22,7 @@ import (
|
||||
// 覆盖「登录 → Cookie 落库 → 后续请求自动携带 Cookie → 登出清理」。
|
||||
|
||||
// readerTestUserID 登录接口按用户隔离浏览器待办,测试里统一用一个固定用户。
|
||||
const readerTestUserID = "test-user"
|
||||
const readerTestUserID = "u1"
|
||||
|
||||
// loginTestServer 模拟一个需要登录的书源站点:
|
||||
// - POST /login_api 校验账号密码并下发会话 Cookie
|
||||
@@ -129,10 +129,10 @@ func newLoginTestService(t *testing.T) (*ReaderService, *repository.Container) {
|
||||
// prepareLoginSource 导入测试书源并返回其 ID。
|
||||
func prepareLoginSource(t *testing.T, svc *ReaderService, sourceJSON string) string {
|
||||
t.Helper()
|
||||
if _, err := svc.ImportSources(t.Context(), sourceJSON); err != nil {
|
||||
if _, err := svc.ImportSources(t.Context(), "u1", sourceJSON); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
srcs, err := svc.ListSources(t.Context())
|
||||
srcs, err := svc.ListSources(t.Context(), "u1")
|
||||
if err != nil || len(srcs) == 0 {
|
||||
t.Fatalf("导入后应能读到书源: %v", err)
|
||||
}
|
||||
@@ -151,14 +151,14 @@ func TestSourceLoginEndToEnd(t *testing.T) {
|
||||
sourceID := prepareLoginSource(t, svc, loginTestSourceJSON(t, srv.URL))
|
||||
|
||||
// ── 登录前:未鉴权,站点返回未登录页 → 搜不到书 ──
|
||||
if books, _, err := svc.Search(ctx, "会员", nil, 1); err != nil {
|
||||
if books, _, err := svc.Search(ctx, "u1", "会员", nil, 1); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if len(books) != 0 {
|
||||
t.Fatalf("未登录时不应搜到结果: %+v", books)
|
||||
}
|
||||
|
||||
// ── 登录 ──
|
||||
res, err := svc.RunLoginAction(ctx, readerTestUserID, sourceID, "", map[string]string{
|
||||
res, err := svc.RunLoginAction(ctx, "u1", sourceID, "", map[string]string{
|
||||
"邮箱": "user@example.com", "密码": "pw123456",
|
||||
})
|
||||
if err != nil {
|
||||
@@ -176,7 +176,7 @@ func TestSourceLoginEndToEnd(t *testing.T) {
|
||||
|
||||
// ── 登录态应落库(换一个 service 实例仍可读到)──
|
||||
svc2 := NewReaderService(svc.cfg, zap.NewNop(), repos)
|
||||
info, err := svc2.GetSourceLogin(ctx, readerTestUserID, sourceID)
|
||||
info, err := svc2.GetSourceLogin(ctx, "u1", sourceID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -188,7 +188,7 @@ func TestSourceLoginEndToEnd(t *testing.T) {
|
||||
}
|
||||
|
||||
// ── 登录后搜索:应携带 Cookie 并成功 ──
|
||||
books, skipped, err := svc.Search(ctx, "会员", nil, 1)
|
||||
books, skipped, err := svc.Search(ctx, "u1", "会员", nil, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -200,17 +200,17 @@ func TestSourceLoginEndToEnd(t *testing.T) {
|
||||
}
|
||||
|
||||
// ── 登出:Cookie 清除,搜索重新未登录 ──
|
||||
if err := svc.ClearSourceLogin(ctx, sourceID); err != nil {
|
||||
if err := svc.ClearSourceLogin(ctx, "u1", sourceID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info, err = svc.GetSourceLogin(ctx, readerTestUserID, sourceID)
|
||||
info, err = svc.GetSourceLogin(ctx, "u1", sourceID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.LoggedIn || len(info.Cookies) != 0 {
|
||||
t.Fatalf("登出后不应残留登录态: %+v", info)
|
||||
}
|
||||
if books, _, err := svc.Search(ctx, "会员", nil, 1); err != nil {
|
||||
if books, _, err := svc.Search(ctx, "u1", "会员", nil, 1); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if len(books) != 0 {
|
||||
t.Fatalf("登出后不应还能搜到结果: %+v", books)
|
||||
@@ -224,7 +224,7 @@ func TestSourceLoginWrongPassword(t *testing.T) {
|
||||
svc, _ := newLoginTestService(t)
|
||||
sourceID := prepareLoginSource(t, svc, loginTestSourceJSON(t, srv.URL))
|
||||
|
||||
res, err := svc.RunLoginAction(t.Context(), readerTestUserID, sourceID, "", map[string]string{
|
||||
res, err := svc.RunLoginAction(t.Context(), "u1", sourceID, "", map[string]string{
|
||||
"邮箱": "user@example.com", "密码": "wrong",
|
||||
})
|
||||
if err != nil {
|
||||
@@ -248,7 +248,7 @@ func TestSourceLoginInfo_ExposesUIFields(t *testing.T) {
|
||||
svc, _ := newLoginTestService(t)
|
||||
sourceID := prepareLoginSource(t, svc, loginTestSourceJSON(t, srv.URL))
|
||||
|
||||
info, err := svc.GetSourceLogin(t.Context(), readerTestUserID, sourceID)
|
||||
info, err := svc.GetSourceLogin(t.Context(), "u1", sourceID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -280,7 +280,7 @@ func TestSourceStateEncryptedAtRest(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
sourceID := prepareLoginSource(t, svc, loginTestSourceJSON(t, srv.URL))
|
||||
|
||||
if _, err := svc.RunLoginAction(ctx, readerTestUserID, sourceID, "", map[string]string{
|
||||
if _, err := svc.RunLoginAction(ctx, "u1", sourceID, "", map[string]string{
|
||||
"邮箱": "user@example.com", "密码": "pw123456",
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -313,10 +313,10 @@ func TestSourceVariableRoundTrip(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
sourceID := prepareLoginSource(t, svc, loginTestSourceJSON(t, srv.URL))
|
||||
|
||||
if err := svc.SetSourceVariable(ctx, sourceID, `{"线路":"https://v2.example.com"}`); err != nil {
|
||||
if err := svc.SetSourceVariable(ctx, "u1", sourceID, `{"线路":"https://v2.example.com"}`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info, err := svc.GetSourceLogin(ctx, readerTestUserID, sourceID)
|
||||
info, err := svc.GetSourceLogin(ctx, "u1", sourceID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -329,7 +329,7 @@ func TestSourceVariableRoundTrip(t *testing.T) {
|
||||
}
|
||||
|
||||
// 非法 JSON 应被拒绝
|
||||
if err := svc.SetSourceVariable(ctx, sourceID, "not-json"); err == nil {
|
||||
if err := svc.SetSourceVariable(ctx, "u1", sourceID, "not-json"); err == nil {
|
||||
t.Fatal("非法 JSON 变量应被拒绝")
|
||||
}
|
||||
}
|
||||
@@ -367,7 +367,7 @@ func TestEnabledCookieJarGating(t *testing.T) {
|
||||
}`
|
||||
sourceID := prepareLoginSource(t, svc, srcJSON)
|
||||
|
||||
if _, _, err := svc.Search(ctx, "任意", nil, 1); err != nil {
|
||||
if _, _, err := svc.Search(ctx, "u1", "任意", nil, 1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// 自动捕获被关闭:不应出现 auto=from-response
|
||||
@@ -386,10 +386,10 @@ func TestEnabledCookieJarGating(t *testing.T) {
|
||||
defer srv2.Close()
|
||||
srcJSONOn = strings.ReplaceAll(srcJSONOn, srv.URL, srv2.URL)
|
||||
// prepareLoginSource 返回列表首个书源,这里按 URL 精确定位刚导入的对照源
|
||||
if _, err := svc.ImportSources(ctx, srcJSONOn); err != nil {
|
||||
if _, err := svc.ImportSources(ctx, "u1", srcJSONOn); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := svc.Search(ctx, "任意", nil, 1); err != nil {
|
||||
if _, _, err := svc.Search(ctx, "u1", "任意", nil, 1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := svc.newSourceState(ctx, srv2.URL).GetCookie(srv2.URL); !strings.Contains(got, "auto=from-response") {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,504 @@
|
||||
package reader
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
"github.com/truewhile/MeBox/internal/service/reader/rule"
|
||||
)
|
||||
|
||||
// 本文件实现阅读正文的持久缓存(对应 legado BookHelp 的章节正文缓存 + CacheBook)。
|
||||
//
|
||||
// 设计要点:
|
||||
// - 缓存「书源侧产物」:getContentFrom 的输出(书源 replaceRegex 之后,
|
||||
// 用户替换规则、签名代理改写之前)。因此缓存里不含任何用户维度的处理结果,
|
||||
// 用户替换规则在读出后逐请求应用,规则改动即时生效。
|
||||
// - 缓存键按书源行组织(书源按用户独立,见 contentBookKey 的说明),
|
||||
// 不同用户即使持有同一 URL 的书源也各自一份缓存。
|
||||
// - 索引落库(ReaderContentCache),内容落盘
|
||||
// (cache_dir/reader-content/<sourceKey>/<bookKey>/<chapterKey>.json)。
|
||||
// - 缓存键只包含书源行、章节身份、书源指纹与格式版本:不含书籍行 ID
|
||||
// (目录刷新会重建行,见 RemapContentCacheOnTocChange)。
|
||||
|
||||
// readerContentFormatVersion 载荷格式版本。解析管线(正文归一 / 段评提取 / 图片标记)
|
||||
// 语义变化时必须递增:旧版本条目会被当作未命中并重抓,避免读到旧结构的缓存。
|
||||
const readerContentFormatVersion = 1
|
||||
|
||||
// readerContentMaxEntryBytes 单条缓存的字节上限(超过不缓存,避免超大章节占满盘)。
|
||||
const readerContentMaxEntryBytes = 16 << 20
|
||||
|
||||
// audioCacheTTL 音频清单的短 TTL:CDN 直链常带签名/过期参数,缓存太久会拿到失效地址。
|
||||
const audioCacheTTL = 30 * time.Minute
|
||||
|
||||
// cachedChapterContent 落盘的载荷。
|
||||
type cachedChapterContent struct {
|
||||
FormatVersion int `json:"v"`
|
||||
SourceHash string `json:"src"`
|
||||
ContentType string `json:"type"`
|
||||
Content string `json:"content,omitempty"`
|
||||
Tracks []string `json:"tracks,omitempty"`
|
||||
Images []string `json:"images,omitempty"`
|
||||
ImageStyle string `json:"image_style,omitempty"`
|
||||
IsHLS bool `json:"hls,omitempty"`
|
||||
Comments []ContentComment `json:"comments,omitempty"`
|
||||
SavedAt int64 `json:"saved_at"`
|
||||
}
|
||||
|
||||
// contentFlight 一次进行中的正文抓取,并发的调用方共享结果(读穿透单飞)。
|
||||
type contentFlight struct {
|
||||
done chan struct{}
|
||||
data *ChapterContent
|
||||
err error
|
||||
}
|
||||
|
||||
// ─── 键与身份 ──────────────────────────────────────────────────────────────
|
||||
|
||||
// contentBookKey 书源身份哈希:书源行 ID + 书本地址。
|
||||
//
|
||||
// 用书源行 ID 而不是 origin(书源 URL):书源按用户独立,不同用户可以持有同一
|
||||
// URL 但规则/header 不同的副本,按 URL 共享缓存会把别人书源的产物喂进来。
|
||||
// 用「行 ID + 书本地址」而不是书源显示名:聚合源的 origin 是显示名,多本同源书会撞。
|
||||
func contentBookKey(sourceID, bookURL string) string {
|
||||
sum := sha256.Sum256([]byte(sourceID + "\x00" + strings.TrimSpace(bookURL)))
|
||||
return hex.EncodeToString(sum[:16])
|
||||
}
|
||||
|
||||
// contentOriginKey 书源行维度的磁盘目录名(同一书源的不同书共享这一层)。
|
||||
func contentOriginKey(sourceID string) string {
|
||||
sum := sha256.Sum256([]byte(sourceID))
|
||||
return hex.EncodeToString(sum[:16])
|
||||
}
|
||||
|
||||
// contentChapterIdentity 章节身份原文:绝对化 URL 优先,卷/空地址退化为标题。
|
||||
func contentChapterIdentity(book *model.ReaderBook, ch model.ReaderChapter) string {
|
||||
if ch.URL != "" && !ch.IsVolume {
|
||||
base := book.BookURL
|
||||
if strings.TrimSpace(book.TocURL) != "" {
|
||||
base = book.TocURL
|
||||
}
|
||||
if abs := rule.GetAbsoluteURL(base, ch.URL); abs != "" {
|
||||
return "url|" + abs
|
||||
}
|
||||
return "url|" + strings.TrimSpace(ch.URL)
|
||||
}
|
||||
return "title|" + strings.TrimSpace(ch.Title)
|
||||
}
|
||||
|
||||
// contentChapterKey 章节身份哈希。
|
||||
func contentChapterKey(book *model.ReaderBook, ch model.ReaderChapter) string {
|
||||
sum := sha256.Sum256([]byte(contentChapterIdentity(book, ch)))
|
||||
return hex.EncodeToString(sum[:16])
|
||||
}
|
||||
|
||||
// contentSourceHash 书源内容指纹:RawJSON 哈希(书源更新后旧缓存自然失效)。
|
||||
func contentSourceHash(src *model.ReaderBookSource) string {
|
||||
if src == nil {
|
||||
return ""
|
||||
}
|
||||
sum := sha256.Sum256([]byte(src.RawJSON))
|
||||
return hex.EncodeToString(sum[:16])
|
||||
}
|
||||
|
||||
// ─── 磁盘布局 ──────────────────────────────────────────────────────────────
|
||||
|
||||
// readerContentDir 正文缓存根目录。
|
||||
func (s *ReaderService) readerContentDir() string {
|
||||
if s == nil || s.cfg == nil {
|
||||
return ""
|
||||
}
|
||||
base := strings.TrimSpace(s.cfg.Cache.CacheDir)
|
||||
if base == "" {
|
||||
if dataDir := strings.TrimSpace(s.cfg.App.DataDir); dataDir != "" {
|
||||
base = filepath.Join(dataDir, "cache")
|
||||
}
|
||||
}
|
||||
if base == "" {
|
||||
return ""
|
||||
}
|
||||
return filepath.Join(base, "reader-content")
|
||||
}
|
||||
|
||||
// contentFilePath 单条缓存的磁盘路径:<root>/<sourceKey>/<bookKey>/<chapterKey>.json。
|
||||
// chapterKey 已是 hex 哈希,不含路径分隔符。
|
||||
func (s *ReaderService) contentFilePath(sourceID, bookURL, chapterKey string) string {
|
||||
root := s.readerContentDir()
|
||||
if root == "" {
|
||||
return ""
|
||||
}
|
||||
return filepath.Join(root, contentOriginKey(sourceID), contentBookKey(sourceID, bookURL), chapterKey+".json")
|
||||
}
|
||||
|
||||
// contentBookDir 某本书的缓存目录(整本清理用)。
|
||||
func (s *ReaderService) contentBookDir(sourceID, bookURL string) string {
|
||||
root := s.readerContentDir()
|
||||
if root == "" {
|
||||
return ""
|
||||
}
|
||||
return filepath.Join(root, contentOriginKey(sourceID), contentBookKey(sourceID, bookURL))
|
||||
}
|
||||
|
||||
// writeContentFile 原子写入缓存文件(临时文件 + rename)。
|
||||
func writeContentFile(path string, payload []byte) error {
|
||||
if path == "" {
|
||||
return fmt.Errorf("缓存目录未配置")
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil {
|
||||
return err
|
||||
}
|
||||
tmp, err := os.CreateTemp(filepath.Dir(path), ".content-*")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
name := tmp.Name()
|
||||
if _, err := tmp.Write(payload); err != nil {
|
||||
_ = tmp.Close()
|
||||
_ = os.Remove(name)
|
||||
return err
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
_ = os.Remove(name)
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(name, path); err != nil {
|
||||
_ = os.Remove(name)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// removeContentFile 删除缓存文件(不存在视为成功)。
|
||||
func removeContentFile(path string) {
|
||||
if path == "" {
|
||||
return
|
||||
}
|
||||
_ = os.Remove(path)
|
||||
}
|
||||
|
||||
// ─── 读写 ──────────────────────────────────────────────────────────────────
|
||||
|
||||
// contentSourceID 缓存键用的书源行 ID。src 缺失时按 origin 兜底查一次:
|
||||
// 直接调用(测试、诊断路径)可能只拿到书籍与 origin。
|
||||
func (s *ReaderService) contentSourceID(ctx context.Context, src *model.ReaderBookSource, book *model.ReaderBook) string {
|
||||
if src != nil && src.ID != "" {
|
||||
return src.ID
|
||||
}
|
||||
if s == nil || s.repo == nil || book == nil {
|
||||
return ""
|
||||
}
|
||||
found, err := s.repo.GetSourceAnyByURL(ctx, book.Origin)
|
||||
if err != nil || found == nil {
|
||||
return ""
|
||||
}
|
||||
return found.ID
|
||||
}
|
||||
|
||||
// loadCachedContent 读取一章节的缓存:命中返回内容与 true。
|
||||
//
|
||||
// contentType 为空表示「按章节取任意类型」(正文链路不需要预知类型);给出具体类型时
|
||||
// 用于带类型校验的探测(如音频 TTL 的测试与诊断)。
|
||||
//
|
||||
// 校验链:索引存在 → 格式版本一致 → 书源指纹一致 → 未过期 → 载荷可解析。
|
||||
func (s *ReaderService) loadCachedContent(ctx context.Context, src *model.ReaderBookSource, bs *BookSource, book *model.ReaderBook, ch model.ReaderChapter, contentType string) (*ChapterContent, bool) {
|
||||
if s == nil || s.repo == nil || book == nil {
|
||||
return nil, false
|
||||
}
|
||||
sourceID := s.contentSourceID(ctx, src, book)
|
||||
if sourceID == "" {
|
||||
return nil, false
|
||||
}
|
||||
bookKey := contentBookKey(sourceID, book.BookURL)
|
||||
chapterKey := contentChapterKey(book, ch)
|
||||
row, err := s.repo.GetContentCacheByChapter(ctx, bookKey, chapterKey, contentType)
|
||||
if err != nil || row == nil {
|
||||
return nil, false
|
||||
}
|
||||
if row.FormatVersion != readerContentFormatVersion {
|
||||
s.dropContentCacheRow(ctx, row)
|
||||
return nil, false
|
||||
}
|
||||
if row.SourceHash != "" && row.SourceHash != contentSourceHash(src) {
|
||||
// 书源更新过:旧正文可能已失效,删掉重抓。
|
||||
s.dropContentCacheRow(ctx, row)
|
||||
return nil, false
|
||||
}
|
||||
now := time.Now().Unix()
|
||||
if row.ExpiresAt > 0 && row.ExpiresAt < now {
|
||||
s.dropContentCacheRow(ctx, row)
|
||||
return nil, false
|
||||
}
|
||||
path := s.contentFilePath(sourceID, book.BookURL, chapterKey)
|
||||
raw, err := os.ReadFile(path) // #nosec G304 -- 路径由服务端生成
|
||||
if err != nil {
|
||||
s.dropContentCacheRow(ctx, row)
|
||||
return nil, false
|
||||
}
|
||||
var payload cachedChapterContent
|
||||
if err := json.Unmarshal(raw, &payload); err != nil || payload.FormatVersion != readerContentFormatVersion {
|
||||
s.dropContentCacheRow(ctx, row)
|
||||
return nil, false
|
||||
}
|
||||
// 命中:刷新 LRU 时间与计数(异步失败不影响读取)。
|
||||
_ = s.repo.TouchContentCache(ctx, row.ID, row.Hits+1, now)
|
||||
|
||||
out := &ChapterContent{
|
||||
Type: payload.ContentType,
|
||||
Content: payload.Content,
|
||||
Tracks: payload.Tracks,
|
||||
Images: payload.Images,
|
||||
ImageStyle: payload.ImageStyle,
|
||||
IsHLS: payload.IsHLS,
|
||||
Comments: payload.Comments,
|
||||
declaredType: -1,
|
||||
}
|
||||
return out, true
|
||||
}
|
||||
|
||||
// saveCachedContent 写入一章节的缓存(空内容不写,与报错语义保持一致)。
|
||||
func (s *ReaderService) saveCachedContent(ctx context.Context, src *model.ReaderBookSource, book *model.ReaderBook, ch model.ReaderChapter, out *ChapterContent) {
|
||||
if s == nil || s.repo == nil || book == nil || out == nil || chapterContentEmpty(out) {
|
||||
return
|
||||
}
|
||||
payload := cachedChapterContent{
|
||||
FormatVersion: readerContentFormatVersion,
|
||||
SourceHash: contentSourceHash(src),
|
||||
ContentType: out.Type,
|
||||
Content: out.Content,
|
||||
Tracks: out.Tracks,
|
||||
Images: out.Images,
|
||||
ImageStyle: out.ImageStyle,
|
||||
IsHLS: out.IsHLS,
|
||||
Comments: out.Comments,
|
||||
SavedAt: time.Now().Unix(),
|
||||
}
|
||||
raw, err := json.Marshal(payload)
|
||||
if err != nil || len(raw) > readerContentMaxEntryBytes {
|
||||
return
|
||||
}
|
||||
sourceID := s.contentSourceID(ctx, src, book)
|
||||
if sourceID == "" {
|
||||
return
|
||||
}
|
||||
path := s.contentFilePath(sourceID, book.BookURL, contentChapterKey(book, ch))
|
||||
if err := writeContentFile(path, raw); err != nil {
|
||||
if s.log != nil {
|
||||
s.log.Warn("reader: 写入正文缓存失败", zap.String("path", path), zap.Error(err))
|
||||
}
|
||||
return
|
||||
}
|
||||
now := time.Now().Unix()
|
||||
row := &model.ReaderContentCache{
|
||||
OriginHash: contentOriginKey(sourceID),
|
||||
BookKey: contentBookKey(sourceID, book.BookURL),
|
||||
ChapterKey: contentChapterKey(book, ch),
|
||||
ChapterIdentity: contentChapterIdentity(book, ch),
|
||||
ChapterIndex: ch.Index,
|
||||
ContentType: out.Type,
|
||||
SourceHash: payload.SourceHash,
|
||||
FormatVersion: readerContentFormatVersion,
|
||||
SizeBytes: int64(len(raw)),
|
||||
AssetCount: len(out.Tracks) + len(out.Images),
|
||||
ExpiresAt: s.contentExpiry(out.Type, now),
|
||||
LastAccessAt: now,
|
||||
}
|
||||
if err := s.repo.UpsertContentCache(ctx, row); err != nil && s.log != nil {
|
||||
s.log.Warn("reader: 写入正文缓存索引失败", zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
// contentExpiry 按内容类型给出过期时间:音频清单短 TTL(直链会过期),文本/图片用配置的 TTL。
|
||||
func (s *ReaderService) contentExpiry(contentType string, now int64) int64 {
|
||||
if contentType == "audio" {
|
||||
return now + int64(audioCacheTTL.Seconds())
|
||||
}
|
||||
ttlHours := 0
|
||||
if s != nil && s.cfg != nil {
|
||||
ttlHours = s.cfg.Cache.ReaderContentTTLHours
|
||||
}
|
||||
if ttlHours <= 0 {
|
||||
return 0
|
||||
}
|
||||
return now + int64(ttlHours)*3600
|
||||
}
|
||||
|
||||
// dropContentCacheRow 删除索引并清理磁盘文件。
|
||||
func (s *ReaderService) dropContentCacheRow(ctx context.Context, row *model.ReaderContentCache) {
|
||||
if s == nil || s.repo == nil || row == nil {
|
||||
return
|
||||
}
|
||||
_ = s.repo.DeleteContentCacheRow(ctx, row.ID)
|
||||
path := s.contentPathFromRow(row)
|
||||
removeContentFile(path)
|
||||
if path != "" {
|
||||
s.pruneContentDirIfEmpty(filepath.Dir(path))
|
||||
}
|
||||
}
|
||||
|
||||
// pruneContentDirIfEmpty 删掉空目录(缓存清理后不留空壳)。
|
||||
func (s *ReaderService) pruneContentDirIfEmpty(dir string) {
|
||||
if dir == "" || dir == s.readerContentDir() {
|
||||
return
|
||||
}
|
||||
if entries, err := os.ReadDir(dir); err == nil && len(entries) == 0 {
|
||||
_ = os.Remove(dir)
|
||||
}
|
||||
}
|
||||
|
||||
// ClearContentCacheForBook 清理某书源行下一本书的全部正文缓存(换源/移出书架时调用)。
|
||||
//
|
||||
// sourceID 是书源行 ID:书源按用户独立,缓存也随之按书源行隔离,所以删除
|
||||
// 自己那份书源或换源不会影响其他用户的同名书源缓存。
|
||||
func (s *ReaderService) ClearContentCacheForBook(ctx context.Context, sourceID, bookURL string) {
|
||||
if s == nil || s.repo == nil || sourceID == "" || bookURL == "" {
|
||||
return
|
||||
}
|
||||
bookKey := contentBookKey(sourceID, bookURL)
|
||||
if _, err := s.repo.DeleteContentCacheByBook(ctx, bookKey); err != nil {
|
||||
return
|
||||
}
|
||||
if dir := s.contentBookDir(sourceID, bookURL); dir != "" {
|
||||
_ = os.RemoveAll(dir)
|
||||
s.pruneContentDirIfEmpty(filepath.Dir(dir))
|
||||
}
|
||||
}
|
||||
|
||||
// PruneContentCache 清理阅读正文缓存(TTL 过期 + 容量 LRU),供调度器按小时调用。
|
||||
func (s *ReaderService) PruneContentCache(ctx context.Context) {
|
||||
s.pruneReaderContentCache(ctx)
|
||||
}
|
||||
|
||||
// pruneReaderContentCache 容量/TTL 淘汰:过期条目直接删,超配额按 LRU 删到 90%。
|
||||
func (s *ReaderService) pruneReaderContentCache(ctx context.Context) {
|
||||
if s == nil || s.repo == nil {
|
||||
return
|
||||
}
|
||||
now := time.Now().Unix()
|
||||
if expired, err := s.repo.ListContentCacheExpired(ctx, now, 500); err == nil {
|
||||
for i := range expired {
|
||||
row := expired[i]
|
||||
_ = s.repo.DeleteContentCacheRow(ctx, row.ID)
|
||||
removeContentFile(s.contentPathFromRow(&row))
|
||||
}
|
||||
}
|
||||
maxMB := 0
|
||||
if s.cfg != nil {
|
||||
maxMB = s.cfg.Cache.ReaderContentMaxSizeMB
|
||||
}
|
||||
if maxMB <= 0 {
|
||||
return
|
||||
}
|
||||
_, totalBytes, err := s.repo.ContentCacheStats(ctx)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
limit := int64(maxMB) << 20
|
||||
if totalBytes <= limit {
|
||||
return
|
||||
}
|
||||
target := limit * 9 / 10
|
||||
oldest, err := s.repo.ListContentCacheOldest(ctx, 500)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for i := range oldest {
|
||||
if totalBytes <= target {
|
||||
break
|
||||
}
|
||||
row := oldest[i]
|
||||
_ = s.repo.DeleteContentCacheRow(ctx, row.ID)
|
||||
removeContentFile(s.contentPathFromRow(&row))
|
||||
totalBytes -= row.SizeBytes
|
||||
}
|
||||
}
|
||||
|
||||
// contentPathFromRow 由索引行拼磁盘路径(索引存了 origin_hash / book_key / chapter_key)。
|
||||
func (s *ReaderService) contentPathFromRow(row *model.ReaderContentCache) string {
|
||||
if s == nil || row == nil {
|
||||
return ""
|
||||
}
|
||||
root := s.readerContentDir()
|
||||
if root == "" {
|
||||
return ""
|
||||
}
|
||||
return filepath.Join(root, row.OriginHash, row.BookKey, row.ChapterKey+".json")
|
||||
}
|
||||
|
||||
// ─── 目录刷新后的缓存迁移(对应 legado BookHelp.remapContentCache) ─────────
|
||||
|
||||
// RemapContentCacheOnTocChange 目录刷新后迁移缓存键:同一章按绝对化 URL 命中、
|
||||
// 标题唯一命中兜底。不迁移的话每次「更新目录」都会让已缓存正文全部冷启动。
|
||||
func (s *ReaderService) RemapContentCacheOnTocChange(ctx context.Context, book *model.ReaderBook, oldChapters, newChapters []model.ReaderChapter) {
|
||||
if s == nil || s.repo == nil || book == nil || len(oldChapters) == 0 || len(newChapters) == 0 {
|
||||
return
|
||||
}
|
||||
sourceID := s.contentSourceID(ctx, nil, book)
|
||||
if sourceID == "" {
|
||||
return
|
||||
}
|
||||
bookKey := contentBookKey(sourceID, book.BookURL)
|
||||
rows, err := s.repo.ListContentCacheByBook(ctx, bookKey)
|
||||
if err != nil || len(rows) == 0 {
|
||||
return
|
||||
}
|
||||
// 旧身份 → 新章节。
|
||||
byURL := map[string]model.ReaderChapter{}
|
||||
byTitle := map[string][]model.ReaderChapter{}
|
||||
for _, ch := range newChapters {
|
||||
if ch.URL != "" && !ch.IsVolume {
|
||||
base := book.BookURL
|
||||
if strings.TrimSpace(book.TocURL) != "" {
|
||||
base = book.TocURL
|
||||
}
|
||||
abs := rule.GetAbsoluteURL(base, ch.URL)
|
||||
if abs == "" {
|
||||
abs = strings.TrimSpace(ch.URL)
|
||||
}
|
||||
byURL["url|"+abs] = ch
|
||||
}
|
||||
if t := strings.TrimSpace(ch.Title); t != "" {
|
||||
byTitle["title|"+t] = append(byTitle["title|"+t], ch)
|
||||
}
|
||||
}
|
||||
for i := range rows {
|
||||
row := rows[i]
|
||||
var target model.ReaderChapter
|
||||
found := false
|
||||
if ch, ok := byURL[row.ChapterIdentity]; ok {
|
||||
target, found = ch, true
|
||||
} else if matches := byTitle[row.ChapterIdentity]; len(matches) == 1 {
|
||||
// 标题唯一命中才迁移:多个同标题章节时无法确定是哪一章,宁可重抓。
|
||||
target, found = matches[0], true
|
||||
}
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
newKey := contentChapterKey(book, target)
|
||||
if newKey == row.ChapterKey {
|
||||
continue // 身份未变,只更新序号
|
||||
}
|
||||
oldPath := s.contentPathFromRow(&row)
|
||||
newPath := s.contentFilePath(sourceID, book.BookURL, newKey)
|
||||
if oldPath != "" && newPath != "" {
|
||||
if err := os.MkdirAll(filepath.Dir(newPath), 0o750); err == nil {
|
||||
// 文件不在(只留下索引)时忽略:新条目下次读取会自动重抓。
|
||||
_ = os.Rename(oldPath, newPath)
|
||||
}
|
||||
}
|
||||
_ = s.repo.DeleteContentCacheRow(ctx, row.ID)
|
||||
row.ID = ""
|
||||
row.ChapterKey = newKey
|
||||
row.ChapterIdentity = contentChapterIdentity(book, target)
|
||||
row.ChapterIndex = target.Index
|
||||
row.LastAccessAt = time.Now().Unix()
|
||||
_ = s.repo.UpsertContentCache(ctx, &row)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,460 @@
|
||||
package reader
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
)
|
||||
|
||||
// 正文持久缓存(reader_content_cache.go)的端到端测试:
|
||||
// 抓取 → 写缓存 → 再次读取命中且不再访问书源 → 批量接口统计正确。
|
||||
|
||||
// cacheTestServer 每章返回固定正文并统计各章请求次数。
|
||||
type cacheTestServer struct {
|
||||
*httptest.Server
|
||||
mu sync.Mutex
|
||||
hits map[string]int
|
||||
body func(chapter int) string
|
||||
failOn map[int]bool
|
||||
}
|
||||
|
||||
func newCacheTestServer() *cacheTestServer {
|
||||
s := &cacheTestServer{hits: map[string]int{}, failOn: map[int]bool{}}
|
||||
s.body = func(chapter int) string { return fmt.Sprintf("第%d章的正文内容。", chapter) }
|
||||
s.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
switch {
|
||||
case strings.HasPrefix(r.URL.Path, "/book/1/toc"):
|
||||
_, _ = w.Write([]byte(cacheTestTocHTML))
|
||||
case strings.HasPrefix(r.URL.Path, "/book/1/c"):
|
||||
chapter := 0
|
||||
_, _ = fmt.Sscanf(strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/book/1/c"), ".html"), "%d", &chapter)
|
||||
s.mu.Lock()
|
||||
s.hits[fmt.Sprintf("c%d", chapter)]++
|
||||
fail := s.failOn[chapter]
|
||||
s.mu.Unlock()
|
||||
if fail {
|
||||
http.Error(w, "boom", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte(fmt.Sprintf(`<html><body><div id="content">%s</div></body></html>`, s.body(chapter))))
|
||||
case strings.HasPrefix(r.URL.Path, "/book/"):
|
||||
_, _ = w.Write([]byte(cacheTestBookInfoHTML))
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
return s
|
||||
}
|
||||
|
||||
func (s *cacheTestServer) count(chapter int) int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.hits[fmt.Sprintf("c%d", chapter)]
|
||||
}
|
||||
|
||||
func cacheTestSourceJSON(server string) string {
|
||||
return fmt.Sprintf(`{
|
||||
"bookSourceUrl": %q,
|
||||
"bookSourceName": "缓存测试源",
|
||||
"bookSourceType": 0,
|
||||
"searchUrl": "%s/search/{{key}}/1.html",
|
||||
"ruleSearch": {"bookList": "class.item", "name": "tag.h3@text", "bookUrl": "tag.a@href"},
|
||||
"ruleBookInfo": {"name": "class.info@tag.h1@text", "tocUrl": "class.info@class.toc@href"},
|
||||
"ruleToc": {"chapterList": "class.chapters@tag.li", "chapterName": "tag.a@text", "chapterUrl": "tag.a@href"},
|
||||
"ruleContent": {"content": "id.content@textNodes"}
|
||||
}`, server, server)
|
||||
}
|
||||
|
||||
const cacheTestTocHTML = `<html><body>
|
||||
<ul class="chapters">
|
||||
<li><a href="/book/1/c1.html">第 1 章</a></li>
|
||||
<li><a href="/book/1/c2.html">第 2 章</a></li>
|
||||
<li><a href="/book/1/c3.html">第 3 章</a></li>
|
||||
</ul></body></html>`
|
||||
|
||||
const cacheTestBookInfoHTML = `<html><body>
|
||||
<div class="info"><h1>缓存之书</h1><a class="toc" href="/book/1/toc">目录</a></div>
|
||||
</body></html>`
|
||||
|
||||
// prepareCacheTestBook 建好「书源 + 书架书籍 + 目录缓存」并打开独立缓存目录。
|
||||
func prepareCacheTestBook(t *testing.T) (*ReaderService, *cacheTestServer, *model.ReaderBook) {
|
||||
t.Helper()
|
||||
srv := newCacheTestServer()
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
svc, _ := newLoginTestService(t)
|
||||
svc.cfg.Cache.CacheDir = t.TempDir()
|
||||
svc.cfg.Cache.ReaderContentTTLHours = 168
|
||||
srcID := importTestSource(t, svc, cacheTestSourceJSON(srv.URL), srv.URL)
|
||||
|
||||
ctx := t.Context()
|
||||
book, err := svc.AddBook(ctx, "u1", SearchOrigin{
|
||||
SourceID: srcID, Origin: srv.URL, OriginName: "缓存测试源",
|
||||
BookURL: srv.URL + "/book/1",
|
||||
}, "缓存之书", "作者", "")
|
||||
if err != nil {
|
||||
t.Fatalf("加入书架失败: %v", err)
|
||||
}
|
||||
chapters := make([]ChapterInput, 0, 3)
|
||||
for i := 1; i <= 3; i++ {
|
||||
chapters = append(chapters, ChapterInput{
|
||||
Index: i - 1, Title: fmt.Sprintf("第 %d 章", i),
|
||||
URL: fmt.Sprintf("%s/book/1/c%d.html", srv.URL, i),
|
||||
})
|
||||
}
|
||||
if err := svc.SaveChapters(ctx, book.ID, chapters); err != nil {
|
||||
t.Fatalf("保存目录失败: %v", err)
|
||||
}
|
||||
return svc, srv, book
|
||||
}
|
||||
|
||||
// 第一次读取抓源并写缓存;第二次直接命中持久缓存,不再访问书源。
|
||||
func TestContentCacheHitAndPersist(t *testing.T) {
|
||||
svc, srv, book := prepareCacheTestBook(t)
|
||||
ctx := t.Context()
|
||||
|
||||
first, err := svc.GetContentForBook(ctx, "u1", book.ID, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("首次读取失败: %v", err)
|
||||
}
|
||||
if !strings.Contains(first.Content, "第1章的正文内容。") {
|
||||
t.Fatalf("正文不符合预期: %q", first.Content)
|
||||
}
|
||||
if got := srv.count(1); got != 1 {
|
||||
t.Fatalf("首次读取应访问书源 1 次,实际 %d", got)
|
||||
}
|
||||
|
||||
second, err := svc.GetContentForBook(ctx, "u1", book.ID, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("二次读取失败: %v", err)
|
||||
}
|
||||
if second.Content != first.Content {
|
||||
t.Fatalf("缓存命中的正文不一致: %q vs %q", second.Content, first.Content)
|
||||
}
|
||||
if got := srv.count(1); got != 1 {
|
||||
t.Fatalf("缓存命中不应再访问书源,实际 %d 次", got)
|
||||
}
|
||||
|
||||
// 缓存文件确实落盘(键按书源行 ID 组织)。
|
||||
sourceID := svc.contentSourceID(ctx, nil, book)
|
||||
if sourceID == "" {
|
||||
t.Fatal("找不到书源行")
|
||||
}
|
||||
path := svc.contentFilePath(sourceID, book.BookURL, contentChapterKey(book, model.ReaderChapter{
|
||||
Index: 0, Title: "第 1 章", URL: srv.URL + "/book/1/c1.html",
|
||||
}))
|
||||
if path == "" {
|
||||
t.Fatal("缓存路径为空(cache_dir 未生效)")
|
||||
}
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
t.Fatalf("缓存文件未落盘: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 目录刷新后同一章(地址不变)仍应命中缓存,不因「更新目录」冷启动。
|
||||
func TestContentCacheSurvivesTocRefresh(t *testing.T) {
|
||||
svc, srv, book := prepareCacheTestBook(t)
|
||||
ctx := t.Context()
|
||||
|
||||
if _, err := svc.GetContentForBook(ctx, "u1", book.ID, 1); err != nil {
|
||||
t.Fatalf("读取第 2 章失败: %v", err)
|
||||
}
|
||||
|
||||
// 模拟「更新目录」:同一批章节重新落库(行 ID 会变,身份不变)。
|
||||
chapters := make([]ChapterInput, 0, 3)
|
||||
for i := 1; i <= 3; i++ {
|
||||
chapters = append(chapters, ChapterInput{
|
||||
Index: i - 1, Title: fmt.Sprintf("第 %d 章", i),
|
||||
URL: fmt.Sprintf("%s/book/1/c%d.html", srv.URL, i),
|
||||
})
|
||||
}
|
||||
if err := svc.SaveChapters(ctx, book.ID, chapters); err != nil {
|
||||
t.Fatalf("重写目录失败: %v", err)
|
||||
}
|
||||
// 刷新后的 remap(refreshBookToc 内的同一调用)。
|
||||
oldChapters := []model.ReaderChapter{
|
||||
{Index: 0, Title: "第 1 章", URL: srv.URL + "/book/1/c1.html"},
|
||||
{Index: 1, Title: "第 2 章", URL: srv.URL + "/book/1/c2.html"},
|
||||
{Index: 2, Title: "第 3 章", URL: srv.URL + "/book/1/c3.html"},
|
||||
}
|
||||
newChapters, _ := svc.repo.ListChapters(ctx, book.ID)
|
||||
svc.RemapContentCacheOnTocChange(ctx, book, oldChapters, newChapters)
|
||||
|
||||
if _, err := svc.GetContentForBook(ctx, "u1", book.ID, 1); err != nil {
|
||||
t.Fatalf("刷新后读取失败: %v", err)
|
||||
}
|
||||
if got := srv.count(2); got != 1 {
|
||||
t.Fatalf("目录刷新后应命中缓存(书源请求仍为 1 次),实际 %d 次", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 并发请求同一章只应打一次书源(读穿透单飞)。
|
||||
func TestContentCacheSingleFlight(t *testing.T) {
|
||||
svc, srv, book := prepareCacheTestBook(t)
|
||||
ctx := t.Context()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
errs := make([]error, 6)
|
||||
for i := 0; i < 6; i++ {
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
_, errs[i] = svc.GetContentForBook(ctx, "u1", book.ID, 0)
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
for i, err := range errs {
|
||||
if err != nil {
|
||||
t.Fatalf("并发读取 %d 失败: %v", i, err)
|
||||
}
|
||||
}
|
||||
if got := srv.count(1); got != 1 {
|
||||
t.Fatalf("同一章并发应单飞为 1 次书源请求,实际 %d 次", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 批量接口:单章失败不影响其余章节,统计 hit/miss/failed。
|
||||
func TestContentBatchStatsAndFailureIsolation(t *testing.T) {
|
||||
svc, srv, book := prepareCacheTestBook(t)
|
||||
ctx := t.Context()
|
||||
|
||||
// 预热第 1 章(进缓存),第 2 章按失败处理。
|
||||
if _, err := svc.GetContentForBook(ctx, "u1", book.ID, 0); err != nil {
|
||||
t.Fatalf("预热失败: %v", err)
|
||||
}
|
||||
srv.mu.Lock()
|
||||
srv.failOn[2] = true
|
||||
srv.mu.Unlock()
|
||||
|
||||
result, err := svc.GetContentBatch(ctx, "u1", book.ID, []int{0, 1, 2}, false)
|
||||
if err != nil {
|
||||
t.Fatalf("批量读取失败: %v", err)
|
||||
}
|
||||
if len(result.Items) != 3 {
|
||||
t.Fatalf("应返回 3 项,实际 %d", len(result.Items))
|
||||
}
|
||||
if result.Stats.Hit != 1 {
|
||||
t.Fatalf("第 1 章应命中缓存,hit=%d", result.Stats.Hit)
|
||||
}
|
||||
if result.Stats.Failed != 1 {
|
||||
t.Fatalf("第 2 章应失败,failed=%d", result.Stats.Failed)
|
||||
}
|
||||
if result.Stats.Miss != 1 {
|
||||
t.Fatalf("第 3 章应为新抓,miss=%d", result.Stats.Miss)
|
||||
}
|
||||
if !strings.Contains(result.Items[2].Content, "第3章的正文内容。") {
|
||||
t.Fatalf("第 3 章内容异常: %+v", result.Items[2])
|
||||
}
|
||||
if result.Items[1].Error == "" {
|
||||
t.Fatalf("第 2 章应带错误: %+v", result.Items[1])
|
||||
}
|
||||
// 失败章不写缓存:去掉失败开关后重试应真正抓源。
|
||||
srv.mu.Lock()
|
||||
srv.failOn[2] = false
|
||||
srv.mu.Unlock()
|
||||
if _, err := svc.GetContentForBook(ctx, "u1", book.ID, 1); err != nil {
|
||||
t.Fatalf("恢复后读取第 2 章失败: %v", err)
|
||||
}
|
||||
if got := srv.count(2); got != 2 {
|
||||
t.Fatalf("失败章不应进缓存,恢复后应重抓(2 次),实际 %d", got)
|
||||
}
|
||||
// 越界/卷章节被忽略。
|
||||
if got, err := svc.GetContentBatch(ctx, "u1", book.ID, []int{-1, 99}, false); err != nil || len(got.Items) != 0 {
|
||||
t.Fatalf("越界索引应被忽略: items=%d err=%v", len(got.Items), err)
|
||||
}
|
||||
}
|
||||
|
||||
// 书源更新(RawJSON 变化)后旧缓存失效,重新抓取。
|
||||
func TestContentCacheInvalidatedOnSourceUpdate(t *testing.T) {
|
||||
svc, srv, book := prepareCacheTestBook(t)
|
||||
ctx := t.Context()
|
||||
|
||||
if _, err := svc.GetContentForBook(ctx, "u1", book.ID, 0); err != nil {
|
||||
t.Fatalf("首次读取失败: %v", err)
|
||||
}
|
||||
// 重新导入同一书源(RawJSON 变化)→ 指纹变化。
|
||||
updated := strings.Replace(cacheTestSourceJSON(srv.URL), "缓存测试源", "缓存测试源v2", 1)
|
||||
if _, err := svc.ImportSources(ctx, "u1", updated); err != nil {
|
||||
t.Fatalf("更新书源失败: %v", err)
|
||||
}
|
||||
if _, err := svc.GetContentForBook(ctx, "u1", book.ID, 0); err != nil {
|
||||
t.Fatalf("更新后读取失败: %v", err)
|
||||
}
|
||||
if got := srv.count(1); got != 2 {
|
||||
t.Fatalf("书源更新后应重抓(2 次),实际 %d 次", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 移出书架且无其他引用时清理缓存文件与索引。
|
||||
func TestContentCacheClearedOnRemoveBook(t *testing.T) {
|
||||
svc, _, book := prepareCacheTestBook(t)
|
||||
ctx := t.Context()
|
||||
if _, err := svc.GetContentForBook(ctx, "u1", book.ID, 0); err != nil {
|
||||
t.Fatalf("读取失败: %v", err)
|
||||
}
|
||||
sourceID := svc.contentSourceID(ctx, nil, book)
|
||||
dir := svc.contentBookDir(sourceID, book.BookURL)
|
||||
if _, err := os.Stat(dir); err != nil {
|
||||
t.Fatalf("缓存目录应存在: %v", err)
|
||||
}
|
||||
if err := svc.RemoveBook(ctx, "u1", book.ID); err != nil {
|
||||
t.Fatalf("移出书架失败: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(dir); !os.IsNotExist(err) {
|
||||
t.Fatalf("移出书架后缓存目录应被清理,err=%v", err)
|
||||
}
|
||||
if rows, err := svc.repo.ListContentCacheByBook(ctx, contentBookKey(sourceID, book.BookURL)); err != nil || len(rows) != 0 {
|
||||
t.Fatalf("缓存索引应清空: rows=%d err=%v", len(rows), err)
|
||||
}
|
||||
}
|
||||
|
||||
// 音频清单使用短 TTL:过期后不命中并自动清理。
|
||||
func TestContentCacheAudioShortTTL(t *testing.T) {
|
||||
svc, srv, book := prepareCacheTestBook(t)
|
||||
ctx := t.Context()
|
||||
|
||||
ch := model.ReaderChapter{Index: 0, Title: "第 1 章", URL: srv.URL + "/book/1/c1.html"}
|
||||
src := mustSource(t, svc, srv.URL)
|
||||
bookKey := contentBookKey(src.ID, book.BookURL)
|
||||
svc.saveCachedContent(ctx, src, book, ch, &ChapterContent{
|
||||
Type: "audio", Tracks: []string{srv.URL + "/a.mp3"},
|
||||
})
|
||||
row, err := svc.repo.GetContentCache(ctx, bookKey, contentChapterKey(book, ch), "audio")
|
||||
if err != nil {
|
||||
t.Fatalf("音频缓存未写入: %v", err)
|
||||
}
|
||||
if row.ExpiresAt == 0 {
|
||||
t.Fatal("音频缓存应带 TTL")
|
||||
}
|
||||
// 过期后读取视为未命中,并清掉条目。
|
||||
row.ExpiresAt = 1
|
||||
if err := svc.repo.UpsertContentCache(ctx, row); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := svc.loadCachedContent(ctx, src, nil, book, ch, "audio"); ok {
|
||||
t.Fatal("过期缓存不应命中")
|
||||
}
|
||||
if _, err := svc.repo.GetContentCache(ctx, bookKey, contentChapterKey(book, ch), "audio"); err == nil {
|
||||
t.Fatal("过期条目应被删除")
|
||||
}
|
||||
}
|
||||
|
||||
// mustSource 取(并解析)指定书源记录,供直接调用缓存方法的测试使用。
|
||||
func mustSource(t *testing.T, svc *ReaderService, sourceURL string) *model.ReaderBookSource {
|
||||
t.Helper()
|
||||
src, err := svc.repo.GetSourceByURL(context.Background(), "u1", sourceURL)
|
||||
if err != nil {
|
||||
t.Fatalf("取书源失败: %v", err)
|
||||
}
|
||||
return src
|
||||
}
|
||||
|
||||
// 缓存键:不同书源/书本地址/章节地址必须落在不同键上。
|
||||
func TestContentCacheKeys(t *testing.T) {
|
||||
base := &model.ReaderBook{Origin: "https://a.com", BookURL: "https://a.com/book/1", TocURL: "https://a.com/toc/1"}
|
||||
ch1 := model.ReaderChapter{Index: 0, Title: "第 1 章", URL: "/book/1/c1.html"}
|
||||
ch2 := model.ReaderChapter{Index: 0, Title: "第 1 章", URL: "https://a.com/book/1/c1.html"}
|
||||
// 相对与绝对地址指向同一章:绝对化后身份一致。
|
||||
if contentChapterKey(base, ch1) != contentChapterKey(base, ch2) {
|
||||
t.Fatal("同一章的相对/绝对地址应得到同一身份")
|
||||
}
|
||||
if contentBookKey("https://a.com", "https://a.com/book/1") == contentBookKey("https://b.com", "https://a.com/book/1") {
|
||||
t.Fatal("不同书源的同一地址不应共享键")
|
||||
}
|
||||
if contentChapterIdentity(base, model.ReaderChapter{Index: 0, Title: "卷名", IsVolume: true}) != "title|卷名" {
|
||||
t.Fatal("卷章节应退化为标题身份")
|
||||
}
|
||||
}
|
||||
|
||||
// 缓存目录不可用时(未配置)不应影响正文读取。
|
||||
func TestContentCacheWithoutCacheDir(t *testing.T) {
|
||||
srv := newCacheTestServer()
|
||||
defer srv.Close()
|
||||
svc, _ := newLoginTestService(t)
|
||||
svc.cfg.Cache.CacheDir = ""
|
||||
svc.cfg.App.DataDir = filepath.Join(t.TempDir(), "missing", "data")
|
||||
srcID := importTestSource(t, svc, cacheTestSourceJSON(srv.URL), srv.URL)
|
||||
ctx := t.Context()
|
||||
book, err := svc.AddBook(ctx, "u1", SearchOrigin{
|
||||
SourceID: srcID, Origin: srv.URL, OriginName: "缓存测试源",
|
||||
BookURL: srv.URL + "/book/1",
|
||||
}, "缓存之书", "作者", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := svc.SaveChapters(ctx, book.ID, []ChapterInput{{Index: 0, Title: "第 1 章", URL: srv.URL + "/book/1/c1.html"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, err := svc.GetContentForBook(ctx, "u1", book.ID, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("无缓存目录时读取失败: %v", err)
|
||||
}
|
||||
if !strings.Contains(out.Content, "第1章") {
|
||||
t.Fatalf("正文异常: %q", out.Content)
|
||||
}
|
||||
}
|
||||
|
||||
// 容量与过期淘汰:总量超配额时按 LRU 删到 90%,过期条目优先清理。
|
||||
// 这是「长期运行不会撑满磁盘」的保障,覆盖 pruneReaderContentCache 两条分支。
|
||||
func TestContentCachePruneByTTLAndQuota(t *testing.T) {
|
||||
svc, srv, book := prepareCacheTestBook(t)
|
||||
ctx := t.Context()
|
||||
src := mustSource(t, svc, srv.URL)
|
||||
|
||||
// 造 4 条缓存,其中 1 条已过期。
|
||||
for i := 1; i <= 4; i++ {
|
||||
ch := model.ReaderChapter{
|
||||
Index: i - 1, Title: fmt.Sprintf("第 %d 章", i),
|
||||
URL: fmt.Sprintf("%s/book/1/c%d.html", srv.URL, i),
|
||||
}
|
||||
svc.saveCachedContent(ctx, src, book, ch, &ChapterContent{
|
||||
Type: "text", Content: strings.Repeat("正", 200),
|
||||
})
|
||||
}
|
||||
rows, err := svc.repo.ListContentCacheByBook(ctx, contentBookKey(src.ID, book.BookURL))
|
||||
if err != nil || len(rows) != 4 {
|
||||
t.Fatalf("应有 4 条缓存: n=%d err=%v", len(rows), err)
|
||||
}
|
||||
// 把第 1 条标记为已过期。
|
||||
expired := rows[0]
|
||||
expired.ExpiresAt = time.Now().Add(-time.Hour).Unix()
|
||||
if err := svc.repo.UpsertContentCache(ctx, &expired); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
svc.PruneContentCache(ctx)
|
||||
rows, err = svc.repo.ListContentCacheByBook(ctx, contentBookKey(src.ID, book.BookURL))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, row := range rows {
|
||||
if row.ChapterKey == expired.ChapterKey {
|
||||
t.Fatal("过期条目应被清理")
|
||||
}
|
||||
}
|
||||
|
||||
// 容量配额:把上限压到远小于现有体积,触发 LRU 淘汰。
|
||||
var total int64
|
||||
for _, row := range rows {
|
||||
total += row.SizeBytes
|
||||
}
|
||||
if total <= 0 {
|
||||
t.Fatal("缓存体积应大于 0")
|
||||
}
|
||||
svc.cfg.Cache.ReaderContentMaxSizeMB = 0 // 0 = 不限,先确认不误删
|
||||
svc.PruneContentCache(ctx)
|
||||
after, _ := svc.repo.ListContentCacheByBook(ctx, contentBookKey(src.ID, book.BookURL))
|
||||
if len(after) != len(rows) {
|
||||
t.Fatalf("未超配额不应淘汰: before=%d after=%d", len(rows), len(after))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
package reader
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"golang.org/x/text/encoding/simplifiedchinese"
|
||||
)
|
||||
|
||||
// 本文件:拷贝系列轻小说源的端到端回归。
|
||||
//
|
||||
// 合成一个「照搬拷贝轻小说写法」的书源,一次覆盖四个修复点:
|
||||
// 1. 搜索列表规则 `results.list.[*]`(legado 的 `.[*]` 形式);
|
||||
// 2. 目录规则用 `new Packages.java.util.LinkedHashMap()/ArrayList()` 拼装;
|
||||
// 3. 正文规则用 `java.cacheFile(url)` 取整卷文本;
|
||||
// 4. 正文是 GBK 文本——必须走 Raw 原始字节通道落盘,否则会被 UTF-8 解码成乱码。
|
||||
|
||||
func TestCopyLightNovelSourceEndToEnd(t *testing.T) {
|
||||
var srv *httptest.Server
|
||||
gbkBody, err := simplifiedchinese.GBK.NewEncoder().Bytes([]byte("第一章 开始\n第二章 继续"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mux := http.NewServeMux()
|
||||
// 搜索:Django REST 风格的 JSON。q_type/_update 参数照抄真实源。
|
||||
mux.HandleFunc("/api/search", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"code":200,"results":{"list":[
|
||||
{"name":"转生成为史莱姆","path_word":"slime","author":[{"name":"作者甲"}],"cover":"https://img.example.com/1.jpg"}
|
||||
]}}`))
|
||||
})
|
||||
mux.HandleFunc("/api/book/", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"code":200,"results":{"book":{"name":"转生成为史莱姆","path_word":"slime"}}}`))
|
||||
})
|
||||
mux.HandleFunc("/api/volumes", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"code":200,"results":{"list":[{"name":"第一卷","id":7}]}}`))
|
||||
})
|
||||
mux.HandleFunc("/api/volume/", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = fmt.Fprintf(w, `{"code":200,"results":{"volume":{"txt_addr":"%s/api/txt/7","txt_encoding":"GBK"}}}`, srv.URL)
|
||||
})
|
||||
mux.HandleFunc("/api/txt/", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
_, _ = w.Write(gbkBody)
|
||||
})
|
||||
srv = httptest.NewServer(mux)
|
||||
defer srv.Close()
|
||||
|
||||
source := map[string]any{
|
||||
"bookSourceName": "合成拷贝轻小说源",
|
||||
"bookSourceUrl": srv.URL,
|
||||
"enabled": true,
|
||||
"enabledExplore": true,
|
||||
"searchUrl": srv.URL + "/api/search?limit=20&q_type=&q={{key}}&_update=true",
|
||||
"ruleSearch": map[string]any{
|
||||
"bookList": "results.list.[*]",
|
||||
"name": "$.name",
|
||||
"author": "$.author[*].name",
|
||||
"coverUrl": "$.cover",
|
||||
"bookUrl": srv.URL + "/api/book/{{$.path_word}}?_update=true",
|
||||
},
|
||||
"ruleBookInfo": map[string]any{
|
||||
"init": "$.results.book",
|
||||
"name": "$.name",
|
||||
"intro": "$.brief",
|
||||
"tocUrl": srv.URL + "/api/volumes?book={{$.path_word}}&_update=true",
|
||||
},
|
||||
"ruleToc": map[string]any{
|
||||
"chapterList": `<js>
|
||||
function mk(n, u) { var m = new Packages.java.util.LinkedHashMap(); m.put('name', n); m.put('url', u); return m; }
|
||||
var base = '` + srv.URL + `/api/volume/';
|
||||
var out = new Packages.java.util.ArrayList();
|
||||
var vols = JSON.parse(result).results.list;
|
||||
for (var i = 0; i < vols.length; i++) { out.add(mk(vols[i].name, base + vols[i].id + '?_update=true')); }
|
||||
out;</js>`,
|
||||
"chapterName": "$.name",
|
||||
"chapterUrl": "$.url",
|
||||
},
|
||||
"ruleContent": map[string]any{
|
||||
"content": `@js:java.cacheFile(java.getString('$..txt_addr'))`,
|
||||
},
|
||||
}
|
||||
raw, err := json.Marshal(source)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
svc, _ := newLoginTestService(t)
|
||||
svc.cfg.Cache.CacheDir = t.TempDir()
|
||||
|
||||
res := svc.SmokeSource(t.Context(), string(raw), "史莱姆")
|
||||
if !res.OK {
|
||||
t.Fatalf("链路失败于 %s: %s\n%s", res.FailedAt, res.Error, joinLogs(res))
|
||||
}
|
||||
if res.SearchHits != 1 {
|
||||
t.Fatalf("搜索命中=%d, want 1(`.[*]` 路径修复)", res.SearchHits)
|
||||
}
|
||||
if res.Chapters != 1 {
|
||||
t.Fatalf("章节数=%d, want 1(Packages 修复)", res.Chapters)
|
||||
}
|
||||
if res.ContentLen == 0 {
|
||||
t.Fatalf("正文长度=0(cacheFile/Raw 修复未生效)\n%s", joinLogs(res))
|
||||
}
|
||||
if !strings.Contains(joinLogs(res), "第一章") {
|
||||
t.Fatalf("正文未正确解码(GBK 落盘被 utf-8 破坏)\n%s", joinLogs(res))
|
||||
}
|
||||
}
|
||||
|
||||
func joinLogs(res *SmokeChainResult) string {
|
||||
var b strings.Builder
|
||||
for _, l := range res.Logs {
|
||||
fmt.Fprintf(&b, "[%s/%s] %s\n", l.Stage, l.Level, l.Message)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -26,14 +26,20 @@ type AnalyzeRule struct {
|
||||
chapterVars map[string]string
|
||||
bookVars map[string]string
|
||||
vars map[string]string
|
||||
chapterTitle string
|
||||
chapterIndex int
|
||||
bookName string
|
||||
// sourceDefaults 书源 JSON 的 variables 字段(作者设定的默认值)。
|
||||
// 它只作为 @get 的兜底:显式保存过的书源变量与会话内 @put 的值优先级更高。
|
||||
sourceDefaults map[string]string
|
||||
chapterTitle string
|
||||
chapterIndex int
|
||||
bookName string
|
||||
// bookMeta 书籍元数据(对应 legado 规则 JS 里的 Book 实体字段)。
|
||||
bookMeta map[string]any
|
||||
// bookCustom 书籍自定义变量(对应 legado Book.variableMap),
|
||||
// 由规则 JS 的 book.getVariable / book.putVariable 读写。
|
||||
bookCustom map[string]string
|
||||
// bookVarPutter 由服务层注册:book.putVariable 写入后触发,
|
||||
// 用于把变量变更持久化回书架记录(对应 legado 的 Book.upVariable)。
|
||||
bookVarPutter func()
|
||||
// bookTypeOverride 书源在规则 JS 里给 book.type 赋的值
|
||||
// (听书/漫画/短剧源靠它声明书籍类型),由服务层读回。
|
||||
bookTypeOverride *int
|
||||
@@ -120,6 +126,17 @@ func (a *AnalyzeRule) SetBookCustomVars(vars map[string]string) {
|
||||
}
|
||||
}
|
||||
|
||||
// BookCustomVars 返回书籍自定义变量的当前值(可能被 book.putVariable 改过),
|
||||
// 服务层据此把变更写回书架记录。没有书籍上下文时为 nil。
|
||||
func (a *AnalyzeRule) BookCustomVars() map[string]string {
|
||||
return a.bookCustom
|
||||
}
|
||||
|
||||
// RegisterBookVariablePutter 注册书籍变量变更回调(book.putVariable 写入后触发)。
|
||||
func (a *AnalyzeRule) RegisterBookVariablePutter(putter func()) {
|
||||
a.bookVarPutter = putter
|
||||
}
|
||||
|
||||
// SetBookType 记录书源声明的书籍类型(legado Book.type)。
|
||||
func (a *AnalyzeRule) SetBookType(t int) {
|
||||
v := t
|
||||
@@ -154,6 +171,14 @@ func (a *AnalyzeRule) SetBookContext(name string, vars map[string]string) {
|
||||
}
|
||||
}
|
||||
|
||||
// SetSourceDefaults 注入书源 variables 字段的默认值(@get 的兜底)。
|
||||
// 显式保存过的源变量与会话内 @put 的值优先级都高于它。
|
||||
func (a *AnalyzeRule) SetSourceDefaults(vars map[string]string) {
|
||||
if len(vars) > 0 {
|
||||
a.sourceDefaults = vars
|
||||
}
|
||||
}
|
||||
|
||||
// SetSourceVariables 注入书源级变量读写(source.variableMap)。
|
||||
func (a *AnalyzeRule) SetSourceVariables(getter func(key string) string, putter func(key, value string)) {
|
||||
a.sourceGetter = getter
|
||||
@@ -172,13 +197,19 @@ func jsonpathGet(path string, root any) (v any, err error) {
|
||||
|
||||
// ─── 变量存取(对应 put/get) ───────────────────────────────────────────────
|
||||
|
||||
// Put 对应 put(key, value):chapter → book → 局部 → source。
|
||||
// Put 对应 put(key, value):chapter → book → 局部 → 书源持久变量。
|
||||
//
|
||||
// 「书源持久变量」是 @put 在 legado 里的真实落点(BaseSource.putVariable):
|
||||
// 写进去的值跨请求可见。书源默认变量(variables 字段)与书籍自定义变量都
|
||||
// 不是这个通道,所以最后才回退到 sourcePutter,而不是写一次性的 bookVars。
|
||||
func (a *AnalyzeRule) Put(key, value string) string {
|
||||
switch {
|
||||
case a.chapterVars != nil:
|
||||
a.chapterVars[key] = value
|
||||
case a.bookVars != nil:
|
||||
a.bookVars[key] = value
|
||||
case a.sourcePutter != nil:
|
||||
a.sourcePutter(key, value)
|
||||
default:
|
||||
if a.vars == nil {
|
||||
a.vars = map[string]string{}
|
||||
@@ -189,6 +220,7 @@ func (a *AnalyzeRule) Put(key, value string) string {
|
||||
}
|
||||
|
||||
// Get 对应 get(key):特殊键 bookName/title 优先取上下文。
|
||||
// 查找顺序 chapter → book → book 变量之上的显式源变量 → 书源默认变量。
|
||||
func (a *AnalyzeRule) Get(key string) string {
|
||||
switch key {
|
||||
case "bookName":
|
||||
@@ -212,6 +244,9 @@ func (a *AnalyzeRule) Get(key string) string {
|
||||
return v
|
||||
}
|
||||
}
|
||||
if v, ok := a.sourceDefaults[key]; ok {
|
||||
return v
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
|
||||
@@ -10,6 +10,8 @@ import (
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"hash"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/dop251/goja"
|
||||
@@ -245,6 +247,12 @@ func newJavaObject(vm *goja.Runtime, r *JSRunner, a *AnalyzeRule) *goja.Object {
|
||||
if len(call.Arguments) == 0 || goja.IsUndefined(call.Arguments[0]) || goja.IsNull(call.Arguments[0]) {
|
||||
return vm.ToValue("")
|
||||
}
|
||||
// 字节参数可能是 ArrayBuffer / typed array:这类值的 Export() 返回属性 map,
|
||||
// 只有 ExportTo 才能取回字节,旧写法会退化成 String(value) 得到 "[object ArrayBuffer]"。
|
||||
var buf []byte
|
||||
if err := vm.ExportTo(call.Arguments[0], &buf); err == nil {
|
||||
return vm.ToValue(string(buf))
|
||||
}
|
||||
if buf, ok := call.Arguments[0].Export().([]byte); ok {
|
||||
return vm.ToValue(string(buf))
|
||||
}
|
||||
@@ -435,6 +443,95 @@ func newJavaObject(vm *goja.Runtime, r *JSRunner, a *AnalyzeRule) *goja.Object {
|
||||
return goja.Null()
|
||||
})
|
||||
|
||||
// ── 本地文件(对应 legado JsExtensions 的缓存/文件系列) ──
|
||||
//
|
||||
// 服务端把文件限制在自己的缓存目录(<cache>/reader/files)内,
|
||||
// 书源只能用相对路径(允许以 / 开头,downloadFile 的返回值即此形态)。
|
||||
set("downloadFile", func(call goja.FunctionCall) goja.Value {
|
||||
// 旧签名 downloadFile(contentHex, url):把 hex 字符串落成文件。
|
||||
if len(call.Arguments) >= 2 && !goja.IsUndefined(call.Arguments[1]) && !goja.IsNull(call.Arguments[1]) {
|
||||
url := stringArg(call, 1)
|
||||
raw, err := hex.DecodeString(strings.TrimSpace(stringArg(call, 0)))
|
||||
if err != nil {
|
||||
bridgeErr("downloadFile", err)
|
||||
}
|
||||
rel, err := writeCacheFile(r.cfg.CacheDir, cacheFileName(url), raw)
|
||||
if err != nil {
|
||||
bridgeErr("downloadFile", err)
|
||||
}
|
||||
return vm.ToValue(rel)
|
||||
}
|
||||
rel, err := downloadToCache(r, stringArg(call, 0))
|
||||
if err != nil {
|
||||
bridgeErr("downloadFile", err)
|
||||
}
|
||||
return vm.ToValue(rel)
|
||||
})
|
||||
// cacheFile(url): 下载并缓存文本文件,返回文件**内容**(对应 legado 语义)。
|
||||
set("cacheFile", func(call goja.FunctionCall) goja.Value {
|
||||
url := stringArg(call, 0)
|
||||
if dir := readerFilesDir(r.cfg.CacheDir); dir != "" {
|
||||
if b, err := os.ReadFile(filepath.Join(dir, cacheFileName(url))); err == nil && len(b) > 0 {
|
||||
return vm.ToValue(decodeTextAuto(b))
|
||||
}
|
||||
}
|
||||
rel, err := downloadToCache(r, url)
|
||||
if err != nil {
|
||||
bridgeErr("cacheFile", err)
|
||||
}
|
||||
full, err := resolveCacheFilePath(r.cfg.CacheDir, rel)
|
||||
if err != nil {
|
||||
bridgeErr("cacheFile", err)
|
||||
}
|
||||
b, err := os.ReadFile(full)
|
||||
if err != nil {
|
||||
bridgeErr("cacheFile", err)
|
||||
}
|
||||
return vm.ToValue(decodeTextAuto(b))
|
||||
})
|
||||
set("readFile", func(call goja.FunctionCall) goja.Value {
|
||||
full, err := resolveCacheFilePath(r.cfg.CacheDir, stringArg(call, 0))
|
||||
if err != nil {
|
||||
return goja.Null()
|
||||
}
|
||||
b, err := os.ReadFile(full)
|
||||
if err != nil {
|
||||
return goja.Null()
|
||||
}
|
||||
return vm.ToValue(vm.NewArrayBuffer(b))
|
||||
})
|
||||
set("readTxtFile", func(call goja.FunctionCall) goja.Value {
|
||||
full, err := resolveCacheFilePath(r.cfg.CacheDir, stringArg(call, 0))
|
||||
if err != nil {
|
||||
return vm.ToValue("")
|
||||
}
|
||||
b, err := os.ReadFile(full)
|
||||
if err != nil {
|
||||
return vm.ToValue("")
|
||||
}
|
||||
if cs := stringArgOr(call, 1, ""); cs != "" {
|
||||
if s, derr := DecodeBytes(b, cs); derr == nil {
|
||||
return vm.ToValue(s)
|
||||
}
|
||||
}
|
||||
return vm.ToValue(decodeTextAuto(b))
|
||||
})
|
||||
set("deleteFile", func(call goja.FunctionCall) goja.Value {
|
||||
full, err := resolveCacheFilePath(r.cfg.CacheDir, stringArg(call, 0))
|
||||
if err != nil {
|
||||
return vm.ToValue(false)
|
||||
}
|
||||
return vm.ToValue(os.Remove(full) == nil)
|
||||
})
|
||||
// getFile(path): 返回 File 对象(提供书源常用的 exists/length/name/delete/readText)。
|
||||
set("getFile", func(call goja.FunctionCall) goja.Value {
|
||||
full, err := resolveCacheFilePath(r.cfg.CacheDir, stringArg(call, 0))
|
||||
if err != nil {
|
||||
full = ""
|
||||
}
|
||||
return newFileObject(vm, full, stringArg(call, 0))
|
||||
})
|
||||
|
||||
// ── 书源会话状态(legado 中 `java` 与 `source` 是同一对象) ──
|
||||
if r.state != nil {
|
||||
bindSourceState(vm, set, r.state, r.cfg.SourceProps)
|
||||
@@ -540,13 +637,18 @@ func newJavaObject(vm *goja.Runtime, r *JSRunner, a *AnalyzeRule) *goja.Object {
|
||||
set(name, unsupported(name, "需要无头浏览器(WebView)"))
|
||||
}
|
||||
for _, name := range []string{
|
||||
"importScript", "cacheFile", "downloadFile", "readFile", "readTxtFile", "deleteFile",
|
||||
"unzipFile", "un7zFile", "unrarFile", "unArchiveFile", "getTxtInFolder",
|
||||
"getZipStringContent", "getZipByteArrayContent",
|
||||
"getRarStringContent", "get7zStringContent",
|
||||
} {
|
||||
set(name, unsupported(name, "需要本地文件/压缩包能力"))
|
||||
set(name, unsupported(name, "需要压缩包解压能力"))
|
||||
}
|
||||
// importScript 在 legado 里是「下载 JS 文件并 eval」,服务端可做但会引入
|
||||
// 任意脚本执行面,暂不支持,保持明确报错。
|
||||
set("importScript", unsupported("importScript", "服务端不支持动态加载外部脚本"))
|
||||
|
||||
// 字体混淆还原(queryTTF / queryBase64TTF / replaceFont)。
|
||||
r.installFontBridge(vm, set)
|
||||
|
||||
return o
|
||||
}
|
||||
|
||||
@@ -0,0 +1,209 @@
|
||||
package rule
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/dop251/goja"
|
||||
)
|
||||
|
||||
// 本文件实现 legado JsExtensions 里的「本地文件」能力
|
||||
// (cacheFile / downloadFile / readFile / readTxtFile / deleteFile / getFile)。
|
||||
//
|
||||
// 与 legado 的语义保持一致:
|
||||
// - downloadFile(url) 下载到缓存目录,返回「相对缓存根目录」的路径;
|
||||
// - cacheFile(url) 下载(带缓存)后返回文件的**文本内容**(不是路径);
|
||||
// - readFile/readTxtFile/deleteFile/getFile 接受相对路径(允许以 / 开头),
|
||||
// 一律解析到缓存目录下。
|
||||
//
|
||||
// 服务端把缓存根目录固定为 <cache>/reader/files,并且只允许访问该目录内的文件,
|
||||
// 避免书源通过 `../../` 之类读到宿主上的任意文件。
|
||||
|
||||
// filesSubdir 书源文件缓存目录(相对 ReaderService 的缓存根目录)。
|
||||
const filesSubdir = "reader/files"
|
||||
|
||||
// readerFilesDir 返回书源文件缓存目录;未配置缓存目录时返回空串。
|
||||
func readerFilesDir(cacheDir string) string {
|
||||
if strings.TrimSpace(cacheDir) == "" {
|
||||
return ""
|
||||
}
|
||||
return filepath.Join(cacheDir, "reader", "files")
|
||||
}
|
||||
|
||||
// cacheFileName 计算某个 URL 在缓存目录里的文件名(对应 legado 的
|
||||
// `${md5Encode16(url)}.${type}`:md5 十六进制取中间 16 位,保留 URL 后缀)。
|
||||
func cacheFileName(rawURL string) string {
|
||||
name := md5Hex(rawURL, true)
|
||||
if ext := urlFileSuffix(rawURL); ext != "" {
|
||||
name += "." + ext
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
// urlFileSuffix 取 URL 路径部分的后缀(不含 `.`),只接受字母数字(长度 1~8)。
|
||||
func urlFileSuffix(rawURL string) string {
|
||||
u := rawURL
|
||||
if i := strings.IndexAny(u, "?#"); i >= 0 {
|
||||
u = u[:i]
|
||||
}
|
||||
if i := strings.LastIndex(u, "/"); i >= 0 {
|
||||
u = u[i+1:]
|
||||
}
|
||||
dot := strings.LastIndex(u, ".")
|
||||
if dot < 0 || dot == len(u)-1 {
|
||||
return ""
|
||||
}
|
||||
ext := u[dot+1:]
|
||||
if len(ext) > 8 {
|
||||
return ""
|
||||
}
|
||||
for i := 0; i < len(ext); i++ {
|
||||
c := ext[i]
|
||||
if !(c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9') {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
return strings.ToLower(ext)
|
||||
}
|
||||
|
||||
// resolveCacheFilePath 把书源给的相对路径解析成缓存目录下的绝对路径。
|
||||
// 允许以 `/` 开头(downloadFile 的返回值就是这种形态),但拒绝越出缓存目录。
|
||||
func resolveCacheFilePath(cacheDir, p string) (string, error) {
|
||||
dir := readerFilesDir(cacheDir)
|
||||
if dir == "" {
|
||||
return "", fmt.Errorf("未配置缓存目录")
|
||||
}
|
||||
p = strings.TrimSpace(p)
|
||||
if p == "" {
|
||||
return "", fmt.Errorf("文件路径为空")
|
||||
}
|
||||
rel := filepath.Clean(filepath.FromSlash(strings.Trim(p, `/\`)))
|
||||
if rel == "." || rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) {
|
||||
return "", fmt.Errorf("非法文件路径: %s", p)
|
||||
}
|
||||
return filepath.Join(dir, rel), nil
|
||||
}
|
||||
|
||||
// writeCacheFile 把字节写入缓存目录,返回相对路径(以 / 开头,对应 legado)。
|
||||
func writeCacheFile(cacheDir, name string, data []byte) (string, error) {
|
||||
dir := readerFilesDir(cacheDir)
|
||||
if dir == "" {
|
||||
return "", fmt.Errorf("未配置缓存目录")
|
||||
}
|
||||
if err := os.MkdirAll(dir, 0o750); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, name), data, 0o600); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "/" + name, nil
|
||||
}
|
||||
|
||||
// downloadToCache 下载 URL 并落到缓存目录(已存在则直接复用),
|
||||
// 返回相对缓存根目录的路径。下载走书源会话,因此会带上书源请求头与 Cookie。
|
||||
func downloadToCache(r *JSRunner, rawURL string) (string, error) {
|
||||
dir := readerFilesDir(r.cfg.CacheDir)
|
||||
if dir == "" {
|
||||
return "", fmt.Errorf("未配置缓存目录")
|
||||
}
|
||||
name := cacheFileName(rawURL)
|
||||
full := filepath.Join(dir, name)
|
||||
if st, err := os.Stat(full); err == nil && st.Size() > 0 {
|
||||
return "/" + name, nil
|
||||
}
|
||||
req, err := ParseAnalyzeUrlWithJS(rawURL, "", 0, r.cfg.BaseURL, r)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if req.Unsupported != nil {
|
||||
return "", req.Unsupported
|
||||
}
|
||||
// Raw:响应按原始字节返回,避免 charset 解码破坏二进制/非 UTF-8 文本。
|
||||
// 同时清掉 HexBody:URL 选项里的 `type` 在 legado 是文件后缀提示,
|
||||
// 这里不该把下载内容按 hex 字符串落盘。
|
||||
req.Raw = true
|
||||
req.HexBody = false
|
||||
body, _, code, err := r.fetch(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if code >= 400 {
|
||||
return "", fmt.Errorf("下载失败 HTTP %d: %s", code, rawURL)
|
||||
}
|
||||
return writeCacheFile(r.cfg.CacheDir, name, []byte(body))
|
||||
}
|
||||
|
||||
// decodeTextAuto 解码文本文件:UTF-8 BOM → 去 BOM;合法 UTF-8 → 原样;
|
||||
// 否则按 GBK 解(中文站点/书源最常见的另一种编码),失败再原样返回。
|
||||
func decodeTextAuto(b []byte) string {
|
||||
if len(b) >= 3 && b[0] == 0xEF && b[1] == 0xBB && b[2] == 0xBF {
|
||||
return string(b[3:])
|
||||
}
|
||||
if utf8.Valid(b) {
|
||||
return string(b)
|
||||
}
|
||||
if s, err := DecodeBytes(b, "gbk"); err == nil && s != "" {
|
||||
return s
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// fileSuffixFromName 取路径的最后一段(getFile 的 name 语义)。
|
||||
func fileSuffixFromName(p string) string {
|
||||
return path.Base(strings.ReplaceAll(p, "\\", "/"))
|
||||
}
|
||||
|
||||
// newFileObject 构造 `java.getFile(path)` 返回的 File 等价物,
|
||||
// 提供书源常用的 exists / length / name / delete / readText。
|
||||
func newFileObject(vm *goja.Runtime, full, raw string) *goja.Object {
|
||||
o := vm.NewObject()
|
||||
set := func(k string, v any) { _ = o.Set(k, v) }
|
||||
set("path", full)
|
||||
set("name", fileSuffixFromName(raw))
|
||||
set("exists", func(goja.FunctionCall) goja.Value {
|
||||
if full == "" {
|
||||
return vm.ToValue(false)
|
||||
}
|
||||
_, err := os.Stat(full)
|
||||
return vm.ToValue(err == nil)
|
||||
})
|
||||
set("isDirectory", func(goja.FunctionCall) goja.Value {
|
||||
if full == "" {
|
||||
return vm.ToValue(false)
|
||||
}
|
||||
st, err := os.Stat(full)
|
||||
return vm.ToValue(err == nil && st.IsDir())
|
||||
})
|
||||
set("length", func(goja.FunctionCall) goja.Value {
|
||||
if full == "" {
|
||||
return vm.ToValue(0)
|
||||
}
|
||||
st, err := os.Stat(full)
|
||||
if err != nil {
|
||||
return vm.ToValue(0)
|
||||
}
|
||||
return vm.ToValue(st.Size())
|
||||
})
|
||||
set("delete", func(goja.FunctionCall) goja.Value {
|
||||
if full == "" {
|
||||
return vm.ToValue(false)
|
||||
}
|
||||
return vm.ToValue(os.Remove(full) == nil)
|
||||
})
|
||||
set("readText", func(goja.FunctionCall) goja.Value {
|
||||
if full == "" {
|
||||
return vm.ToValue("")
|
||||
}
|
||||
b, err := os.ReadFile(full)
|
||||
if err != nil {
|
||||
return vm.ToValue("")
|
||||
}
|
||||
return vm.ToValue(decodeTextAuto(b))
|
||||
})
|
||||
set("toString", func(goja.FunctionCall) goja.Value { return vm.ToValue(full) })
|
||||
return o
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
package rule
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"golang.org/x/text/encoding/simplifiedchinese"
|
||||
)
|
||||
|
||||
// 本文件:java 文件接口(cacheFile / downloadFile / readTxtFile / deleteFile)
|
||||
// 的语义回归。对应 legado JsExtensions:
|
||||
// - cacheFile(url) → 返回文件**文本内容**(不是路径);
|
||||
// - downloadFile(url) → 返回相对缓存根目录的路径;
|
||||
// - readTxtFile(path[, charset]) → 返回文本;
|
||||
// - 只允许访问缓存目录内的文件。
|
||||
|
||||
func newFileTestRunner(cacheDir, baseURL string, hits *int32) *JSRunner {
|
||||
return NewJSRunner(JSConfig{
|
||||
CacheDir: cacheDir,
|
||||
BaseURL: baseURL,
|
||||
Fetch: func(req *Request) (string, string, int, error) {
|
||||
if hits != nil {
|
||||
atomic.AddInt32(hits, 1)
|
||||
}
|
||||
resp, err := http.Get(req.URL) //nolint:gosec // 测试内固定 httptest 地址
|
||||
if err != nil {
|
||||
return "", "", 0, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
return string(b), req.URL, resp.StatusCode, nil
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func TestCacheFileReturnsTextContent(t *testing.T) {
|
||||
var hits int32
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write([]byte("第一章 开始\n第二章 继续"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
r := newFileTestRunner(t.TempDir(), srv.URL, &hits)
|
||||
ar := NewAnalyzeRule()
|
||||
v, err := r.Run(ar, "java.cacheFile('"+srv.URL+"/vol.txt')", nil, "")
|
||||
if err != nil {
|
||||
t.Fatalf("cacheFile 失败: %v", err)
|
||||
}
|
||||
if got := anyToString(v); got != "第一章 开始\n第二章 继续" {
|
||||
t.Fatalf("cacheFile 应返回文本内容,实际 %q", got)
|
||||
}
|
||||
// 第二次应命中缓存,不再发起请求。
|
||||
if _, err := r.Run(ar, "java.cacheFile('"+srv.URL+"/vol.txt')", nil, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := atomic.LoadInt32(&hits); n != 1 {
|
||||
t.Fatalf("重复调用应命中缓存(请求次数=%d)", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCacheFileDecodesGBK(t *testing.T) {
|
||||
gbk, err := simplifiedchinese.GBK.NewEncoder().Bytes([]byte("中文内容测试"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write(gbk)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
r := newFileTestRunner(t.TempDir(), srv.URL, nil)
|
||||
v, err := r.Run(NewAnalyzeRule(), "java.cacheFile('"+srv.URL+"/gbk.txt')", nil, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := anyToString(v); got != "中文内容测试" {
|
||||
t.Fatalf("GBK 文本未正确解码: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadFileThenReadTxtFile(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write([]byte("正文内容"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
cacheDir := t.TempDir()
|
||||
r := newFileTestRunner(cacheDir, srv.URL, nil)
|
||||
ar := NewAnalyzeRule()
|
||||
|
||||
rawPath, err := r.Run(ar, "java.downloadFile('"+srv.URL+"/a.txt')", nil, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rel := anyToString(rawPath)
|
||||
if !strings.HasPrefix(rel, "/") || !strings.HasSuffix(rel, ".txt") {
|
||||
t.Fatalf("downloadFile 应返回相对缓存路径,实际 %q", rel)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(cacheDir, "reader", "files", strings.TrimPrefix(rel, "/"))); err != nil {
|
||||
t.Fatalf("文件未落到缓存目录: %v", err)
|
||||
}
|
||||
|
||||
v, err := r.Run(ar, "java.readTxtFile('"+rel+"')", nil, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := anyToString(v); got != "正文内容" {
|
||||
t.Fatalf("readTxtFile = %q", got)
|
||||
}
|
||||
|
||||
del, err := r.Run(ar, "String(java.deleteFile('"+rel+"'))", nil, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if anyToString(del) != "true" {
|
||||
t.Fatalf("deleteFile 应返回 true,实际 %q", anyToString(del))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileOpsRejectPathTraversal(t *testing.T) {
|
||||
cacheDir := t.TempDir()
|
||||
secret := filepath.Join(cacheDir, "secret.txt")
|
||||
if err := os.WriteFile(secret, []byte("不该被读到"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := newFileTestRunner(cacheDir, "", nil)
|
||||
ar := NewAnalyzeRule()
|
||||
|
||||
for _, p := range []string{"/../../secret.txt", "../secret.txt", "/../reader/files/../../secret.txt"} {
|
||||
v, err := r.Run(ar, "java.readTxtFile('"+p+"')", nil, "")
|
||||
if err != nil {
|
||||
t.Fatalf("%s 不应报错: %v", p, err)
|
||||
}
|
||||
if got := anyToString(v); strings.Contains(got, "不该被读到") {
|
||||
t.Fatalf("越权读到了缓存目录外的文件(%s)", p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileOpsWithoutCacheDirFail(t *testing.T) {
|
||||
r := NewJSRunner(JSConfig{})
|
||||
_, err := r.Run(NewAnalyzeRule(), "java.cacheFile('https://example.com/a.txt')", nil, "")
|
||||
if err == nil || !strings.Contains(err.Error(), "缓存目录") {
|
||||
t.Fatalf("未配置缓存目录时应明确报错,实际 %v", err)
|
||||
}
|
||||
}
|
||||
@@ -72,6 +72,12 @@ type JSConfig struct {
|
||||
// Browser 宿主浏览器实现(java.startBrowser / startBrowserAwait)。
|
||||
// nil 时这两个函数抛出不支持错误。
|
||||
Browser BrowserHost
|
||||
// CacheDir 书源文件缓存根目录(java.downloadFile / cacheFile 落盘用)。
|
||||
// 为空时这些函数抛出明确错误。
|
||||
CacheDir string
|
||||
// FetchBytes 拉原始字节(queryTTF 的 URL 形态、图片解密前的取图)。
|
||||
// 由服务层注入,复用书源 header / Cookie / 限速 / 重试。
|
||||
FetchBytes func(absURL string) ([]byte, error)
|
||||
// Ctx 本次执行的可取消上下文,透传给 BrowserHost 的等待。
|
||||
Ctx context.Context
|
||||
}
|
||||
@@ -193,6 +199,18 @@ func (r *JSRunner) fetch(req *Request) (string, string, int, error) {
|
||||
return r.cfg.Fetch(req)
|
||||
}
|
||||
|
||||
// cacheNamespace cache 对象的命名空间:优先取书源 URL,退回 BaseURL。
|
||||
func (c JSConfig) cacheNamespace() string {
|
||||
if c.SourceProps != nil {
|
||||
if v, ok := c.SourceProps["bookSourceUrl"]; ok {
|
||||
if s, ok := v.(string); ok && strings.TrimSpace(s) != "" {
|
||||
return s
|
||||
}
|
||||
}
|
||||
}
|
||||
return c.BaseURL
|
||||
}
|
||||
|
||||
// NewJSRunner 创建运行时:注入全局对象 cookie / cache / source,并执行 jsLib。
|
||||
func NewJSRunner(cfg JSConfig) *JSRunner {
|
||||
vm := goja.New()
|
||||
@@ -205,17 +223,107 @@ func NewJSRunner(cfg JSConfig) *JSRunner {
|
||||
}
|
||||
r := &JSRunner{vm: vm, cfg: cfg, vars: map[string]string{}, state: state}
|
||||
vm.Set("cookie", newCookieObject(vm, state))
|
||||
vm.Set("cache", newCacheObject(vm))
|
||||
// cache 按书源命名空间隔离:namespace 取书源 URL,持久层落 CacheDir/reader-js-cache。
|
||||
vm.Set("cache", newCacheObject(vm, cfg.cacheNamespace(), cfg.CacheDir))
|
||||
// source 必须在 jsLib 之前注入:jsLib 的 getVariable/BaseUrl 依赖它。
|
||||
srcObj := newSourceObject(vm, state, cfg.SourceProps)
|
||||
vm.Set("source", srcObj)
|
||||
// java 也要在 jsLib 之前就位(jsLib 顶层可能引用 java.*)。
|
||||
r.installJava(nil)
|
||||
// Packages(Java 类命名空间)同样要在 jsLib 之前注入:不少书源在 jsLib 或
|
||||
// 目录/正文规则里用 `new Packages.java.util.LinkedHashMap()` 这类写法。
|
||||
r.installPackages()
|
||||
// 部分源把 source 的方法也当 java 成员用(同一 Kotlin 对象暴露两份)。
|
||||
r.loadJSLib()
|
||||
return r
|
||||
}
|
||||
|
||||
// installPackages 注入 Java 风格类命名空间(Packages.java.util.*)。
|
||||
//
|
||||
// goja 是纯 JS 运行时,没有 Java。书源(尤其照搬 Java 教程写的)常用
|
||||
// `new Packages.java.util.ArrayList()` / `LinkedHashMap` / `HashMap` 来拼
|
||||
// 目录列表和请求头,缺了它整条规则直接 ReferenceError(拷贝漫画轻小说源的
|
||||
// 目录规则就是这么写的)。
|
||||
//
|
||||
// 实现要点:容器用「真正的 JS 数组 / 对象」,方法用 Object.defineProperty 定义为
|
||||
// 不可枚举属性。这样容器交给引擎侧(GetElements/GetString)时导出的是干净的
|
||||
// 数组([]any)或映射(map[string]any),而不会把 add/put 这些方法也当成元素或
|
||||
// 请求头带出去。
|
||||
func (r *JSRunner) installPackages() {
|
||||
if _, err := r.vm.RunString(packagesPreamble); err != nil {
|
||||
// 注入失败不影响其它规则,只是 Packages.* 不可用。
|
||||
r.jsLibErr = fmt.Errorf("Packages 环境初始化失败: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// packagesPreamble 定义全局 Packages(以及与之等价的 java.util / java.lang 常用类)。
|
||||
const packagesPreamble = `
|
||||
var Packages = (function () {
|
||||
function def(o, k, v) {
|
||||
Object.defineProperty(o, k, { value: v, enumerable: false, writable: true, configurable: true });
|
||||
}
|
||||
function makeMap() {
|
||||
var m = {};
|
||||
def(m, 'put', function (k, v) { m[k] = v; return v; });
|
||||
def(m, 'putAll', function (o) { if (o) { for (var k in o) { m[k] = o[k]; } } });
|
||||
def(m, 'get', function (k) { return m[k] === undefined ? null : m[k]; });
|
||||
def(m, 'getOrDefault', function (k, d) { return m[k] === undefined ? d : m[k]; });
|
||||
def(m, 'containsKey', function (k) { return Object.prototype.hasOwnProperty.call(m, k); });
|
||||
def(m, 'containsValue', function (v) { for (var k in m) { if (m[k] === v) { return true; } } return false; });
|
||||
def(m, 'remove', function (k) { var v = m[k]; delete m[k]; return v; });
|
||||
def(m, 'size', function () { return Object.keys(m).length; });
|
||||
def(m, 'isEmpty', function () { return Object.keys(m).length === 0; });
|
||||
def(m, 'clear', function () { for (var k in m) { delete m[k]; } });
|
||||
def(m, 'keySet', function () { return Object.keys(m); });
|
||||
def(m, 'values', function () { var a = []; for (var k in m) { a.push(m[k]); } return a; });
|
||||
def(m, 'entrySet', function () { var a = []; for (var k in m) { a.push({ key: k, value: m[k] }); } return a; });
|
||||
def(m, 'toString', function () { return JSON.stringify(m); });
|
||||
return m;
|
||||
}
|
||||
function makeList() {
|
||||
var a = [];
|
||||
def(a, 'add', function (x) { a.push(x); return true; });
|
||||
def(a, 'addAll', function (xs) { if (xs) { for (var i = 0; i < xs.length; i++) { a.push(xs[i]); } } return true; });
|
||||
def(a, 'get', function (i) { return a[i]; });
|
||||
def(a, 'size', function () { return a.length; });
|
||||
def(a, 'isEmpty', function () { return a.length === 0; });
|
||||
def(a, 'contains', function (x) { return a.indexOf(x) >= 0; });
|
||||
def(a, 'remove', function (i) { return a.splice(i, 1)[0]; });
|
||||
def(a, 'clear', function () { a.length = 0; });
|
||||
def(a, 'toArray', function () { return a.slice(); });
|
||||
def(a, 'toString', function () { return a.join(','); });
|
||||
return a;
|
||||
}
|
||||
var util = {};
|
||||
var mapNames = ['LinkedHashMap', 'HashMap', 'TreeMap', 'Hashtable', 'ConcurrentHashMap', 'LinkedTreeMap'];
|
||||
for (var i = 0; i < mapNames.length; i++) { util[mapNames[i]] = makeMap; }
|
||||
var listNames = ['ArrayList', 'LinkedList', 'Vector'];
|
||||
for (var j = 0; j < listNames.length; j++) { util[listNames[j]] = makeList; }
|
||||
util.HashSet = makeList;
|
||||
util.Arrays = { asList: function () { return Array.prototype.slice.call(arguments); } };
|
||||
util.Collections = {
|
||||
emptyList: function () { return []; },
|
||||
emptyMap: function () { return makeMap(); },
|
||||
singletonList: function (x) { return [x]; }
|
||||
};
|
||||
var lang = {
|
||||
String: function (v) { return v == null ? '' : String(v); },
|
||||
Integer: function (v) { return parseInt(v, 10) || 0; },
|
||||
Long: function (v) { return parseInt(v, 10) || 0; },
|
||||
Double: function (v) { return parseFloat(v) || 0; },
|
||||
Boolean: function (v) { return !!v; },
|
||||
StringBuilder: function () {
|
||||
var s = '';
|
||||
var o = {};
|
||||
def(o, 'append', function (x) { s += (x == null ? '' : x); return o; });
|
||||
def(o, 'toString', function () { return s; });
|
||||
return o;
|
||||
}
|
||||
};
|
||||
return { java: { util: util, lang: lang } };
|
||||
})();
|
||||
`
|
||||
|
||||
// installJava 安装/刷新本次执行可见的 java 对象(桥回指定解析器)。
|
||||
func (r *JSRunner) installJava(a *AnalyzeRule) {
|
||||
r.vm.Set("java", newJavaObject(r.vm, r, a))
|
||||
@@ -399,7 +507,7 @@ func (r *JSRunner) Run(a *AnalyzeRule, js string, result any, baseURL string) (a
|
||||
}
|
||||
vm.Set("nextChapterUrl", nil)
|
||||
|
||||
prog, err := compileCached(scopedRuleJS(js))
|
||||
prog, err := compileRuleJS(js)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("JS 编译失败: %w", err)
|
||||
}
|
||||
@@ -441,6 +549,80 @@ func scopedRuleJS(js string) string {
|
||||
return "{\n" + js + "\n}"
|
||||
}
|
||||
|
||||
// functionRuleJS 把规则 JS 包成函数体后求值。
|
||||
//
|
||||
// legado 允许书源规则用顶层 `return` 提前返回(拷贝漫画轻小说源的正文规则
|
||||
// 就在 if 分支里 `return '<img ...>'`),而 JS 脚本语法不允许顶层 return,
|
||||
// 只有函数体允许。这里作为块形式的兜底。
|
||||
func functionRuleJS(js string) string {
|
||||
return "(function(){\n" + js + "\n})()"
|
||||
}
|
||||
|
||||
// compileRuleJS 编译一段书源规则 JS。
|
||||
//
|
||||
// 优先用「块」形式:块的完成值就是最后一条语句的值,大量书源依赖它
|
||||
// (如 `@js:1+2` 直接产出 3),且块作用域能隔离顶层 let/const(见 scopedRuleJS)。
|
||||
// 块形式编译失败时退回「函数体」形式,兼容 legado 允许的顶层 return。
|
||||
func compileRuleJS(js string) (*goja.Program, error) {
|
||||
prog, err := compileCached(scopedRuleJS(js))
|
||||
if err == nil {
|
||||
return prog, nil
|
||||
}
|
||||
if prog2, err2 := compileCached(functionRuleJS(js)); err2 == nil {
|
||||
return prog2, nil
|
||||
}
|
||||
return nil, err // 两种形式都编译不过,返回块形式的错误(更贴近书源原文)
|
||||
}
|
||||
|
||||
// RunImageDecode 执行图片字节二次解密 JS(coverDecodeJs / ruleContent.imageDecode)。
|
||||
//
|
||||
// 对应 legado ImageUtils.getDecodeResult:绑定 result=图片字节、src=图片地址,
|
||||
// 规则返回解密后的字节(ArrayBuffer / typed array)。执行失败返回 error,
|
||||
// 调用方决定是回 502 还是原样透传。
|
||||
func (r *JSRunner) RunImageDecode(js string, data []byte, src string) ([]byte, error) {
|
||||
vm := r.vm
|
||||
r.installJava(nil)
|
||||
vm.Set("book", nil)
|
||||
vm.Set("chapter", nil)
|
||||
vm.Set("title", nil)
|
||||
vm.Set("baseUrl", r.cfg.BaseURL)
|
||||
vm.Set("result", vm.ToValue(vm.NewArrayBuffer(data)))
|
||||
vm.Set("src", src)
|
||||
vm.Set("key", nil)
|
||||
vm.Set("page", nil)
|
||||
vm.Set("nextChapterUrl", nil)
|
||||
|
||||
prog, err := compileRuleJS(stripRuleJSWrapper(js))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("图片解密 JS 编译失败: %w", err)
|
||||
}
|
||||
g := newInterruptGuard(vm, r.cfg.Timeout, "图片解密超时")
|
||||
r.setGuard(g)
|
||||
defer func() {
|
||||
g.Stop()
|
||||
r.setGuard(nil)
|
||||
}()
|
||||
v, err := vm.RunProgram(prog)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("图片解密失败: %v", err)
|
||||
}
|
||||
out, ok := exportBytes(vm, v)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("图片解密规则没有返回字节")
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// FetchBytes 用书源的网络栈拉原始字节(queryTTF 的 URL 形态、图片解密前的取图)。
|
||||
func (r *JSRunner) FetchBytes(absURL string) ([]byte, error) {
|
||||
if r.cfg.FetchBytes == nil {
|
||||
return nil, ErrJsUnsupported
|
||||
}
|
||||
resume := r.pauseTimeout()
|
||||
defer resume()
|
||||
return r.cfg.FetchBytes(absURL)
|
||||
}
|
||||
|
||||
// exportValue 把 JS 返回值转为 Go 值(字符串/数值/映射/切片)。
|
||||
func exportValue(v goja.Value) any {
|
||||
if v == nil || goja.IsUndefined(v) || goja.IsNull(v) {
|
||||
@@ -454,6 +636,41 @@ func exportValue(v goja.Value) any {
|
||||
}
|
||||
}
|
||||
|
||||
// stripRuleJSWrapper 去掉 JS 规则的 @js: / <js>…</js> 包裹。
|
||||
// 与 reader 包的 stripJSWrapper 同语义;规则包不能反向依赖 reader 包,故此处保留一份。
|
||||
func stripRuleJSWrapper(s string) string {
|
||||
trimmed := strings.TrimSpace(s)
|
||||
lower := strings.ToLower(trimmed)
|
||||
switch {
|
||||
case strings.HasPrefix(lower, "@js:"):
|
||||
return strings.TrimSpace(trimmed[len("@js:"):])
|
||||
case strings.HasPrefix(lower, "<js>"):
|
||||
body := trimmed[len("<js>"):]
|
||||
body = strings.TrimSuffix(strings.TrimSpace(body), "</js>")
|
||||
body = strings.TrimSuffix(strings.TrimSpace(body), "<")
|
||||
return body
|
||||
default:
|
||||
return trimmed
|
||||
}
|
||||
}
|
||||
|
||||
// exportBytes 把 JS 返回值按字节取出:ArrayBuffer / typed array / 字符串。
|
||||
// 图片与字体解密规则返回的都是字节,ArrayBuffer 的 Export() 只给出属性 map,
|
||||
// 必须走 ExportTo 才能拿到真正的字节。
|
||||
func exportBytes(vm *goja.Runtime, v goja.Value) ([]byte, bool) {
|
||||
if v == nil || goja.IsUndefined(v) || goja.IsNull(v) {
|
||||
return nil, false
|
||||
}
|
||||
var buf []byte
|
||||
if err := vm.ExportTo(v, &buf); err == nil {
|
||||
return buf, true
|
||||
}
|
||||
if s, ok := v.Export().(string); ok {
|
||||
return []byte(s), true
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// toJSValue 把引擎内部结果转为可注入 JS 的值。
|
||||
// Element 列表等 DOM 结果以序列化字符串传入(对应 Rhino 的 Java 对象字符串化)。
|
||||
func toJSValue(vm *goja.Runtime, v any) any {
|
||||
@@ -495,6 +712,9 @@ func newResponseObject(vm *goja.Runtime, body string, code int, finalURL string,
|
||||
}
|
||||
mustSet("body", func(call goja.FunctionCall) goja.Value { return vm.ToValue(body) })
|
||||
mustSet("bodyStr", body)
|
||||
// bodyAsBytes:部分书源用 `java.bytesToStr(resp.bodyAsBytes(), enc)` 处理
|
||||
// 非 UTF-8(GBK)正文(拷贝漫画轻小说源即如此)。
|
||||
mustSet("bodyAsBytes", func(call goja.FunctionCall) goja.Value { return vm.ToValue(vm.NewArrayBuffer([]byte(body))) })
|
||||
mustSet("code", func(call goja.FunctionCall) goja.Value { return vm.ToValue(code) })
|
||||
mustSet("url", func(call goja.FunctionCall) goja.Value { return vm.ToValue(finalURL) })
|
||||
mustSet("header", func(call goja.FunctionCall) goja.Value {
|
||||
|
||||
@@ -332,14 +332,51 @@ func TestJSUnsupportedStillWorks(t *testing.T) {
|
||||
|
||||
// ─── 沙箱边界 ───────────────────────────────────────────────────────────────
|
||||
|
||||
// TestJSSandboxUnsupported 服务端不接受书源读取宿主任意文件。
|
||||
//
|
||||
// 文件类接口(cacheFile/downloadFile/readTxtFile/...)已实现,但路径被限制在
|
||||
// 书源自己的缓存目录内:越权路径安全地返回空,而不是把 /etc/passwd 交出去,
|
||||
// 也不应因为越权路径抛异常(否则书源会误判为「线路故障」反复重试)。
|
||||
// 真正无实现的能力(解压包、无头浏览器等)继续明确抛「不支持」。
|
||||
func TestJSSandboxUnsupported(t *testing.T) {
|
||||
r := newTestRunner()
|
||||
a := NewAnalyzeRule()
|
||||
a.SetJSRunner(r.ForAnalyzer(a))
|
||||
a.SetContent("", "")
|
||||
_, err := a.GetString(`@js:java.readTxtFile('/etc/passwd')`, nil, false)
|
||||
if err == nil || !strings.Contains(err.Error(), "不支持") {
|
||||
t.Fatalf("expected sandbox error, got %v", err)
|
||||
got, err := a.GetString(`@js:java.readTxtFile('/etc/passwd')`, nil, false)
|
||||
if err != nil {
|
||||
t.Fatalf("越权路径应安全返回空而不是抛异常: %v", err)
|
||||
}
|
||||
if strings.Contains(got, "root:") || strings.Contains(got, "/bin/") {
|
||||
t.Fatalf("沙箱被突破,读到了宿主文件: %q", got)
|
||||
}
|
||||
for _, call := range []string{`java.unzipFile('/tmp/a.zip')`, `java.webView('x')`} {
|
||||
if _, err := a.GetString(`@js:`+call, nil, false); err == nil || !strings.Contains(err.Error(), "不支持") {
|
||||
t.Fatalf("%s 应继续抛「不支持」,实际 %v", call, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestRuleJSTopLevelReturn legado 允许规则 JS 顶层 return;无 return 时仍取最后
|
||||
// 一条语句的值(`@js:1+2` → 3)。
|
||||
func TestRuleJSTopLevelReturn(t *testing.T) {
|
||||
r := NewJSRunner(JSConfig{})
|
||||
ar := NewAnalyzeRule()
|
||||
|
||||
v, err := r.Run(ar, "if (1) { return '早退'; }\nreturn '兜底';", nil, "")
|
||||
if err != nil {
|
||||
t.Fatalf("顶层 return 应可用: %v", err)
|
||||
}
|
||||
if got := anyToString(v); got != "早退" {
|
||||
t.Fatalf("top-level return = %q, want 早退", got)
|
||||
}
|
||||
|
||||
v2, err := r.Run(ar, "1 + 2", nil, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := anyToString(v2); got != "3" {
|
||||
t.Fatalf("最后一条语句的值 = %q, want 3", got)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -5,6 +5,9 @@ import (
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -171,6 +174,9 @@ func newBookObject(vm *goja.Runtime, a *AnalyzeRule) *goja.Object {
|
||||
a.bookCustom = map[string]string{}
|
||||
}
|
||||
a.bookCustom[stringArg(call, 0)] = stringArgOr(call, 1, "")
|
||||
if a.bookVarPutter != nil {
|
||||
a.bookVarPutter()
|
||||
}
|
||||
return goja.Null()
|
||||
})
|
||||
return o
|
||||
@@ -381,16 +387,183 @@ func initLoginInfoFromUI(props map[string]any) map[string]string {
|
||||
// 这类临时状态——光遇聚合的 paraForAndroid 每一段带段评的文字都会调
|
||||
// cache.putMemory(url, 0),缺了它整条正文规则会抛 TypeError 直接失败。
|
||||
// 两套存储分开,否则 getFromMemory 会读到 put 写进去的持久值。
|
||||
var jsCache = struct {
|
||||
mu sync.Mutex
|
||||
m map[string]string
|
||||
mem map[string]string
|
||||
}{m: map[string]string{}, mem: map[string]string{}}
|
||||
//
|
||||
// 持久层按书源(bookSourceUrl)命名空间隔离,并落到 CacheDir/reader-js-cache:
|
||||
// 之前是包级全局 map,任何书源的 put/get 全局可见,多个源用同一个 key 会互相串值,
|
||||
// 一个源写满 4096 条还会把别的源的缓存一起清掉;重启后也全部丢失。
|
||||
|
||||
// jsCacheMaxEntries 单套存储的条目上限:超了整体清空,避免书源把内存吃满。
|
||||
// jsCacheMaxEntries 单个书源命名空间的条目上限:超了按写入时间淘汰最旧的一半,
|
||||
// 只影响本命名空间,不再波及其它书源。
|
||||
const jsCacheMaxEntries = 4096
|
||||
|
||||
func newCacheObject(vm *goja.Runtime) *goja.Object {
|
||||
// jsCacheNamespace 一个书源的持久/内存缓存命名空间。
|
||||
type jsCacheNamespace struct {
|
||||
mu sync.Mutex
|
||||
data map[string]string
|
||||
seq map[string]int64
|
||||
next int64
|
||||
path string
|
||||
}
|
||||
|
||||
// jsCacheRegistry 按命名空间持有 cache,命名空间取书源 URL。
|
||||
var jsCacheRegistry = struct {
|
||||
mu sync.Mutex
|
||||
m map[string]*jsCacheNamespace
|
||||
}{m: map[string]*jsCacheNamespace{}}
|
||||
|
||||
// jsCacheFor 取(或创建)命名空间;cacheDir 非空时尝试从磁盘恢复。
|
||||
func jsCacheFor(namespace, cacheDir string) *jsCacheNamespace {
|
||||
if namespace == "" {
|
||||
namespace = "__global__"
|
||||
}
|
||||
jsCacheRegistry.mu.Lock()
|
||||
ns, ok := jsCacheRegistry.m[namespace]
|
||||
if !ok {
|
||||
ns = &jsCacheNamespace{data: map[string]string{}, seq: map[string]int64{}}
|
||||
if cacheDir != "" {
|
||||
ns.path = filepath.Join(cacheDir, "reader-js-cache", md5Hex(namespace, true)+".json")
|
||||
}
|
||||
ns.load()
|
||||
jsCacheRegistry.m[namespace] = ns
|
||||
}
|
||||
jsCacheRegistry.mu.Unlock()
|
||||
return ns
|
||||
}
|
||||
|
||||
// load 从磁盘恢复命名空间(仅供 jsCacheFor 在注册表锁内首次调用)。
|
||||
func (n *jsCacheNamespace) load() {
|
||||
if n.path == "" {
|
||||
return
|
||||
}
|
||||
raw, err := os.ReadFile(n.path) // #nosec G304 -- 路径由服务端生成
|
||||
if err != nil || len(raw) == 0 {
|
||||
return
|
||||
}
|
||||
var stored struct {
|
||||
Data map[string]string `json:"data"`
|
||||
Seq map[string]int64 `json:"seq"`
|
||||
}
|
||||
if json.Unmarshal(raw, &stored) != nil {
|
||||
return
|
||||
}
|
||||
if stored.Data != nil {
|
||||
n.data = stored.Data
|
||||
}
|
||||
if stored.Seq != nil {
|
||||
n.seq = stored.Seq
|
||||
for _, v := range stored.Seq {
|
||||
if v > n.next {
|
||||
n.next = v
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// persist 把命名空间写回磁盘(临时文件 + 原子 rename)。
|
||||
func (n *jsCacheNamespace) persist() {
|
||||
if n.path == "" {
|
||||
return
|
||||
}
|
||||
payload, err := json.Marshal(struct {
|
||||
Data map[string]string `json:"data"`
|
||||
Seq map[string]int64 `json:"seq"`
|
||||
}{n.data, n.seq})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(n.path), 0o750); err != nil {
|
||||
return
|
||||
}
|
||||
tmp, err := os.CreateTemp(filepath.Dir(n.path), ".js-cache-*")
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
name := tmp.Name()
|
||||
if _, err := tmp.Write(payload); err != nil {
|
||||
_ = tmp.Close()
|
||||
_ = os.Remove(name)
|
||||
return
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
_ = os.Remove(name)
|
||||
return
|
||||
}
|
||||
_ = os.Rename(name, n.path)
|
||||
}
|
||||
|
||||
// put 写一个持久键值(返回最终值)。
|
||||
func (n *jsCacheNamespace) put(key, value string) string {
|
||||
n.mu.Lock()
|
||||
defer n.mu.Unlock()
|
||||
if _, exists := n.data[key]; !exists && len(n.data) >= jsCacheMaxEntries {
|
||||
n.evictOldestLocked()
|
||||
}
|
||||
n.next++
|
||||
n.data[key] = value
|
||||
n.seq[key] = n.next
|
||||
n.persist()
|
||||
return value
|
||||
}
|
||||
|
||||
// evictOldestLocked 淘汰最旧的一半条目(调用方需持锁)。
|
||||
func (n *jsCacheNamespace) evictOldestLocked() {
|
||||
type entry struct {
|
||||
key string
|
||||
seq int64
|
||||
}
|
||||
entries := make([]entry, 0, len(n.data))
|
||||
for k := range n.data {
|
||||
entries = append(entries, entry{k, n.seq[k]})
|
||||
}
|
||||
sort.Slice(entries, func(i, j int) bool { return entries[i].seq < entries[j].seq })
|
||||
drop := len(entries)/2 + 1
|
||||
for i := 0; i < drop && i < len(entries); i++ {
|
||||
delete(n.data, entries[i].key)
|
||||
delete(n.seq, entries[i].key)
|
||||
}
|
||||
}
|
||||
|
||||
// get 读一个持久键。
|
||||
func (n *jsCacheNamespace) get(key string) (string, bool) {
|
||||
n.mu.Lock()
|
||||
defer n.mu.Unlock()
|
||||
v, ok := n.data[key]
|
||||
return v, ok
|
||||
}
|
||||
|
||||
// del 删除一个持久键。
|
||||
func (n *jsCacheNamespace) del(key string) {
|
||||
n.mu.Lock()
|
||||
defer n.mu.Unlock()
|
||||
delete(n.data, key)
|
||||
delete(n.seq, key)
|
||||
n.persist()
|
||||
}
|
||||
|
||||
// memoryStore 进程内内存缓存(putMemory/getFromMemory),同样按命名空间隔离。
|
||||
var jsMemoryRegistry = struct {
|
||||
mu sync.Mutex
|
||||
m map[string]map[string]string
|
||||
}{m: map[string]map[string]string{}}
|
||||
|
||||
func jsMemoryFor(namespace string) map[string]string {
|
||||
if namespace == "" {
|
||||
namespace = "__global__"
|
||||
}
|
||||
jsMemoryRegistry.mu.Lock()
|
||||
defer jsMemoryRegistry.mu.Unlock()
|
||||
store, ok := jsMemoryRegistry.m[namespace]
|
||||
if !ok {
|
||||
store = map[string]string{}
|
||||
jsMemoryRegistry.m[namespace] = store
|
||||
}
|
||||
return store
|
||||
}
|
||||
|
||||
// newCacheObject 构造 JS 的 `cache` 对象。
|
||||
// namespace 取书源 URL;cacheDir 非空时 put/get 持久化到磁盘。
|
||||
func newCacheObject(vm *goja.Runtime, namespace, cacheDir string) *goja.Object {
|
||||
ns := jsCacheFor(namespace, cacheDir)
|
||||
o := vm.NewObject()
|
||||
set := func(k string, v any) {
|
||||
if err := o.Set(k, v); err != nil {
|
||||
@@ -398,51 +571,50 @@ func newCacheObject(vm *goja.Runtime) *goja.Object {
|
||||
}
|
||||
}
|
||||
|
||||
// 两套存储共用同一份读写实现,只有落点不同。
|
||||
putTo := func(store *map[string]string) func(goja.FunctionCall) goja.Value {
|
||||
return func(call goja.FunctionCall) goja.Value {
|
||||
key := stringArg(call, 0)
|
||||
val := ""
|
||||
if len(call.Arguments) > 1 && !goja.IsUndefined(call.Arguments[1]) && !goja.IsNull(call.Arguments[1]) {
|
||||
val = call.Arguments[1].String()
|
||||
}
|
||||
jsCache.mu.Lock()
|
||||
if len(*store) >= jsCacheMaxEntries {
|
||||
*store = map[string]string{}
|
||||
}
|
||||
(*store)[key] = val
|
||||
jsCache.mu.Unlock()
|
||||
return vm.ToValue(val)
|
||||
set("put", func(call goja.FunctionCall) goja.Value {
|
||||
key := stringArg(call, 0)
|
||||
val := ""
|
||||
if len(call.Arguments) > 1 && !goja.IsUndefined(call.Arguments[1]) && !goja.IsNull(call.Arguments[1]) {
|
||||
val = call.Arguments[1].String()
|
||||
}
|
||||
}
|
||||
getFrom := func(store *map[string]string) func(goja.FunctionCall) goja.Value {
|
||||
return func(call goja.FunctionCall) goja.Value {
|
||||
key := stringArg(call, 0)
|
||||
jsCache.mu.Lock()
|
||||
v, ok := (*store)[key]
|
||||
jsCache.mu.Unlock()
|
||||
if !ok {
|
||||
return goja.Null()
|
||||
}
|
||||
return vm.ToValue(ns.put(key, val))
|
||||
})
|
||||
set("get", func(call goja.FunctionCall) goja.Value {
|
||||
if v, ok := ns.get(stringArg(call, 0)); ok {
|
||||
return vm.ToValue(v)
|
||||
}
|
||||
}
|
||||
deleteFrom := func(store *map[string]string) func(goja.FunctionCall) goja.Value {
|
||||
return func(call goja.FunctionCall) goja.Value {
|
||||
key := stringArg(call, 0)
|
||||
jsCache.mu.Lock()
|
||||
delete(*store, key)
|
||||
jsCache.mu.Unlock()
|
||||
return goja.Null()
|
||||
})
|
||||
set("delete", func(call goja.FunctionCall) goja.Value {
|
||||
ns.del(stringArg(call, 0))
|
||||
return goja.Null()
|
||||
})
|
||||
// 内存缓存(legado Cache.getFromMemory / putMemory):进程内、不落盘。
|
||||
mem := jsMemoryFor(namespace)
|
||||
set("putMemory", func(call goja.FunctionCall) goja.Value {
|
||||
key := stringArg(call, 0)
|
||||
val := ""
|
||||
if len(call.Arguments) > 1 && !goja.IsUndefined(call.Arguments[1]) && !goja.IsNull(call.Arguments[1]) {
|
||||
val = call.Arguments[1].String()
|
||||
}
|
||||
jsMemoryRegistry.mu.Lock()
|
||||
if _, exists := mem[key]; !exists && len(mem) >= jsCacheMaxEntries {
|
||||
mem = map[string]string{}
|
||||
jsMemoryRegistry.m[namespace] = mem
|
||||
}
|
||||
mem[key] = val
|
||||
jsMemoryRegistry.mu.Unlock()
|
||||
return vm.ToValue(val)
|
||||
})
|
||||
set("getFromMemory", func(call goja.FunctionCall) goja.Value {
|
||||
jsMemoryRegistry.mu.Lock()
|
||||
v, ok := mem[stringArg(call, 0)]
|
||||
jsMemoryRegistry.mu.Unlock()
|
||||
if !ok {
|
||||
return goja.Null()
|
||||
}
|
||||
}
|
||||
|
||||
set("put", putTo(&jsCache.m))
|
||||
set("get", getFrom(&jsCache.m))
|
||||
set("delete", deleteFrom(&jsCache.m))
|
||||
// 内存缓存(legado Cache.getFromMemory / putMemory)
|
||||
set("putMemory", putTo(&jsCache.mem))
|
||||
set("getFromMemory", getFrom(&jsCache.mem))
|
||||
return vm.ToValue(v)
|
||||
})
|
||||
return o
|
||||
}
|
||||
|
||||
|
||||
@@ -51,3 +51,44 @@ func TestCacheMemoryRoundTrip(t *testing.T) {
|
||||
t.Fatalf("cache.get 读到了内存缓存的值: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// cache 必须按书源隔离:旧实现是包级全局 map,任一源的 put/get 对所有源可见,
|
||||
// 多个源用同一个 key 会互相串值,一个源写满还会清掉别的源的缓存。
|
||||
func TestCacheIsolatedPerSource(t *testing.T) {
|
||||
srcA := map[string]any{"bookSourceUrl": "https://a.example.com"}
|
||||
srcB := map[string]any{"bookSourceUrl": "https://b.example.com"}
|
||||
ar := NewAnalyzeRule()
|
||||
|
||||
runnerA := NewJSRunner(JSConfig{SourceProps: srcA})
|
||||
if _, err := runnerA.Run(ar, `cache.put('shared-key', 'from-a')`, nil, ""); err != nil {
|
||||
t.Fatalf("源 A 写入失败: %v", err)
|
||||
}
|
||||
runnerB := NewJSRunner(JSConfig{SourceProps: srcB})
|
||||
v, err := runnerB.Run(ar, `String(cache.get('shared-key'))`, nil, "")
|
||||
if err != nil {
|
||||
t.Fatalf("源 B 读取失败: %v", err)
|
||||
}
|
||||
if got := anyToString(v); got != "null" {
|
||||
t.Fatalf("源 B 读到了源 A 的缓存: %q", got)
|
||||
}
|
||||
// 源 A 自己重开运行时仍应读到(落到 CacheDir 之外的进程内持久层)。
|
||||
runnerA2 := NewJSRunner(JSConfig{SourceProps: srcA})
|
||||
v, err = runnerA2.Run(ar, `String(cache.get('shared-key'))`, nil, "")
|
||||
if err != nil {
|
||||
t.Fatalf("源 A 二次读取失败: %v", err)
|
||||
}
|
||||
if got := anyToString(v); got != "from-a" {
|
||||
t.Fatalf("源 A 应读到自己的缓存: %q", got)
|
||||
}
|
||||
// 内存缓存同样按源隔离。
|
||||
if _, err := runnerA.Run(ar, `cache.putMemory('mem-key', 'a')`, nil, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v, err = runnerB.Run(ar, `String(cache.getFromMemory('mem-key'))`, nil, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := anyToString(v); got != "null" {
|
||||
t.Fatalf("内存缓存跨源串值: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,13 +32,53 @@ func jsonRead(root any, path string) any {
|
||||
if root == nil || path == "" {
|
||||
return nil
|
||||
}
|
||||
v, err := jsonpathGet(path, root)
|
||||
v, err := jsonpathGet(normalizeJSONPath(path), root)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// normalizeJSONPath 把 legado(Jayway)允许、而 PaesslerAG/jsonpath 拒绝的写法
|
||||
// 归一化成合法路径,共两处差异:
|
||||
//
|
||||
// 1. 缺根 `$`:Jayway 要求路径以 `$`/`@` 开头,legado 书源却常直接写
|
||||
// `results.list`、`results.list.[*]`。PaesslerAG 对「不含 `[]`/`*` 的裸路径」
|
||||
// 恰好能容忍(内部按 `$.` 解析),但一旦出现 `[*]` 就报
|
||||
// `unexpected "*" while scanning extensions`。这里统一补根。
|
||||
// 2. 点后紧跟方括号:`results.list.[*]`、`$.a.[0]`、`$.a.['k']`。Jayway 里
|
||||
// `.[` 等价于 `[`,PaesslerAG 会报 `unexpected "[" while scanning field`。
|
||||
//
|
||||
// 两处都会让整条规则静默返回空——表现为「书源明明有效却搜不出任何结果」
|
||||
// (拷贝轻小说源的搜索列表规则就是 `results.list.[*]`)。
|
||||
func normalizeJSONPath(path string) string {
|
||||
p := strings.TrimSpace(path)
|
||||
if p == "" {
|
||||
return path
|
||||
}
|
||||
switch {
|
||||
case strings.HasPrefix(p, "$"), strings.HasPrefix(p, "@"):
|
||||
// 已有根,保持
|
||||
case strings.HasPrefix(p, "."):
|
||||
p = "$" + p // `..a` 这类递归写法,补 `$` 而不是 `$.`
|
||||
default:
|
||||
p = "$." + p
|
||||
}
|
||||
if !strings.Contains(p, ".[") {
|
||||
return p
|
||||
}
|
||||
var b strings.Builder
|
||||
b.Grow(len(p))
|
||||
for i := 0; i < len(p); i++ {
|
||||
// 丢掉 `[` 前多余的点;递归下降 `..[` 保持原样。
|
||||
if p[i] == '.' && i+1 < len(p) && p[i+1] == '[' && i > 0 && p[i-1] != '.' {
|
||||
continue
|
||||
}
|
||||
b.WriteByte(p[i])
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// jsonValueToString 对应 Kotlin 的 ob.toString() / joinToString("\n")。
|
||||
// 列表按 "\n" 拼接(legado 的 getString 就是这么做的);对象走 Java 的 Map.toString
|
||||
// 形态(见 javaValueString),书源的正则大量依赖这个形态。
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
package rule
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
// 本文件:legado 兼容性回归——`Packages.java.util.*` 容器与 `.[*]` 形式的 JSONPath。
|
||||
//
|
||||
// 背景(拷贝系列轻小说源在 MeBox 搜不到/读不了的根因):
|
||||
// 1. 搜索列表规则写成 `results.list.[*]`(legado/Jayway 接受),PaesslerAG 直接
|
||||
// 解析报错返回空;
|
||||
// 2. 目录规则用 `new Packages.java.util.LinkedHashMap()` / `ArrayList()` 拼装,
|
||||
// goja 里没有 Java,整条规则 ReferenceError。
|
||||
|
||||
// TestNormalizeJSONPathDotBracket 点后跟方括号应归一化成合法路径。
|
||||
func TestNormalizeJSONPathDotBracket(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"results.list.[*]": "$.results.list[*]",
|
||||
"$.results.list.[*]": "$.results.list[*]",
|
||||
"$.a.[0].b": "$.a[0].b",
|
||||
"$.a.['k']": "$.a['k']",
|
||||
"$.results.list": "$.results.list",
|
||||
"$.results..list[*]": "$.results..list[*]",
|
||||
"results.list": "$.results.list",
|
||||
"$[0].name": "$[0].name",
|
||||
"..list[*]": "$..list[*]",
|
||||
}
|
||||
for in, want := range cases {
|
||||
if got := normalizeJSONPath(in); got != want {
|
||||
t.Errorf("normalizeJSONPath(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestGetElementsDotBracketStarJsonPath 真实书源的 `results.list.[*]` 必须能取到列表。
|
||||
func TestGetElementsDotBracketStarJsonPath(t *testing.T) {
|
||||
const body = `{"results":{"list":[{"name":"甲"},{"name":"乙"}]}}`
|
||||
for _, rule := range []string{"results.list.[*]", "$.results.list", "results.list"} {
|
||||
ar := NewAnalyzeRule()
|
||||
ar.SetContent(body, "")
|
||||
els, err := ar.GetElements(rule)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", rule, err)
|
||||
}
|
||||
if len(els) != 2 {
|
||||
t.Fatalf("%s: elements=%d, want 2", rule, len(els))
|
||||
}
|
||||
name, err := ar.GetString("$.name", els[0], false)
|
||||
if err != nil || name != "甲" {
|
||||
t.Fatalf("%s: first name=%q err=%v", rule, name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestPackagesUtilCollections Packages.java.util 的 List/Map 应能拼出目录列表,
|
||||
// 且导出给引擎的是干净的数组/映射(方法不能混进元素里)。
|
||||
func TestPackagesUtilCollections(t *testing.T) {
|
||||
r := NewJSRunner(JSConfig{})
|
||||
if err := r.JSLibErr(); err != nil {
|
||||
t.Fatalf("Packages 环境初始化失败: %v", err)
|
||||
}
|
||||
ar := NewAnalyzeRule()
|
||||
ar.SetContent(`{"results":{"list":[{"name":"第一卷","id":7}]}}`, "https://api.example.com/volumes")
|
||||
ar.SetJSRunner(r.ForAnalyzer(ar))
|
||||
|
||||
js := `<js>
|
||||
function mk(n, u) { var m = new Packages.java.util.LinkedHashMap(); m.put('name', n); m.put('url', u); return m; }
|
||||
var base = 'https://api.example.com/volume/';
|
||||
var out = new Packages.java.util.ArrayList();
|
||||
var vols = JSON.parse(result).results.list;
|
||||
for (var i = 0; i < vols.length; i++) { out.add(mk(vols[i].name, base + vols[i].id)); }
|
||||
out;</js>`
|
||||
els, err := ar.GetElements(js)
|
||||
if err != nil {
|
||||
t.Fatalf("GetElements: %v", err)
|
||||
}
|
||||
if len(els) != 1 {
|
||||
t.Fatalf("elements=%d, want 1", len(els))
|
||||
}
|
||||
if m, ok := els[0].(map[string]any); ok {
|
||||
if _, hasMethod := m["put"]; hasMethod {
|
||||
t.Fatalf("容器方法泄漏进了元素: %#v", m)
|
||||
}
|
||||
}
|
||||
name, err := ar.GetString("$.name", els[0], false)
|
||||
if err != nil || name != "第一卷" {
|
||||
t.Fatalf("name=%q err=%v", name, err)
|
||||
}
|
||||
url, _ := ar.GetString("$.url", els[0], false)
|
||||
if url != "https://api.example.com/volume/7" {
|
||||
t.Fatalf("url=%q", url)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPackagesHashMapUsableAsHeaders java.get(url, headers) 的 headers 用
|
||||
// Packages.java.util.HashMap 构造时,导出结果应是干净的键值映射。
|
||||
func TestPackagesHashMapUsableAsHeaders(t *testing.T) {
|
||||
r := NewJSRunner(JSConfig{})
|
||||
v, err := r.Run(NewAnalyzeRule(), `(function () {
|
||||
var h = new Packages.java.util.HashMap();
|
||||
h.put('X-Test', 'v1');
|
||||
return JSON.stringify(h);
|
||||
})()`, nil, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if s := anyToString(v); s != `{"X-Test":"v1"}` {
|
||||
t.Fatalf("HashMap 导出含额外键: %s", s)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,199 @@
|
||||
package rule
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/dop251/goja"
|
||||
)
|
||||
|
||||
// 本文件对应 legado JsExtensions 的 queryTTF / replaceFont:
|
||||
// 解析被字体混淆的正文。书源规则通常写成
|
||||
//
|
||||
// java.replaceFont(result, java.queryTTF(errorFontUrl), java.queryTTF(correctFontUrl))
|
||||
//
|
||||
// 错误字体把码点映射到错字形、正确字体把字形映射回真实码点,按字形轮廓配对即可还原。
|
||||
|
||||
// queryTTFCacheMax 已解析字体缓存条数(对齐 legado 的 LruCache 小容量策略)。
|
||||
const queryTTFCacheMax = 32
|
||||
|
||||
var queryTTFCache = struct {
|
||||
mu sync.Mutex
|
||||
m map[string]*queryTTFFont
|
||||
// order 记录插入顺序,超容量时淘汰最早的一条。
|
||||
order []string
|
||||
}{m: map[string]*queryTTFFont{}}
|
||||
|
||||
// queryTTFFromBytes 解析字体并缓存(key = SHA-256)。
|
||||
func queryTTFFromBytes(data []byte, useCache bool) (*queryTTFFont, error) {
|
||||
sum := sha256.Sum256(data)
|
||||
key := hex.EncodeToString(sum[:])
|
||||
if useCache {
|
||||
queryTTFCache.mu.Lock()
|
||||
if f, ok := queryTTFCache.m[key]; ok {
|
||||
queryTTFCache.mu.Unlock()
|
||||
return f, nil
|
||||
}
|
||||
queryTTFCache.mu.Unlock()
|
||||
}
|
||||
f, err := parseQueryTTFFont(data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !useCache {
|
||||
return f, nil
|
||||
}
|
||||
queryTTFCache.mu.Lock()
|
||||
if len(queryTTFCache.order) >= queryTTFCacheMax {
|
||||
oldest := queryTTFCache.order[0]
|
||||
queryTTFCache.order = queryTTFCache.order[1:]
|
||||
delete(queryTTFCache.m, oldest)
|
||||
}
|
||||
queryTTFCache.m[key] = f
|
||||
queryTTFCache.order = append(queryTTFCache.order, key)
|
||||
queryTTFCache.mu.Unlock()
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// installFontBridge 注册 java.queryTTF / queryBase64TTF / replaceFont。
|
||||
// set 是 java 对象的属性写入闭包(其中已包含 bridgeErr 的异常抛出语义)。
|
||||
func (r *JSRunner) installFontBridge(vm *goja.Runtime, set func(string, any)) {
|
||||
// fontErr 把错误翻译成 JS 异常:书源常把 queryTTF 放在 try/catch 里降级。
|
||||
fontErr := func(name string, msg string) goja.Value {
|
||||
return vm.ToValue("java." + name + ": " + msg)
|
||||
}
|
||||
set("queryTTF", func(call goja.FunctionCall) goja.Value {
|
||||
useCache := true
|
||||
if len(call.Arguments) > 1 {
|
||||
useCache = call.Arguments[1].ToBoolean()
|
||||
}
|
||||
data, err := r.fontDataFromArg(call)
|
||||
if err != nil {
|
||||
panic(fontErr("queryTTF", err.Error()))
|
||||
}
|
||||
font, err := queryTTFFromBytes(data, useCache)
|
||||
if err != nil {
|
||||
panic(fontErr("queryTTF", err.Error()))
|
||||
}
|
||||
return vm.ToValue(font)
|
||||
})
|
||||
// queryBase64TTF 是 legado 的旧别名,语义完全相同。
|
||||
set("queryBase64TTF", func(call goja.FunctionCall) goja.Value {
|
||||
data, err := r.fontDataFromArg(call)
|
||||
if err != nil {
|
||||
panic(fontErr("queryBase64TTF", err.Error()))
|
||||
}
|
||||
font, err := queryTTFFromBytes(data, true)
|
||||
if err != nil {
|
||||
panic(fontErr("queryBase64TTF", err.Error()))
|
||||
}
|
||||
return vm.ToValue(font)
|
||||
})
|
||||
set("replaceFont", func(call goja.FunctionCall) goja.Value {
|
||||
text := stringArg(call, 0)
|
||||
errorFont := queryTTFFontArg(call, 1)
|
||||
correctFont := queryTTFFontArg(call, 2)
|
||||
filter := false
|
||||
if len(call.Arguments) > 3 {
|
||||
filter = call.Arguments[3].ToBoolean()
|
||||
}
|
||||
if errorFont == nil || correctFont == nil {
|
||||
return vm.ToValue(text)
|
||||
}
|
||||
return vm.ToValue(replaceFontText(text, errorFont, correctFont, filter))
|
||||
})
|
||||
}
|
||||
|
||||
// fontDataFromArg 从 JS 参数取字体字节:URL / base64 / ArrayBuffer / typed array。
|
||||
func (r *JSRunner) fontDataFromArg(call goja.FunctionCall) ([]byte, error) {
|
||||
if len(call.Arguments) == 0 || goja.IsUndefined(call.Arguments[0]) || goja.IsNull(call.Arguments[0]) {
|
||||
return nil, errQueryTTF("缺少字体参数")
|
||||
}
|
||||
// 字节形态(ArrayBuffer / Uint8Array)优先。
|
||||
if data, ok := exportBytes(r.vm, call.Arguments[0]); ok && len(data) > 0 {
|
||||
return data, nil
|
||||
}
|
||||
raw := strings.TrimSpace(call.Arguments[0].String())
|
||||
if raw == "" {
|
||||
return nil, errQueryTTF("缺少字体参数")
|
||||
}
|
||||
if strings.HasPrefix(raw, "http://") || strings.HasPrefix(raw, "https://") {
|
||||
data, err := r.FetchBytes(raw)
|
||||
if err != nil {
|
||||
return nil, errQueryTTF("下载字体失败: " + err.Error())
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
// base64(支持 data:font/...;base64, 前缀与 URL-safe 变体)。
|
||||
if i := strings.Index(raw, "base64,"); i >= 0 {
|
||||
raw = raw[i+len("base64,"):]
|
||||
}
|
||||
data, err := base64DecodeBytes(raw)
|
||||
if err != nil {
|
||||
return nil, errQueryTTF("字体数据不是 base64")
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// errQueryTTF 构造 queryTTF 相关错误(拼上函数名前缀由调用方负责)。
|
||||
func errQueryTTF(msg string) error { return fmt.Errorf("%s", msg) }
|
||||
|
||||
// queryTTFFontArg 把 JS 参数还原成 *queryTTFFont。
|
||||
// 书源常把 queryTTF 的结果存进变量再传给 replaceFont,goja 会原样保留 Go 指针。
|
||||
func queryTTFFontArg(call goja.FunctionCall, idx int) *queryTTFFont {
|
||||
if len(call.Arguments) <= idx {
|
||||
return nil
|
||||
}
|
||||
if f, ok := call.Arguments[idx].Export().(*queryTTFFont); ok {
|
||||
return f
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// replaceFontText 按字形轮廓把错误字体渲染的文本还原成正确字体对应的真实文字。
|
||||
// 顺序对齐 legado JsExtensions.replaceFont:逐码点取错字形,再到正确字体查回码点。
|
||||
func replaceFontText(text string, errorFont, correctFont *queryTTFFont, filter bool) string {
|
||||
var sb strings.Builder
|
||||
sb.Grow(len(text))
|
||||
for _, cp := range text {
|
||||
if isBlankUnicode(cp) {
|
||||
sb.WriteRune(cp)
|
||||
continue
|
||||
}
|
||||
glyph := errorFont.unicodeToGlyph[cp]
|
||||
if _, ok := errorFont.unicodeToGlyphID[cp]; !ok {
|
||||
// 错误字体里没有这个码点(对应 legado 的 glyfId == 0 → 视为无字形)。
|
||||
glyph = ""
|
||||
}
|
||||
if glyph == "" {
|
||||
if filter {
|
||||
continue
|
||||
}
|
||||
sb.WriteRune(cp)
|
||||
continue
|
||||
}
|
||||
if real, ok := correctFont.glyphToUnicode[glyph]; ok && real != 0 {
|
||||
sb.WriteRune(real)
|
||||
continue
|
||||
}
|
||||
if filter {
|
||||
continue
|
||||
}
|
||||
sb.WriteRune(cp)
|
||||
}
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
// isBlankUnicode 判断码点是否是不可见的空白(对齐 legado 的 isBlankUnicode 列表)。
|
||||
func isBlankUnicode(cp rune) bool {
|
||||
switch cp {
|
||||
case ' ', '\t', '\n', '\r', '\v', '\f', 0x00A0, 0x2000, 0x2001, 0x2002, 0x2003,
|
||||
0x2004, 0x2005, 0x2006, 0x2007, 0x2008, 0x2009, 0x200A, 0x2028, 0x2029,
|
||||
0x202F, 0x205F, 0x3000, 0xFEFF:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -10,6 +10,23 @@ import (
|
||||
// 本文件对应 AnalyzeByRegex.kt。Java 正则语义用 regexp2 对齐
|
||||
// (支持前向后向断言与反向引用),匹配循环对齐 Matcher.find()。
|
||||
|
||||
// regexMatchTimeout 书源正则的匹配预算。
|
||||
//
|
||||
// regexp2 默认永不超时(DefaultMatchTimeout 是 MaxInt64),而书源正则来自
|
||||
// 不可信内容:灾难性回溯会永久占住一个 goroutine 和一颗 CPU 核。这里统一
|
||||
// 设一个短预算,语义与 ApplyUserReplace 的缺省值保持一致。
|
||||
const regexMatchTimeout = 3 * time.Second
|
||||
|
||||
// compileRegex 编译一条 Java 语义正则并设置匹配超时。
|
||||
func compileRegex(pattern string) (*regexp2.Regexp, error) {
|
||||
re, err := regexp2.Compile(pattern, regexp2.None)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
re.MatchTimeout = regexMatchTimeout
|
||||
return re, nil
|
||||
}
|
||||
|
||||
// splitNotBlankAndTrim 对应 String.splitNotBlank("&&"):切分并去空白项。
|
||||
func splitNotBlankAndTrim(s, sep string) []string {
|
||||
var out []string
|
||||
@@ -27,7 +44,7 @@ func regexGetElement(res string, regs []string, index int) []string {
|
||||
if index >= len(regs) {
|
||||
return nil
|
||||
}
|
||||
re, err := regexp2.Compile(regs[index], regexp2.None)
|
||||
re, err := compileRegex(regs[index])
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
@@ -60,7 +77,7 @@ func regexGetElements(res string, regs []string, index int) [][]string {
|
||||
if index >= len(regs) {
|
||||
return nil
|
||||
}
|
||||
re, err := regexp2.Compile(regs[index], regexp2.None)
|
||||
re, err := compileRegex(regs[index])
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
@@ -95,7 +112,7 @@ func regexGetElements(res string, regs []string, index int) [][]string {
|
||||
// regexReplaceAll 对应 Kotlin Regex.replace(result, replacement)
|
||||
// (Java $N 分组替换语义,regexp2 的 Replace 原生支持)。
|
||||
func regexReplaceAll(pattern, result, replacement string) string {
|
||||
re, err := regexp2.Compile(pattern, regexp2.None)
|
||||
re, err := compileRegex(pattern)
|
||||
if err != nil {
|
||||
return strings.ReplaceAll(result, pattern, replacement)
|
||||
}
|
||||
@@ -116,14 +133,12 @@ func ApplyUserReplace(content, pattern, replacement string, isRegex bool, timeou
|
||||
if !isRegex {
|
||||
return strings.ReplaceAll(content, pattern, replacement)
|
||||
}
|
||||
re, err := regexp2.Compile(pattern, regexp2.None)
|
||||
re, err := compileRegex(pattern)
|
||||
if err != nil {
|
||||
return strings.ReplaceAll(content, pattern, replacement)
|
||||
}
|
||||
if timeoutMS > 0 {
|
||||
re.MatchTimeout = time.Duration(timeoutMS) * time.Millisecond
|
||||
} else {
|
||||
re.MatchTimeout = 3 * time.Second
|
||||
}
|
||||
out, err := re.Replace(content, replacement, 0, -1)
|
||||
if err != nil {
|
||||
@@ -160,7 +175,7 @@ func ApplyReplaceRegexString(content, replaceRegex string) string {
|
||||
// regexReplaceFirstOnFirstMatch 对应 replaceRegex 的 replaceFirst 分支:
|
||||
// 找到第一个匹配(无匹配返回 ""),在匹配文本上做首次替换。
|
||||
func regexReplaceFirstOnFirstMatch(pattern, result, replacement string) string {
|
||||
re, err := regexp2.Compile(pattern, regexp2.None)
|
||||
re, err := compileRegex(pattern)
|
||||
if err != nil {
|
||||
return replacement
|
||||
}
|
||||
|
||||
@@ -0,0 +1,375 @@
|
||||
package rule
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// 本文件移植 legado 的 QueryTTF:解析字体(sfnt)的 cmap / glyf / loca / maxp 表,
|
||||
// 建立「Unicode 码点 → 字形」「字形 → Unicode 码点」两张表,供
|
||||
// java.queryTTF / java.replaceFont 还原被字体混淆的正文。
|
||||
//
|
||||
// 阅读站点的常见套路是:正文用一套打乱过的字体渲染,页面上给出「错误字体」(把
|
||||
// 每个码点映射到错字形)与「正确字体」(字形到真实码点的映射)。replaceFont 按
|
||||
// 字形轮廓把错误字体里的字符逐个换成正确字体里同字形的码点,从而还原原文。
|
||||
//
|
||||
// 解析器全程做边界检查:字体字节来自书源(不可信),坏字体只应报错,不能 panic。
|
||||
|
||||
// queryTTFFont 一个已解析的字体。
|
||||
type queryTTFFont struct {
|
||||
// unicodeToGlyphID 码点 → 字形在 glyf 表里的下标。
|
||||
unicodeToGlyphID map[rune]uint16
|
||||
// unicodeToGlyph 码点 → 字形轮廓(用于跨字体比较字形)。
|
||||
unicodeToGlyph map[rune]string
|
||||
// glyphToUnicode 字形轮廓 → 码点(正确字体用来查回真实字符)。
|
||||
glyphToUnicode map[string]rune
|
||||
}
|
||||
|
||||
// sfnt 表标签。
|
||||
var (
|
||||
ttfTagCmap = [4]byte{'c', 'm', 'a', 'p'}
|
||||
ttfTagGlyf = [4]byte{'g', 'l', 'y', 'f'}
|
||||
ttfTagLoca = [4]byte{'l', 'o', 'c', 'a'}
|
||||
ttfTagMaxp = [4]byte{'m', 'a', 'x', 'p'}
|
||||
ttfTagHead = [4]byte{'h', 'e', 'a', 'd'}
|
||||
)
|
||||
|
||||
// parseQueryTTFFont 解析字体字节。支持 sfnt(TTF/OTF)与 ttc 的第一套字体。
|
||||
func parseQueryTTFFont(data []byte) (font *queryTTFFont, err error) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
font, err = nil, fmt.Errorf("字体解析失败: %v", r)
|
||||
}
|
||||
}()
|
||||
if len(data) < 12 {
|
||||
return nil, fmt.Errorf("字体数据过短")
|
||||
}
|
||||
// ttc:取第一套字体的偏移。
|
||||
if string(data[:4]) == "ttcf" {
|
||||
if len(data) < 16 {
|
||||
return nil, fmt.Errorf("ttc 头部不完整")
|
||||
}
|
||||
off := int(binary.BigEndian.Uint32(data[12:16]))
|
||||
if off <= 0 || off >= len(data) {
|
||||
return nil, fmt.Errorf("ttc 字体偏移非法")
|
||||
}
|
||||
data = data[off:]
|
||||
}
|
||||
numTables := int(binary.BigEndian.Uint16(data[4:6]))
|
||||
if numTables <= 0 || 12+numTables*16 > len(data) {
|
||||
return nil, fmt.Errorf("sfnt 表目录非法")
|
||||
}
|
||||
tables := map[[4]byte][]byte{}
|
||||
for i := 0; i < numTables; i++ {
|
||||
rec := data[12+i*16 : 12+i*16+16]
|
||||
var tag [4]byte
|
||||
copy(tag[:], rec[:4])
|
||||
off := int(binary.BigEndian.Uint32(rec[8:12]))
|
||||
length := int(binary.BigEndian.Uint32(rec[12:16]))
|
||||
if off < 0 || length < 0 || off > len(data) {
|
||||
continue
|
||||
}
|
||||
if off+length > len(data) {
|
||||
length = len(data) - off
|
||||
}
|
||||
tables[tag] = data[off : off+length]
|
||||
}
|
||||
|
||||
cmap := tables[ttfTagCmap]
|
||||
if cmap == nil {
|
||||
return nil, fmt.Errorf("字体缺少 cmap 表")
|
||||
}
|
||||
mapping, err := parseTTFCmap(cmap)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f := &queryTTFFont{
|
||||
unicodeToGlyphID: mapping,
|
||||
unicodeToGlyph: map[rune]string{},
|
||||
glyphToUnicode: map[string]rune{},
|
||||
}
|
||||
|
||||
// 有 glyf + loca + head + maxp 才能算字形轮廓;缺了(如 CFF 字体)只保留码点映射。
|
||||
head := tables[ttfTagHead]
|
||||
maxp := tables[ttfTagMaxp]
|
||||
loca := tables[ttfTagLoca]
|
||||
glyf := tables[ttfTagGlyf]
|
||||
if head == nil || maxp == nil || loca == nil || glyf == nil || len(head) < 54 {
|
||||
return f, nil
|
||||
}
|
||||
indexToLocFormat := int16(binary.BigEndian.Uint16(head[50:52]))
|
||||
numGlyphs := int(binary.BigEndian.Uint16(maxp[4:6]))
|
||||
offsets, err := parseTTFLoca(loca, numGlyphs, indexToLocFormat)
|
||||
if err != nil {
|
||||
return f, nil
|
||||
}
|
||||
glyphCache := map[uint16]string{}
|
||||
for cp, gid := range mapping {
|
||||
outline := ttfGlyphOutline(glyf, offsets, gid, glyphCache, 0)
|
||||
f.unicodeToGlyph[cp] = outline
|
||||
if _, exists := f.glyphToUnicode[outline]; !exists {
|
||||
f.glyphToUnicode[outline] = cp
|
||||
}
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// parseTTFCmap 解析 cmap 表,返回码点 → 字形下标。
|
||||
// 支持 format 0 / 4 / 6(legado QueryTTF 同样只支持这三种)。
|
||||
func parseTTFCmap(cmap []byte) (map[rune]uint16, error) {
|
||||
if len(cmap) < 4 {
|
||||
return nil, fmt.Errorf("cmap 表过短")
|
||||
}
|
||||
numTables := int(binary.BigEndian.Uint16(cmap[2:4]))
|
||||
if 4+numTables*8 > len(cmap) {
|
||||
return nil, fmt.Errorf("cmap 子表目录非法")
|
||||
}
|
||||
out := map[rune]uint16{}
|
||||
for i := 0; i < numTables; i++ {
|
||||
rec := cmap[4+i*8 : 4+i*8+8]
|
||||
off := int(binary.BigEndian.Uint32(rec[4:8]))
|
||||
if off < 0 || off+2 > len(cmap) {
|
||||
continue
|
||||
}
|
||||
sub := cmap[off:]
|
||||
switch binary.BigEndian.Uint16(sub[0:2]) {
|
||||
case 0:
|
||||
parseCmapFormat0(sub, out)
|
||||
case 4:
|
||||
parseCmapFormat4(sub, out)
|
||||
case 6:
|
||||
parseCmapFormat6(sub, out)
|
||||
}
|
||||
// 优先保留第一个子表解析到的映射;后续子表只补缺失项。
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil, fmt.Errorf("cmap 没有可用的 format 0/4/6 子表")
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func parseCmapFormat0(sub []byte, out map[rune]uint16) {
|
||||
if len(sub) < 262 {
|
||||
return
|
||||
}
|
||||
length := int(binary.BigEndian.Uint16(sub[2:4]))
|
||||
if length > len(sub) {
|
||||
length = len(sub)
|
||||
}
|
||||
glyphs := sub[6:min(6+256, length)]
|
||||
for i, gid := range glyphs {
|
||||
if gid != 0 {
|
||||
if _, exists := out[rune(i)]; !exists {
|
||||
out[rune(i)] = uint16(gid)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func parseCmapFormat4(sub []byte, out map[rune]uint16) {
|
||||
if len(sub) < 14 {
|
||||
return
|
||||
}
|
||||
segCountX2 := int(binary.BigEndian.Uint16(sub[6:8]))
|
||||
segCount := segCountX2 / 2
|
||||
if segCount == 0 {
|
||||
return
|
||||
}
|
||||
endBase := 14
|
||||
startBase := endBase + segCountX2 + 2
|
||||
deltaBase := startBase + segCountX2
|
||||
rangeBase := deltaBase + segCountX2
|
||||
if rangeBase+segCountX2 > len(sub) {
|
||||
return
|
||||
}
|
||||
for i := 0; i < segCount; i++ {
|
||||
end := int(binary.BigEndian.Uint16(sub[endBase+i*2:]))
|
||||
start := int(binary.BigEndian.Uint16(sub[startBase+i*2:]))
|
||||
delta := int16(binary.BigEndian.Uint16(sub[deltaBase+i*2:]))
|
||||
rangeOffset := int(binary.BigEndian.Uint16(sub[rangeBase+i*2:]))
|
||||
if start > end {
|
||||
continue
|
||||
}
|
||||
for cp := start; cp <= end && cp <= 0xFFFF; cp++ {
|
||||
if cp == 0xFFFF {
|
||||
continue
|
||||
}
|
||||
var gid uint16
|
||||
if rangeOffset == 0 {
|
||||
gid = uint16(int(cp) + int(delta))
|
||||
} else {
|
||||
idx := rangeBase + i*2 + rangeOffset + (cp-start)*2
|
||||
if idx+2 > len(sub) {
|
||||
continue
|
||||
}
|
||||
g := binary.BigEndian.Uint16(sub[idx : idx+2])
|
||||
if g == 0 {
|
||||
continue
|
||||
}
|
||||
gid = uint16(int(g) + int(delta))
|
||||
}
|
||||
if gid != 0 {
|
||||
if _, exists := out[rune(cp)]; !exists {
|
||||
out[rune(cp)] = gid
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func parseCmapFormat6(sub []byte, out map[rune]uint16) {
|
||||
if len(sub) < 10 {
|
||||
return
|
||||
}
|
||||
first := int(binary.BigEndian.Uint16(sub[6:8]))
|
||||
count := int(binary.BigEndian.Uint16(sub[8:10]))
|
||||
for i := 0; i < count; i++ {
|
||||
idx := 10 + i*2
|
||||
if idx+2 > len(sub) {
|
||||
return
|
||||
}
|
||||
gid := binary.BigEndian.Uint16(sub[idx : idx+2])
|
||||
if gid == 0 {
|
||||
continue
|
||||
}
|
||||
cp := rune(first + i)
|
||||
if _, exists := out[cp]; !exists {
|
||||
out[cp] = gid
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// parseTTFLoca 解析 loca 表,返回每个字形的字节区间起止。
|
||||
func parseTTFLoca(loca []byte, numGlyphs int, indexToLocFormat int16) ([]int, error) {
|
||||
if indexToLocFormat == 0 {
|
||||
need := (numGlyphs + 1) * 2
|
||||
if len(loca) < need {
|
||||
return nil, fmt.Errorf("loca 表过短")
|
||||
}
|
||||
out := make([]int, numGlyphs+1)
|
||||
for i := 0; i <= numGlyphs; i++ {
|
||||
out[i] = int(binary.BigEndian.Uint16(loca[i*2:])) * 2
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
need := (numGlyphs + 1) * 4
|
||||
if len(loca) < need {
|
||||
return nil, fmt.Errorf("loca 表过短")
|
||||
}
|
||||
out := make([]int, numGlyphs+1)
|
||||
for i := 0; i <= numGlyphs; i++ {
|
||||
out[i] = int(binary.BigEndian.Uint32(loca[i*4:]))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ttfGlyphOutline 把字形转成轮廓字符串(对应 legado QueryTTF.Glyf.toString)。
|
||||
// 复合字形递归展开组件,深度上限防自引用。
|
||||
func ttfGlyphOutline(glyf []byte, offsets []int, gid uint16, cache map[uint16]string, depth int) string {
|
||||
if depth > 8 {
|
||||
return fmt.Sprintf("glyph%d", gid)
|
||||
}
|
||||
if v, ok := cache[gid]; ok {
|
||||
return v
|
||||
}
|
||||
if int(gid)+1 >= len(offsets) {
|
||||
return fmt.Sprintf("glyph%d", gid)
|
||||
}
|
||||
start, end := offsets[gid], offsets[gid+1]
|
||||
if start < 0 || end > len(glyf) || end <= start {
|
||||
// 空字形(如空格):用下标本身当轮廓,保证不同码点不会互相误判。
|
||||
out := fmt.Sprintf("glyph%d", gid)
|
||||
cache[gid] = out
|
||||
return out
|
||||
}
|
||||
numberOfContours := int16(binary.BigEndian.Uint16(glyf[start : start+2]))
|
||||
if numberOfContours >= 0 {
|
||||
out := fmt.Sprintf("simple:%d:%s", numberOfContours, ttfSimpleGlyphPoints(glyf[start:end]))
|
||||
cache[gid] = out
|
||||
return out
|
||||
}
|
||||
// 复合字形:逐组件展开。
|
||||
var sb strings.Builder
|
||||
sb.WriteString("composite")
|
||||
pos := start + 10
|
||||
for pos+4 <= end {
|
||||
flags := binary.BigEndian.Uint16(glyf[pos : pos+2])
|
||||
component := binary.BigEndian.Uint16(glyf[pos+2 : pos+4])
|
||||
pos += 4
|
||||
if flags&0x0001 != 0 { // ARG_1_AND_2_ARE_WORDS
|
||||
pos += 4
|
||||
} else {
|
||||
pos += 2
|
||||
}
|
||||
switch {
|
||||
case flags&0x0008 != 0: // WE_HAVE_A_SCALE
|
||||
pos += 2
|
||||
case flags&0x0040 != 0: // WE_HAVE_AN_X_AND_Y_SCALE
|
||||
pos += 4
|
||||
case flags&0x0080 != 0: // WE_HAVE_A_TWO_BY_TWO
|
||||
pos += 8
|
||||
}
|
||||
sb.WriteString("+")
|
||||
sb.WriteString(ttfGlyphOutline(glyf, offsets, component, cache, depth+1))
|
||||
if flags&0x0020 == 0 { // MORE_COMPONENTS
|
||||
break
|
||||
}
|
||||
}
|
||||
out := sb.String()
|
||||
cache[gid] = out
|
||||
return out
|
||||
}
|
||||
|
||||
// ttfSimpleGlyphPoints 取简单字形的轮廓点(标志与坐标的紧凑编码)。
|
||||
func ttfSimpleGlyphPoints(data []byte) string {
|
||||
if len(data) < 10 {
|
||||
return ""
|
||||
}
|
||||
numberOfContours := int(binary.BigEndian.Uint16(data[0:2]))
|
||||
if numberOfContours <= 0 {
|
||||
return ""
|
||||
}
|
||||
endPtsPos := 10
|
||||
if endPtsPos+numberOfContours*2+2 > len(data) {
|
||||
return ""
|
||||
}
|
||||
numPoints := int(binary.BigEndian.Uint16(data[endPtsPos+(numberOfContours-1)*2:])) + 1
|
||||
if numPoints <= 0 {
|
||||
return ""
|
||||
}
|
||||
pos := endPtsPos + numberOfContours*2 + 2 // + instructionLength
|
||||
if pos > len(data) {
|
||||
return ""
|
||||
}
|
||||
instrLen := int(binary.BigEndian.Uint16(data[pos-2 : pos]))
|
||||
pos += instrLen
|
||||
flags := make([]byte, 0, numPoints)
|
||||
for len(flags) < numPoints && pos < len(data) {
|
||||
flag := data[pos]
|
||||
pos++
|
||||
flags = append(flags, flag)
|
||||
if flag&0x08 != 0 { // REPEAT
|
||||
if pos >= len(data) {
|
||||
break
|
||||
}
|
||||
repeat := int(data[pos])
|
||||
pos++
|
||||
for i := 0; i < repeat && len(flags) < numPoints; i++ {
|
||||
flags = append(flags, flag)
|
||||
}
|
||||
}
|
||||
}
|
||||
// 解析 x / y 坐标(与点数等长的增量序列,这里只用于区分字形)。
|
||||
var sb strings.Builder
|
||||
sb.WriteString(fmt.Sprintf("n=%d;", numPoints))
|
||||
for _, flag := range flags {
|
||||
sb.WriteByte('0' + flag&0x0F)
|
||||
}
|
||||
sb.WriteString(";")
|
||||
// 跳过坐标数据不影响「同名轮廓一致性」的判断:同字形的字体坐标编码一致。
|
||||
if pos < len(data) {
|
||||
sb.WriteString(fmt.Sprintf("d=%d", len(data)-pos))
|
||||
}
|
||||
return sb.String()
|
||||
}
|
||||
@@ -0,0 +1,231 @@
|
||||
package rule
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// 字体混淆还原(ttf.go / queryttf.go)的测试:用合成的 TTF 覆盖 cmap format 0/4、
|
||||
// loca 短格式、简单/复合字形,验证「错误字体 → 正确字体」的按字形还原。
|
||||
|
||||
// buildTestTTF 构造一个最小可解析的 TTF:
|
||||
// cmap(format 4)把给定码点映射到指定字形下标,glyf 里每个字形一个方框。
|
||||
func buildTestTTF(t *testing.T, entries map[rune]uint16) []byte {
|
||||
t.Helper()
|
||||
const numGlyphs = 8
|
||||
// 每个字形一个简单方框:轮廓数 1,1 个点。
|
||||
glyph := func() []byte {
|
||||
g := make([]byte, 0, 20)
|
||||
g = append(g, 0x00, 0x01) // numberOfContours = 1
|
||||
g = append(g, 0, 0, 0, 0, 0, 0, 0, 0) // bbox
|
||||
g = append(g, 0x00, 0x00) // endPtsOfContours[0] = 0
|
||||
g = append(g, 0x00, 0x00) // instructionLength = 0
|
||||
g = append(g, 0x01, 0x01) // flag: on-curve | x-short | y-short
|
||||
g = append(g, 0x00, 0x00) // x=0, y=0
|
||||
return g
|
||||
}
|
||||
glyfData := make([]byte, 0, 128)
|
||||
offsets := make([]int, 0, numGlyphs+1)
|
||||
for i := 0; i < numGlyphs; i++ {
|
||||
offsets = append(offsets, len(glyfData))
|
||||
if i == 0 {
|
||||
continue // .notdef 空字形
|
||||
}
|
||||
g := glyph()
|
||||
// 让不同字形的字节长度不同,轮廓字符串才能区分。
|
||||
for j := 0; j < i-1; j++ {
|
||||
g = append(g, 0x01, 0x00, 0x00) // 额外点
|
||||
}
|
||||
glyfData = append(glyfData, g...)
|
||||
}
|
||||
offsets = append(offsets, len(glyfData))
|
||||
|
||||
// loca(短格式,偏移/2)
|
||||
loca := make([]byte, (numGlyphs+1)*2)
|
||||
for i, off := range offsets {
|
||||
binary.BigEndian.PutUint16(loca[i*2:], uint16(off/2))
|
||||
}
|
||||
|
||||
// cmap:format 4 单段 + 结束段
|
||||
var cmap bytes.Buffer
|
||||
codes := make([]rune, 0, len(entries))
|
||||
for cp := range entries {
|
||||
codes = append(codes, cp)
|
||||
}
|
||||
// 按码点排序,构造连续单点段。
|
||||
for i := 0; i < len(codes); i++ {
|
||||
for j := i + 1; j < len(codes); j++ {
|
||||
if codes[j] < codes[i] {
|
||||
codes[i], codes[j] = codes[j], codes[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
segCount := len(codes) + 1
|
||||
endCodes := make([]uint16, 0, segCount)
|
||||
startCodes := make([]uint16, 0, segCount)
|
||||
idDeltas := make([]uint16, 0, segCount)
|
||||
for _, cp := range codes {
|
||||
endCodes = append(endCodes, uint16(cp))
|
||||
startCodes = append(startCodes, uint16(cp))
|
||||
idDeltas = append(idDeltas, uint16(int(entries[cp])-int(cp)))
|
||||
}
|
||||
endCodes = append(endCodes, 0xFFFF)
|
||||
startCodes = append(startCodes, 0xFFFF)
|
||||
idDeltas = append(idDeltas, 1)
|
||||
rangeOffsets := make([]uint16, segCount) // 全 0:用 idDelta
|
||||
|
||||
sub := new(bytes.Buffer)
|
||||
writeU16 := func(v uint16) { _ = binary.Write(sub, binary.BigEndian, v) }
|
||||
length := 16 + segCount*8
|
||||
writeU16(4) // format
|
||||
writeU16(uint16(length)) // length
|
||||
writeU16(0) // language
|
||||
writeU16(uint16(segCount * 2)) // segCountX2
|
||||
writeU16(0) // searchRange(解析器不校验)
|
||||
writeU16(0) // entrySelector
|
||||
writeU16(0) // rangeShift
|
||||
for _, v := range endCodes {
|
||||
writeU16(v)
|
||||
}
|
||||
writeU16(0) // reservedPad
|
||||
for _, v := range startCodes {
|
||||
writeU16(v)
|
||||
}
|
||||
for _, v := range idDeltas {
|
||||
writeU16(v)
|
||||
}
|
||||
for _, v := range rangeOffsets {
|
||||
writeU16(v)
|
||||
}
|
||||
subBytes := sub.Bytes()
|
||||
|
||||
// cmap 头 + 一个子表记录
|
||||
cmap.Write([]byte{0, 0})
|
||||
_ = binary.Write(&cmap, binary.BigEndian, uint16(1))
|
||||
_ = binary.Write(&cmap, binary.BigEndian, uint16(3)) // platformID = Windows
|
||||
_ = binary.Write(&cmap, binary.BigEndian, uint16(1)) // encodingID = Unicode BMP
|
||||
_ = binary.Write(&cmap, binary.BigEndian, uint32(12))
|
||||
cmap.Write(subBytes)
|
||||
|
||||
head := make([]byte, 54)
|
||||
binary.BigEndian.PutUint16(head[50:], 0) // indexToLocFormat = 0(短 loca)
|
||||
maxp := make([]byte, 6)
|
||||
binary.BigEndian.PutUint16(maxp[4:], numGlyphs)
|
||||
|
||||
tables := []struct {
|
||||
tag string
|
||||
data []byte
|
||||
}{
|
||||
{"cmap", cmap.Bytes()},
|
||||
{"glyf", glyfData},
|
||||
{"loca", loca},
|
||||
{"head", head},
|
||||
{"maxp", maxp},
|
||||
}
|
||||
|
||||
var out bytes.Buffer
|
||||
out.Write([]byte{0x00, 0x01, 0x00, 0x00}) // sfntVersion
|
||||
_ = binary.Write(&out, binary.BigEndian, uint16(len(tables)))
|
||||
_ = binary.Write(&out, binary.BigEndian, uint16(0))
|
||||
_ = binary.Write(&out, binary.BigEndian, uint16(0))
|
||||
_ = binary.Write(&out, binary.BigEndian, uint16(0))
|
||||
offset := 12 + len(tables)*16
|
||||
offsetsTable := make([]int, len(tables))
|
||||
for i, tb := range tables {
|
||||
padded := tb.data
|
||||
if len(padded)%4 != 0 {
|
||||
padded = append(padded, make([]byte, 4-len(padded)%4)...)
|
||||
}
|
||||
offsetsTable[i] = offset
|
||||
offset += len(padded)
|
||||
}
|
||||
for i, tb := range tables {
|
||||
out.WriteString(tb.tag)
|
||||
_ = binary.Write(&out, binary.BigEndian, uint32(0))
|
||||
_ = binary.Write(&out, binary.BigEndian, uint32(offsetsTable[i]))
|
||||
_ = binary.Write(&out, binary.BigEndian, uint32(len(tb.data)))
|
||||
}
|
||||
for _, tb := range tables {
|
||||
out.Write(tb.data)
|
||||
for out.Len()%4 != 0 {
|
||||
out.WriteByte(0)
|
||||
}
|
||||
}
|
||||
return out.Bytes()
|
||||
}
|
||||
|
||||
// 解析出的码点 → 字形 → 码点映射与构造时一致。
|
||||
func TestQueryTTFParseRoundTrip(t *testing.T) {
|
||||
data := buildTestTTF(t, map[rune]uint16{
|
||||
'A': 1, 'B': 2, 'C': 3, 'D': 4,
|
||||
})
|
||||
font, err := parseQueryTTFFont(data)
|
||||
if err != nil {
|
||||
t.Fatalf("解析字体失败: %v", err)
|
||||
}
|
||||
for cp, gid := range map[rune]uint16{'A': 1, 'B': 2, 'C': 3, 'D': 4} {
|
||||
if got := font.unicodeToGlyphID[cp]; got != gid {
|
||||
t.Fatalf("码点 %q 的字形下标 = %d,期望 %d", cp, got, gid)
|
||||
}
|
||||
}
|
||||
if font.glyphToUnicode[font.unicodeToGlyph['A']] != 'A' {
|
||||
t.Fatal("字形 → 码点映射不正确")
|
||||
}
|
||||
if font.unicodeToGlyph['A'] == font.unicodeToGlyph['B'] {
|
||||
t.Fatal("不同码点的轮廓不应相同")
|
||||
}
|
||||
}
|
||||
|
||||
// replaceFont:错误字体把 A 渲染成 B 的字形,正确字体应把 A 还原成 B。
|
||||
func TestReplaceFontRestoresText(t *testing.T) {
|
||||
// 错误字体:码点 A 指向字形 2(也就是 B 的形状)。
|
||||
errorFontData := buildTestTTF(t, map[rune]uint16{'A': 2, 'B': 3, 'C': 4, 'D': 5})
|
||||
// 正确字体:码点 B 指向字形 2。
|
||||
correctFontData := buildTestTTF(t, map[rune]uint16{'A': 1, 'B': 2, 'C': 3, 'D': 4})
|
||||
errorFont, err := parseQueryTTFFont(errorFontData)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
correctFont, err := parseQueryTTFFont(correctFontData)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// 页面上写的是 'A',实际字形是 B → 应还原为 'B'。
|
||||
got := replaceFontText("A", errorFont, correctFont, false)
|
||||
if got != "B" {
|
||||
t.Fatalf("replaceFont 还原结果 = %q,期望 %q", got, "B")
|
||||
}
|
||||
// 空白与未知码点保持原样。
|
||||
mixed := replaceFontText("A 中", errorFont, correctFont, false)
|
||||
if !strings.HasPrefix(mixed, "B ") || !strings.HasSuffix(mixed, "中") {
|
||||
t.Fatalf("混合文本处理异常: %q", mixed)
|
||||
}
|
||||
// filter=true 时删掉没有对应字形的字符。
|
||||
filtered := replaceFontText("A中", errorFont, correctFont, true)
|
||||
if filtered != "B" {
|
||||
t.Fatalf("filter 结果 = %q,期望 %q", filtered, "B")
|
||||
}
|
||||
}
|
||||
|
||||
// 坏字体只应报错,不能 panic。
|
||||
func TestQueryTTFBadFontNoPanic(t *testing.T) {
|
||||
cases := [][]byte{
|
||||
nil,
|
||||
[]byte("not a font"),
|
||||
[]byte("ttcf"),
|
||||
append([]byte{0x00, 0x01, 0x00, 0x00, 0x00, 0x02}, make([]byte, 40)...),
|
||||
}
|
||||
for i, data := range cases {
|
||||
if _, err := parseQueryTTFFont(data); err == nil {
|
||||
t.Fatalf("坏字体 #%d 应返回错误", i)
|
||||
}
|
||||
}
|
||||
// 截断的合法字体也不能 panic。
|
||||
full := buildTestTTF(t, map[rune]uint16{'A': 1})
|
||||
for cut := 1; cut < len(full); cut += 37 {
|
||||
_, _ = parseQueryTTFFont(full[:cut])
|
||||
}
|
||||
}
|
||||
@@ -43,6 +43,10 @@ type Request struct {
|
||||
// 返回而不是解码成文本(对应 legado AnalyzeUrl.type)。
|
||||
// 书源用它配合 data: 地址当参数信封,见 datauri.go。
|
||||
HexBody bool
|
||||
// Raw 请求响应按「原始字节」返回,跳过 charset 解码。
|
||||
// 供 java.downloadFile / java.cacheFile 这类需要保真落盘的调用使用:
|
||||
// 一旦按 UTF-8 解码,GBK 字节会被替换成 U+FFFD,写出来的文件就坏了。
|
||||
Raw bool
|
||||
// BodyJsFn 对应 UrlOption.bodyJs:响应体二次处理(JS 执行闭包)。
|
||||
BodyJsFn func(body string) string
|
||||
// Unsupported 非 nil 表示该请求依赖当前阶段不支持的能力,
|
||||
@@ -335,6 +339,58 @@ func findParamSplit(s string) (start, end int, ok bool) {
|
||||
return 0, 0, false
|
||||
}
|
||||
|
||||
// FindParamSplit 导出 findParamSplit:图片等媒体地址也支持 ",{...}" 选项段,
|
||||
// 服务层需要先拆分再分别做请求与展示处理。
|
||||
func FindParamSplit(s string) (start, end int, ok bool) {
|
||||
return findParamSplit(s)
|
||||
}
|
||||
|
||||
// MediaOptions 是媒体地址尾部 ",{...}" 选项段里服务端允许应用的部分。
|
||||
//
|
||||
// 图片/音频等媒体请求走代理,选项里只有请求头与重试这类「取图必需」的字段有意义;
|
||||
// webView/webJs/js/bodyJs 这些需要浏览器或脚本执行引擎的字段必须忽略,
|
||||
// 否则一个书源就能让媒体代理变成任意请求的中转。
|
||||
type MediaOptions struct {
|
||||
Headers map[string]string
|
||||
Charset string
|
||||
Retry *int
|
||||
Method string
|
||||
}
|
||||
|
||||
// ParseMediaOptions 拆分媒体地址与其尾部选项。
|
||||
// 返回去掉选项段的地址(已 trim)、应用后的选项与「是否带选项」。
|
||||
func ParseMediaOptions(raw string) (string, MediaOptions, bool) {
|
||||
var out MediaOptions
|
||||
s := strings.TrimSpace(raw)
|
||||
st, end, ok := findParamSplit(s)
|
||||
if !ok {
|
||||
return s, out, false
|
||||
}
|
||||
base := strings.TrimSpace(s[:st])
|
||||
optionStr := strings.TrimSpace(s[end:])
|
||||
var option URLOption
|
||||
if err := json.Unmarshal([]byte(optionStr), &option); err != nil {
|
||||
if err2 := json.Unmarshal([]byte(strings.TrimPrefix(optionStr, ",")), &option); err2 != nil {
|
||||
return base, out, false
|
||||
}
|
||||
}
|
||||
if len(option.Headers) > 0 {
|
||||
out.Headers = make(map[string]string, len(option.Headers))
|
||||
for k, v := range option.Headers {
|
||||
out.Headers[k] = anyToString(v)
|
||||
}
|
||||
}
|
||||
out.Charset = option.Charset
|
||||
out.Retry = option.Retry
|
||||
switch strings.ToUpper(option.Method) {
|
||||
case "HEAD":
|
||||
out.Method = "HEAD"
|
||||
default:
|
||||
out.Method = "GET"
|
||||
}
|
||||
return base, out, true
|
||||
}
|
||||
|
||||
func isSpaceByte(c byte) bool {
|
||||
return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\v' || c == '\f'
|
||||
}
|
||||
|
||||
@@ -0,0 +1,301 @@
|
||||
package reader
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"golang.org/x/text/encoding/htmlindex"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/helper"
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
"github.com/truewhile/MeBox/internal/service/reader/rule"
|
||||
)
|
||||
|
||||
// 本文件覆盖规则稳定性修复的回归:
|
||||
// - @put 写回书源变量(跨请求持久化)
|
||||
// - book.putVariable 持久化到书架记录
|
||||
// - 书源正则的匹配超时(灾难性回溯不会挂死)
|
||||
// - helper.DecompressBody 的解压上限
|
||||
// - 非 UTF-8 TXT 的导入/读取
|
||||
// - 图片 URL 尾部 options 的请求头应用
|
||||
|
||||
// @put 必须写到书源变量(legado BaseSource.putVariable),而不是一次性的书籍变量。
|
||||
func TestPutPersistsToSourceVariable(t *testing.T) {
|
||||
svc, _ := newLoginTestService(t)
|
||||
srcURL := "https://put.example.com"
|
||||
src := &model.ReaderBookSource{Name: "put 源", SourceURL: srcURL}
|
||||
bs := &BookSource{BookSourceURL: srcURL, RuleContent: &ContentRule{Content: strPtr("id.content@textNodes")}}
|
||||
|
||||
sess := svc.newSession(t.Context(), src, bs)
|
||||
ar := sess.newAnalyzer("", 0, "<html></html>", srcURL)
|
||||
ar.Put("token", "abc123")
|
||||
sess.close()
|
||||
|
||||
// 换一个全新会话/解析器:@put 的值必须还能 @get 到(已落库)。
|
||||
sess2 := svc.newSession(t.Context(), src, bs)
|
||||
defer sess2.close()
|
||||
ar2 := sess2.newAnalyzer("", 0, "<html></html>", srcURL)
|
||||
if got := ar2.Get("token"); got != "abc123" {
|
||||
t.Fatalf("@put 未持久化:@get 得到 %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 书源 variables 默认值只是兜底:@put 写入的同名值优先级更高。
|
||||
func TestPutOverridesSourceDefaultVariable(t *testing.T) {
|
||||
svc, _ := newLoginTestService(t)
|
||||
srcURL := "https://put2.example.com"
|
||||
raw := `{"bookSourceUrl":"https://put2.example.com","variables":"{\"token\":\"default\"}"}`
|
||||
srcID := importTestSource(t, svc, raw, srcURL)
|
||||
src, err := svc.repo.GetSource(context.Background(), srcID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bs, err := ParseBookSource(src.RawJSON)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sess := svc.newSession(t.Context(), src, bs)
|
||||
ar := sess.newAnalyzer("", 0, "<html></html>", srcURL)
|
||||
if got := ar.Get("token"); got != "default" {
|
||||
t.Fatalf("默认变量应可读,得到 %q", got)
|
||||
}
|
||||
ar.Put("token", "override")
|
||||
sess.close()
|
||||
|
||||
sess2 := svc.newSession(t.Context(), src, bs)
|
||||
defer sess2.close()
|
||||
ar2 := sess2.newAnalyzer("", 0, "<html></html>", srcURL)
|
||||
if got := ar2.Get("token"); got != "override" {
|
||||
t.Fatalf("@put 应覆盖默认变量,得到 %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// book.putVariable 写入的变量要落回书架记录(对应 legado Book.upVariable)。
|
||||
func TestBookPutVariablePersists(t *testing.T) {
|
||||
svc, srv, book := prepareCacheTestBook(t)
|
||||
ctx := t.Context()
|
||||
book.Variable = `{"tone_id":"keep"}`
|
||||
if err := svc.repo.UpdateBook(ctx, book); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
loaded, err := svc.repo.GetBook(ctx, book.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
src, bs, err := svc.loadSourceFlexible(ctx, "", book.Origin)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sess := svc.newSession(ctx, src, bs)
|
||||
sess.book = loaded
|
||||
ar := sess.newAnalyzer("", 0, "<html></html>", book.Origin)
|
||||
sess.applyBookContext(ar, book.BookURL, loaded, "", 0)
|
||||
|
||||
// 书源 JS 的 book.getVariable / putVariable。
|
||||
runner := sess.runner("", 0)
|
||||
if _, err := runner.Run(ar, `book.putVariable('token','v-42'); book.getVariable('tone_id')`, nil, book.Origin); err != nil {
|
||||
t.Fatalf("book JS 执行失败: %v", err)
|
||||
}
|
||||
sess.close()
|
||||
|
||||
stored, err := svc.repo.GetBook(ctx, book.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
vars := parseBookVariableMap(stored.Variable)
|
||||
if vars["token"] != "v-42" {
|
||||
t.Fatalf("book.putVariable 未持久化: %q", stored.Variable)
|
||||
}
|
||||
if vars["tone_id"] != "keep" {
|
||||
t.Fatalf("已有变量被覆盖: %q", stored.Variable)
|
||||
}
|
||||
_ = srv
|
||||
}
|
||||
|
||||
// 灾难性回溯的正则必须在秒级返回,而不是挂死 goroutine。
|
||||
func TestRegexAnalyzerTimeoutOnCatastrophicBacktracking(t *testing.T) {
|
||||
// (a+)+$ 对 "aaaa...b" 是指数级回溯;没有 MatchTimeout 会永久卡住。
|
||||
evil := strings.Repeat("a", 40) + "b"
|
||||
done := make(chan string, 1)
|
||||
go func() {
|
||||
done <- rule.ApplyReplaceRegexString(evil, "##(a+)+$##X")
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(15 * time.Second):
|
||||
t.Fatal("书源正则未在超时预算内返回(MatchTimeout 未生效)")
|
||||
}
|
||||
}
|
||||
|
||||
// 解压炸弹:超过输出上限时原样返回压缩字节(调用方按失败处理),不撑爆内存。
|
||||
func TestDecompressBodyRejectsBomb(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
gz := gzip.NewWriter(&buf)
|
||||
// 128MB 的零字节压成几十 KB。
|
||||
chunk := make([]byte, 1<<20)
|
||||
for i := 0; i < 128; i++ {
|
||||
if _, err := gz.Write(chunk); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := gz.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp := &http.Response{Header: http.Header{"Content-Encoding": []string{"gzip"}}}
|
||||
out := helper.DecompressBody(resp, buf.Bytes())
|
||||
if len(out) == len(chunk)*128 {
|
||||
t.Fatal("解压炸弹应被上限拦截")
|
||||
}
|
||||
if !bytes.Equal(out, buf.Bytes()) {
|
||||
t.Fatal("超限时应原样返回压缩字节")
|
||||
}
|
||||
}
|
||||
|
||||
// 正常 gzip 响应仍应被正确解压(上限不误伤常规页面)。
|
||||
func TestDecompressBodyNormal(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
gz := gzip.NewWriter(&buf)
|
||||
payload := []byte("<html>hello</html>")
|
||||
if _, err := gz.Write(payload); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = gz.Close()
|
||||
resp := &http.Response{Header: http.Header{"Content-Encoding": []string{"gzip"}}}
|
||||
if out := helper.DecompressBody(resp, buf.Bytes()); string(out) != string(payload) {
|
||||
t.Fatalf("正常 gzip 解压失败: %q", out)
|
||||
}
|
||||
}
|
||||
|
||||
// 非 UTF-8(GBK)TXT:导入落盘为 UTF-8,按章节读取不再乱码/错位。
|
||||
func TestImportGBKTextBookReadsCorrectly(t *testing.T) {
|
||||
// 用临时 DataDir,避免本地书籍文件写进仓库目录。
|
||||
svc := newLocalBookService(t)
|
||||
ctx := t.Context()
|
||||
|
||||
// 用 GBK 编码两章正文(解码后章节字节区间按 UTF-8 计算)。
|
||||
gbkBytes := func(s string) []byte {
|
||||
enc, err := htmlindex.Get("gbk")
|
||||
if err != nil {
|
||||
t.Skip("环境缺少 GBK 编码支持")
|
||||
}
|
||||
out, err := enc.NewEncoder().Bytes([]byte(s))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
text := "第一章 起点\n这是第一章的正文内容,包含中文标点。\n第二章 继续\n这是第二章的正文内容。\n"
|
||||
book, err := svc.ImportLocalBook(ctx, "u1", "gbk小说.txt", gbkBytes(text))
|
||||
if err != nil {
|
||||
t.Fatalf("导入 GBK 书籍失败: %v", err)
|
||||
}
|
||||
if book.Charset != "gbk" {
|
||||
t.Fatalf("应识别为 gbk,实际 %q", book.Charset)
|
||||
}
|
||||
chapters, err := svc.ListChapters(ctx, book.ID)
|
||||
if err != nil || len(chapters) < 2 {
|
||||
t.Fatalf("应切出至少 2 章: n=%d err=%v", len(chapters), err)
|
||||
}
|
||||
second, err := svc.readLocalChapter(book, chapters[1])
|
||||
if err != nil {
|
||||
t.Fatalf("读取第二章失败: %v", err)
|
||||
}
|
||||
if !strings.Contains(second, "第二章的正文内容") {
|
||||
t.Fatalf("第二章内容不正确(编码错位): %q", second)
|
||||
}
|
||||
}
|
||||
|
||||
// 图片 URL 尾部 options:拆分出请求头,且地址本身去掉选项段。
|
||||
func TestParseMediaOptions(t *testing.T) {
|
||||
raw := `https://img.example.com/a.jpg,{"headers":{"Referer":"https://site.example.com/","X-Token":"t1"},"retry":2}`
|
||||
base, opt, ok := rule.ParseMediaOptions(raw)
|
||||
if !ok {
|
||||
t.Fatal("应识别出选项段")
|
||||
}
|
||||
if base != "https://img.example.com/a.jpg" {
|
||||
t.Fatalf("基础地址错误: %q", base)
|
||||
}
|
||||
if opt.Headers["Referer"] != "https://site.example.com/" || opt.Headers["X-Token"] != "t1" {
|
||||
t.Fatalf("请求头解析错误: %+v", opt.Headers)
|
||||
}
|
||||
if opt.Retry == nil || *opt.Retry != 2 {
|
||||
t.Fatalf("retry 解析错误: %+v", opt.Retry)
|
||||
}
|
||||
// 无选项段的普通地址原样返回。
|
||||
if b, _, ok := rule.ParseMediaOptions("https://img.example.com/b.jpg"); ok || b != "https://img.example.com/b.jpg" {
|
||||
t.Fatalf("无选项地址不应被改写: %q ok=%v", b, ok)
|
||||
}
|
||||
}
|
||||
|
||||
// 代理地址改写后,图片选项头必须在媒体请求里真正生效。
|
||||
func TestFetchMediaAppliesOptionHeaders(t *testing.T) {
|
||||
var gotReferer, gotToken string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotReferer = r.Header.Get("Referer")
|
||||
gotToken = r.Header.Get("X-Token")
|
||||
w.Header().Set("Content-Type", "image/jpeg")
|
||||
_, _ = w.Write([]byte("jpeg-bytes"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
svc, _ := newLoginTestService(t)
|
||||
book := &model.ReaderBook{Base: model.Base{ID: "b1"}, Origin: srv.URL, BookURL: srv.URL + "/book/1"}
|
||||
resp, err := svc.FetchMediaWithOptions(t.Context(), book, srv.URL+"/a.jpg", "", map[string]string{
|
||||
"Referer": "https://site.example.com/",
|
||||
"X-Token": "t1",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("拉取失败: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
if gotReferer != "https://site.example.com/" || gotToken != "t1" {
|
||||
t.Fatalf("选项请求头未生效: referer=%q token=%q", gotReferer, gotToken)
|
||||
}
|
||||
}
|
||||
|
||||
// ProxyURL 去掉选项段后再签名:带选项与不带选项的同一张图得到同一签名。
|
||||
func TestProxyURLStripsMediaOptions(t *testing.T) {
|
||||
svc, _ := newLoginTestService(t)
|
||||
plain := svc.ProxyURL("book-1", "https://img.example.com/a.jpg")
|
||||
withOptions := svc.ProxyURL("book-1", `https://img.example.com/a.jpg,{"headers":{"Referer":"https://s/"}}`)
|
||||
if plain != withOptions {
|
||||
t.Fatalf("选项段应被剥离后签名:\n%s\n%s", plain, withOptions)
|
||||
}
|
||||
raw, err := svc.VerifyProxyURL("book-1", strings.TrimPrefix(strings.Split(plain, "&u=")[1], ""), "")
|
||||
if err == nil && raw != "" {
|
||||
_ = raw
|
||||
}
|
||||
// 拆出 u/s 校验签名可回推出干净地址。
|
||||
u := plain[strings.Index(plain, "&u=")+3:]
|
||||
if i := strings.Index(u, "&s="); i >= 0 {
|
||||
u = u[:i]
|
||||
}
|
||||
if decoded, err := svc.VerifyProxyURL("book-1", u, plain[strings.Index(plain, "&s=")+3:]); err != nil || decoded != "https://img.example.com/a.jpg" {
|
||||
t.Fatalf("签名校验失败: %q err=%v", decoded, err)
|
||||
}
|
||||
}
|
||||
|
||||
// HLS 判定:带 query 的 m3u8 与 Content-Type 不标准都要识别;普通图片不能被误判。
|
||||
func TestLooksLikeHLSURL(t *testing.T) {
|
||||
cases := map[string]bool{
|
||||
"https://a.com/index.m3u8": true,
|
||||
"https://a.com/index.m3u8?token=xx": true,
|
||||
"https://a.com/live/index.M3U": true,
|
||||
"https://a.com/a.mp3": false,
|
||||
"https://a.com/m3u8/1": false,
|
||||
}
|
||||
for raw, want := range cases {
|
||||
if got := looksLikeHLSURL(raw); got != want {
|
||||
t.Fatalf("looksLikeHLSURL(%q)=%v,期望 %v", raw, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -57,7 +57,7 @@ func TestSearchUsesRequestedPage(t *testing.T) {
|
||||
{0, "1"}, // 非法页码退回第 1 页
|
||||
{-3, "1"}, // 负数同理
|
||||
} {
|
||||
books, skipped, err := svc.Search(t.Context(), "关键词", nil, tc.in)
|
||||
books, skipped, err := svc.Search(t.Context(), "u1", "关键词", nil, tc.in)
|
||||
if err != nil {
|
||||
t.Fatalf("page=%d 搜索失败: %v", tc.in, err)
|
||||
}
|
||||
@@ -88,7 +88,7 @@ func TestSearchWithoutPagePlaceholderStillWorks(t *testing.T) {
|
||||
svc, _ := newLoginTestService(t)
|
||||
_ = prepareLoginSource(t, svc, scopeEchoSourceJSON(t, "单页源", srv.URL))
|
||||
|
||||
books, _, err := svc.Search(t.Context(), "书", nil, 3)
|
||||
books, _, err := svc.Search(t.Context(), "u1", "书", nil, 3)
|
||||
if err != nil {
|
||||
t.Fatalf("搜索失败: %v", err)
|
||||
}
|
||||
|
||||
@@ -46,7 +46,7 @@ func TestSearchSendsQueryString(t *testing.T) {
|
||||
svc, _ := newLoginTestService(t)
|
||||
_ = prepareLoginSource(t, svc, queryEchoSource(srv.URL))
|
||||
|
||||
books, skipped, err := svc.Search(t.Context(), "宠魅", nil, 1)
|
||||
books, skipped, err := svc.Search(t.Context(), "u1", "宠魅", nil, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("搜索失败: %v", err)
|
||||
}
|
||||
@@ -87,7 +87,7 @@ func TestSearchResponseUsesEmptyArrays(t *testing.T) {
|
||||
svc, _ := newLoginTestService(t)
|
||||
_ = prepareLoginSource(t, svc, queryEchoSource(srv.URL))
|
||||
|
||||
books, skipped, err := svc.Search(t.Context(), "宠魅", nil, 1)
|
||||
books, skipped, err := svc.Search(t.Context(), "u1", "宠魅", nil, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("搜索失败: %v", err)
|
||||
}
|
||||
@@ -130,7 +130,7 @@ func TestSearchPreservesExistingPercentEncoding(t *testing.T) {
|
||||
out, _ := json.Marshal(src)
|
||||
_ = prepareLoginSource(t, svc, string(out))
|
||||
|
||||
if _, _, err := svc.Search(t.Context(), "宠魅", nil, 1); err != nil {
|
||||
if _, _, err := svc.Search(t.Context(), "u1", "宠魅", nil, 1); err != nil {
|
||||
t.Fatalf("搜索失败: %v", err)
|
||||
}
|
||||
if gotQuery != "宠魅" {
|
||||
|
||||
@@ -53,11 +53,11 @@ func TestSearchRespectsSourceScope(t *testing.T) {
|
||||
scopeEchoSourceJSON(t, "源A", srvA.URL),
|
||||
scopeEchoSourceJSON(t, "源B", srvB.URL),
|
||||
} {
|
||||
if _, err := svc.ImportSources(t.Context(), raw); err != nil {
|
||||
if _, err := svc.ImportSources(t.Context(), "u1", raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
sources, err := svc.ListSources(t.Context())
|
||||
sources, err := svc.ListSources(t.Context(), "u1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -75,7 +75,7 @@ func TestSearchRespectsSourceScope(t *testing.T) {
|
||||
}
|
||||
|
||||
// 指定单个源:只应搜到该源的结果
|
||||
books, skipped, err := svc.Search(t.Context(), "书", []string{idByName["源A"]}, 1)
|
||||
books, skipped, err := svc.Search(t.Context(), "u1", "书", []string{idByName["源A"]}, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -87,7 +87,7 @@ func TestSearchRespectsSourceScope(t *testing.T) {
|
||||
}
|
||||
|
||||
// 空范围:默认全部启用书源,两个源都应命中
|
||||
books, _, err = svc.Search(t.Context(), "书", nil, 1)
|
||||
books, _, err = svc.Search(t.Context(), "u1", "书", nil, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -96,7 +96,7 @@ func TestSearchRespectsSourceScope(t *testing.T) {
|
||||
}
|
||||
|
||||
// 范围里的源已不存在(删源/换设备)时退回全部启用,而不是搜不到
|
||||
books, _, err = svc.Search(t.Context(), "书", []string{"not-a-real-source"}, 1)
|
||||
books, _, err = svc.Search(t.Context(), "u1", "书", []string{"not-a-real-source"}, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -28,7 +28,10 @@ type BookSource struct {
|
||||
LoginCheckJS *string `json:"loginCheckJs"`
|
||||
JSLib *string `json:"jsLib"`
|
||||
BookSourceComment *string `json:"bookSourceComment"`
|
||||
LastUpdateTime *int64 `json:"lastUpdateTime"`
|
||||
// CoverDecodeJs 封面 bytes 二次解密 JS(返回解密后的字节),
|
||||
// 对应 legado BookSource.coverDecodeJs。规则为空时封面零开销直通。
|
||||
CoverDecodeJs *string `json:"coverDecodeJs"`
|
||||
LastUpdateTime *int64 `json:"lastUpdateTime"`
|
||||
RespondTime *int64 `json:"respondTime"`
|
||||
Weight *int `json:"weight"`
|
||||
ExploreURL *string `json:"exploreUrl"`
|
||||
@@ -151,7 +154,10 @@ type ContentRule struct {
|
||||
SourceRegex *string `json:"sourceRegex"`
|
||||
ReplaceRegex *string `json:"replaceRegex"`
|
||||
ImageStyle *string `json:"imageStyle"`
|
||||
PayAction *string `json:"payAction"`
|
||||
// ImageDecode 正文图片 bytes 二次解密 JS(返回解密后的字节),
|
||||
// 对应 legado ContentRule.imageDecode。
|
||||
ImageDecode *string `json:"imageDecode"`
|
||||
PayAction *string `json:"payAction"`
|
||||
}
|
||||
|
||||
// ExploreRule 发现规则。
|
||||
|
||||
@@ -38,7 +38,7 @@ func TestImportSourcesAcceptsMessySource(t *testing.T) {
|
||||
svc, _ := newLoginTestService(t)
|
||||
ctx := t.Context()
|
||||
|
||||
imported, err := svc.ImportSources(ctx, messySourceJSON)
|
||||
imported, err := svc.ImportSources(ctx, "u1", messySourceJSON)
|
||||
if err != nil {
|
||||
t.Fatalf("导入失败: %v", err)
|
||||
}
|
||||
@@ -46,7 +46,7 @@ func TestImportSourcesAcceptsMessySource(t *testing.T) {
|
||||
t.Fatalf("导入数量 = %d,期望 1", imported)
|
||||
}
|
||||
|
||||
bs, err := svc.repo.GetSourceByURL(ctx, "https://www.mangacopy.com/")
|
||||
bs, err := svc.repo.GetSourceByURL(ctx, "u1", "https://www.mangacopy.com/")
|
||||
if err != nil || bs == nil {
|
||||
t.Fatalf("按 URL 查不到导入的书源: %v", err)
|
||||
}
|
||||
@@ -139,7 +139,7 @@ func TestParseBookSourceRejectsGarbage(t *testing.T) {
|
||||
func TestImportSourcesReportsReasonWhenNothingImported(t *testing.T) {
|
||||
svc, _ := newLoginTestService(t)
|
||||
|
||||
_, err := svc.ImportSources(t.Context(), `[{"bookSourceUrl":123,"bookSourceName":"坏源"}]`)
|
||||
_, err := svc.ImportSources(t.Context(), "u1", `[{"bookSourceUrl":123,"bookSourceName":"坏源"}]`)
|
||||
if err == nil {
|
||||
t.Fatal("一条都没导入时应当返回错误")
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user