diff --git a/internal/service/subtitle.go b/internal/service/subtitle.go index a7b2f25..d4d9b31 100644 --- a/internal/service/subtitle.go +++ b/internal/service/subtitle.go @@ -410,14 +410,10 @@ func (s *SubtitleService) Serve(ctx context.Context, mediaID, sub string, w io.W } return s.serveEmbedded(ctx, m, index, w) } - abs, err := filepath.Abs(sub) + abs, err := readExternalSubtitlePath(m, sub) if err != nil { return err } - mediaDir, _ := filepath.Abs(filepath.Dir(m.Path)) - if !pathWithin(abs, mediaDir) { - return fmt.Errorf("path escape") - } f, err := os.Open(abs) // #nosec G304 -- abs is constrained to the media file directory with pathWithin. if err != nil { @@ -428,14 +424,17 @@ func (s *SubtitleService) Serve(ctx context.Context, mediaID, sub string, w io.W if err != nil { return err } + // 非 UTF-8 的外挂字幕(UTF-16、GBK/Big5 等)必须先归一化:浏览器只能按 + // UTF-8 解析 内容,否则整篇都会变成替换字符。 + text := decodeSubtitleText(body) switch strings.ToLower(filepath.Ext(abs)) { case ".vtt": - _, err = w.Write(body) + _, err = io.WriteString(w, text) case ".srt": - _, err = w.Write([]byte(srtToVTT(string(body)))) + _, err = io.WriteString(w, srtToVTT(text)) case ".ass", ".ssa": - _, err = w.Write([]byte(assToVTT(string(body)))) + _, err = io.WriteString(w, assToVTT(text)) default: return errors.New("unsupported subtitle format") } @@ -517,12 +516,42 @@ func (s *SubtitleService) ServeASS(ctx context.Context, mediaID, sub string, w i } switch strings.ToLower(filepath.Ext(sub)) { case ".ass", ".ssa": - return s.ServeRaw(ctx, mediaID, sub, w) + // 外挂 ASS 同样要归一化成 UTF-8:libass 只认 UTF-8,UTF-16/GBK 的 + // 字幕交给它会解析不到任何事件,表现为「字幕选中了却不显示」。 + abs, err := readExternalSubtitlePath(m, sub) + if err != nil { + return err + } + f, err := os.Open(abs) // #nosec G304 -- abs is constrained to the media file directory with pathWithin. + if err != nil { + return err + } + defer f.Close() + body, err := io.ReadAll(f) + if err != nil { + return err + } + _, err = io.WriteString(w, decodeSubtitleText(body)) + return err default: return errors.New("subtitle is not ASS/SSA") } } +// readExternalSubtitlePath 校验外挂字幕路径(必须落在媒体文件所在目录内)并返回 +// 绝对路径。Serve 与 ServeASS 共用,避免两处各自实现出现安全口径不一致。 +func readExternalSubtitlePath(m *model.Media, sub string) (string, error) { + abs, err := filepath.Abs(sub) + if err != nil { + return "", err + } + mediaDir, _ := filepath.Abs(filepath.Dir(m.Path)) + if !pathWithin(abs, mediaDir) { + return "", fmt.Errorf("path escape") + } + return abs, nil +} + func (s *SubtitleService) serveEmbeddedASS(ctx context.Context, media *model.Media, streamIndex int, w io.Writer) error { input, err := s.resolveInput(ctx, media) if err != nil { diff --git a/internal/service/subtitle_encoding.go b/internal/service/subtitle_encoding.go new file mode 100644 index 0000000..c445a4d --- /dev/null +++ b/internal/service/subtitle_encoding.go @@ -0,0 +1,167 @@ +package service + +import ( + "bytes" + "encoding/binary" + "math" + "regexp" + "strings" + "unicode" + "unicode/utf16" + "unicode/utf8" + + "golang.org/x/text/encoding" + "golang.org/x/text/encoding/japanese" + "golang.org/x/text/encoding/korean" + "golang.org/x/text/encoding/simplifiedchinese" + "golang.org/x/text/encoding/traditionalchinese" +) + +// 外挂字幕文件的编码归一化。 +// +// 字幕文件由字幕组/工具生成,编码并不统一:除 UTF-8 外,中文圈常见 GBK/GB18030, +// 港台常见 Big5,部分 Windows 工具(Aegisub 早期版本、Subtitle Edit 的某些导出) +// 会写成带 BOM 的 UTF-16LE。浏览器 与 libass(JASSUB)都只按 UTF-8 解析, +// 后端如果原样下发字节,非 UTF-8 的字幕会整篇变成替换字符——表现就是「字幕轨已经 +// 选中,但画面上一个字都不显示」。所以外挂字幕在服务端就要统一解码成 UTF-8。 +// +// 注意:只用于下发给浏览器/播放器的文本路径。Emby 兼容层的 /subtitles 原始字节 +// 接口(ServeRaw)必须保持字节一致,不能经过这里。 + +// 各编码的 BOM 前缀。 +var ( + utf8BOM = []byte{0xEF, 0xBB, 0xBF} + utf16BEBOM = []byte{0xFE, 0xFF} + utf16LEBOM = []byte{0xFF, 0xFE} +) + +// subtitleLegacyEncodings 是无 BOM 时的候选编码,按优先级排列。 +// GB18030 是 GBK/GB2312 的超集,中文外挂字幕绝大多数是它,因此排在最前—— +// 多个候选都能解出合法字幕时,靠前的优先。 +var subtitleLegacyEncodings = []struct { + name string + enc encoding.Encoding +}{ + {"gb18030", simplifiedchinese.GB18030}, + {"big5", traditionalchinese.Big5}, + {"shift_jis", japanese.ShiftJIS}, + {"euc-kr", korean.EUCKR}, +} + +// subtitleTimecodePattern 匹配 SRT(00:00:01,000 --> ...)与 WebVTT +// (00:00:01.000 --> ...)的时间行。 +var subtitleTimecodePattern = regexp.MustCompile(`\d{1,2}:\d{2}:\d{2}[,.]\d{1,3}\s*-->`) + +// decodeSubtitleText 把字幕文件字节归一化成 UTF-8 文本。识别不出编码时原样返回, +// 保证不会比「直接透传」更糟。 +func decodeSubtitleText(raw []byte) string { + switch { + case bytes.HasPrefix(raw, utf8BOM): + return string(raw[len(utf8BOM):]) + case bytes.HasPrefix(raw, utf16LEBOM): + return decodeUTF16(raw[len(utf16LEBOM):], false) + case bytes.HasPrefix(raw, utf16BEBOM): + return decodeUTF16(raw[len(utf16BEBOM):], true) + } + + // 无 BOM 且是合法 UTF-8:这是绝大多数情况,直接返回。 + if utf8.Valid(raw) { + return string(raw) + } + + best := "" + bestScore := math.MinInt + consider := func(text string) { + if !utf8.ValidString(text) || !looksLikeSubtitle(text) { + return + } + score := subtitlePlausibility(text) + if score > bestScore { + best, bestScore = text, score + } + } + + // 无 BOM 的 UTF-16 少见(多来自手工编辑),但一旦是它,所有单字节编码都会 + // 解出带大量 NUL 的垃圾,因此先用「格式特征」筛掉这些候选。 + consider(decodeUTF16(raw, false)) + consider(decodeUTF16(raw, true)) + + for _, candidate := range subtitleLegacyEncodings { + text, err := candidate.enc.NewDecoder().Bytes(raw) + if err != nil { + continue + } + consider(string(text)) + } + if best != "" { + return best + } + return string(raw) +} + +// decodeUTF16 按给定字节序把 UTF-16 字节解码为字符串(含代理对)。 +func decodeUTF16(raw []byte, bigEndian bool) string { + units := make([]uint16, 0, len(raw)/2) + for i := 0; i+1 < len(raw); i += 2 { + if bigEndian { + units = append(units, binary.BigEndian.Uint16(raw[i:i+2])) + } else { + units = append(units, binary.LittleEndian.Uint16(raw[i:i+2])) + } + } + return string(utf16.Decode(units)) +} + +// looksLikeSubtitle 判断解码结果是否真的像一份字幕。解码器「没报错」并不代表 +// 解对了——很多编码能把任意字节解成合法文本,只有格式特征才能确认。 +func looksLikeSubtitle(text string) bool { + head := text + if len(head) > 4096 { + head = head[:4096] + } + if strings.Contains(head, "[Script Info]") || strings.Contains(head, "Dialogue:") { + return true // ASS/SSA + } + if strings.HasPrefix(strings.TrimSpace(head), "WEBVTT") { + return true + } + return subtitleTimecodePattern.MatchString(head) // SRT / WebVTT 时间行 +} + +// subtitlePlausibility 给解码结果打分,返回 -100..100 的可信度:正常文本字符加分, +// 替换字符(解码失败)、控制字符、未分配码位扣分。 +// +// 刻意用比例而不是字符数量:单字节回退会把一个双字节字符拆成两个字符,字符多的 +// 候选看起来「更正常」,实际是解错了——GBK 中文字幕被当成 Shift_JIS 解,就会得到 +// 一串半角片假名,数量比正确结果还多。半角片假名单独扣分正是为了压住这种情况。 +func subtitlePlausibility(text string) int { + total := 0 + weight := 0 + for _, r := range text { + total++ + switch { + case r == utf8.RuneError: + weight -= 8 + case r == '\n' || r == '\r' || r == '\t': + // 正常换行不计分也不扣分。 + case r < 0x20 || r == 0x7F: + weight -= 8 + case unicode.Is(unicode.Co, r) || unicode.Is(unicode.Cn, r): + weight -= 4 + case r >= 0xFF61 && r <= 0xFF9F: + // 半角片假名:GBK 汉字被按 Shift_JIS 解出来的典型产物。 + weight -= 2 + case unicode.In(r, unicode.Han, unicode.Hiragana, unicode.Katakana, unicode.Hangul), + unicode.Is(unicode.Latin, r), unicode.IsDigit(r), unicode.IsPunct(r): + weight++ + case unicode.IsSpace(r): + // 空格(含全角空格)不参与打分。 + default: + weight-- + } + } + if total == 0 { + return 0 + } + return weight * 100 / total +} diff --git a/internal/service/subtitle_encoding_test.go b/internal/service/subtitle_encoding_test.go new file mode 100644 index 0000000..5b04ace --- /dev/null +++ b/internal/service/subtitle_encoding_test.go @@ -0,0 +1,169 @@ +package service + +import ( + "bytes" + "encoding/binary" + "os" + "path/filepath" + "strings" + "testing" + "unicode/utf16" + + "github.com/glebarez/sqlite" + "go.uber.org/zap" + "golang.org/x/text/encoding/simplifiedchinese" + "gorm.io/gorm" + + "github.com/truewhile/MeBox/internal/config" + "github.com/truewhile/MeBox/internal/model" + "github.com/truewhile/MeBox/internal/repository" +) + +const assFixture = "[Script Info]\n" + + "Title:Railgun 01 BD\n" + + "ScriptType:v4.00+\n" + + "\n" + + "[Events]\n" + + "Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text\n" + + "Dialogue: 0,0:00:01.00,0:00:02.00,Default,,0,0,0,,放て!心に刻んだ夢を\n" + +const srtFixture = "1\n" + + "00:00:01,000 --> 00:00:02,000\n" + + "只有我的超电磁炮\n" + + "\n" + +func encodeUTF16(s string, bigEndian bool, withBOM bool) []byte { + units := utf16.Encode([]rune(s)) + out := make([]byte, 0, len(units)*2+2) + if withBOM { + if bigEndian { + out = append(out, 0xFE, 0xFF) + } else { + out = append(out, 0xFF, 0xFE) + } + } + buf := make([]byte, 2) + for _, unit := range units { + if bigEndian { + binary.BigEndian.PutUint16(buf, unit) + } else { + binary.LittleEndian.PutUint16(buf, unit) + } + out = append(out, buf...) + } + return out +} + +func encodeGBK(t *testing.T, s string) []byte { + t.Helper() + encoded, err := simplifiedchinese.GBK.NewEncoder().Bytes([]byte(s)) + if err != nil { + t.Fatalf("encode GBK: %v", err) + } + return encoded +} + +// 这是本次线上问题的核心场景:字幕组给的 ASS 是带 BOM 的 UTF-16LE, +// 之前原样下发,浏览器按 UTF-8 解开后整篇是替换字符,一个字都渲染不出来。 +func TestDecodeSubtitleTextNormalisesLegacyEncodings(t *testing.T) { + cases := []struct { + name string + raw []byte + }{ + {"utf8", []byte(assFixture)}, + {"utf8-bom", append([]byte{0xEF, 0xBB, 0xBF}, []byte(assFixture)...)}, + {"utf16le-bom", encodeUTF16(assFixture, false, true)}, + {"utf16be-bom", encodeUTF16(assFixture, true, true)}, + {"utf16le-no-bom", encodeUTF16(assFixture, false, false)}, + {"gb18030", encodeGBK(t, assFixture)}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := decodeSubtitleText(tc.raw) + if !strings.Contains(got, "[Script Info]") { + t.Fatalf("decoded text lost ASS header: %q", got) + } + if !strings.Contains(got, "Dialogue:") { + t.Fatalf("decoded text lost Dialogue line: %q", got) + } + if !strings.Contains(got, "放て!心に刻んだ夢を") { + t.Fatalf("decoded text lost non-ASCII content: %q", got) + } + if strings.ContainsRune(got, 0) || strings.ContainsRune(got, '\uFFFD') { + t.Fatalf("decoded text still contains NUL or replacement runes: %q", got) + } + }) + } +} + +func TestDecodeSubtitleTextKeepsUnknownBytes(t *testing.T) { + raw := []byte("not a subtitle at all: \x81\x82\x83\x84") + if got := decodeSubtitleText(raw); got != string(raw) { + t.Fatalf("decodeSubtitleText mangled non-subtitle bytes: %q", got) + } +} + +// 接口层回归:外挂 UTF-16 字幕经 /subtitles/:id/ass(libass 路径)与 +// /subtitles/:id(WebVTT 降级路径)下发时都必须是可解析的 UTF-8。 +func TestSubtitleHandlersNormaliseUTF16Sidecar(t *testing.T) { + db, err := gorm.Open(sqlite.Open("file:subtitle-encoding?mode=memory&cache=shared"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + if err := db.AutoMigrate(&model.Library{}, &model.Media{}); err != nil { + t.Fatal(err) + } + + dir := t.TempDir() + videoPath := filepath.Join(dir, "Railgun S01E01.mkv") + assPath := filepath.Join(dir, "Railgun S01E01.ass") + srtPath := filepath.Join(dir, "Railgun S01E01.zh.srt") + if err := os.WriteFile(videoPath, []byte("video"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(assPath, encodeUTF16(assFixture, false, true), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(srtPath, encodeGBK(t, srtFixture), 0o644); err != nil { + t.Fatal(err) + } + + media := model.Media{Title: "Railgun", Path: videoPath} + if err := db.Create(&media).Error; err != nil { + t.Fatal(err) + } + svc := NewSubtitleService(&config.Config{}, zap.NewNop(), repository.New(db)) + + var buf bytes.Buffer + if err := svc.ServeASS(t.Context(), media.ID, assPath, &buf); err != nil { + t.Fatalf("ServeASS: %v", err) + } + ass := buf.String() + if !bytes.HasPrefix(buf.Bytes(), []byte("[Script Info]")) { + t.Fatalf("ServeASS output is not UTF-8 ASS: %q", ass) + } + if !strings.Contains(ass, "放て!心に刻んだ夢を") { + t.Fatalf("ServeASS output lost dialogue text: %q", ass) + } + + buf.Reset() + if err := svc.Serve(t.Context(), media.ID, assPath, &buf); err != nil { + t.Fatalf("Serve(ass): %v", err) + } + vtt := buf.String() + if !strings.HasPrefix(vtt, "WEBVTT") || !strings.Contains(vtt, "-->") { + t.Fatalf("ASS->WebVTT fallback produced no cues: %q", vtt) + } + if !strings.Contains(vtt, "放て!心に刻んだ夢を") { + t.Fatalf("ASS->WebVTT fallback lost dialogue text: %q", vtt) + } + + buf.Reset() + if err := svc.Serve(t.Context(), media.ID, srtPath, &buf); err != nil { + t.Fatalf("Serve(srt): %v", err) + } + vtt = buf.String() + if !strings.Contains(vtt, "只有我的超电磁炮") { + t.Fatalf("GBK SRT was not transcoded: %q", vtt) + } +}