refactor: split modules and harden scraping workflows

This commit is contained in:
ShukeBta
2026-06-24 11:59:18 +08:00
parent efca3cbe69
commit 192f35d9fa
470 changed files with 47957 additions and 33839 deletions
+143 -306
View File
@@ -2,115 +2,15 @@ package cloud
import (
"context"
"encoding/base64"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"strconv"
"strings"
"testing"
"time"
)
func TestQuarkListAndResolve(t *testing.T) {
var gotCookie string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotCookie = r.Header.Get("Cookie")
switch {
case r.URL.Path == "/file/sort":
if r.URL.Query().Get("pdir_fid") != "0" {
t.Errorf("unexpected pdir_fid %q", r.URL.Query().Get("pdir_fid"))
}
w.Write([]byte(`{"status":200,"code":0,"data":{"list":[
{"fid":"d1","file_name":"Movies","dir":true,"size":0},
{"fid":"f1","file_name":"Inception.mkv","dir":false,"size":123}]}}`))
case r.URL.Path == "/file/download":
if r.Method != http.MethodPost {
t.Errorf("download must be POST, got %s", r.Method)
}
w.Write([]byte(`{"status":200,"code":0,"data":[{"fid":"f1","download_url":"https://cdn.quark/x.mkv?sign=1"}]}`))
default:
t.Errorf("unexpected path %s", r.URL.Path)
}
}))
defer srv.Close()
p, err := New(TypeQuark, map[string]any{"cookie": "kps=abc", "base": srv.URL}, srv.Client())
if err != nil {
t.Fatal(err)
}
entries, err := p.List(context.Background(), "0")
if err != nil {
t.Fatalf("list: %v", err)
}
if len(entries) != 2 || !entries[0].IsDir || entries[1].Name != "Inception.mkv" || entries[1].Size != 123 {
t.Fatalf("unexpected entries: %#v", entries)
}
if gotCookie != "kps=abc" {
t.Fatalf("cookie not forwarded: %q", gotCookie)
}
link, err := p.Resolve(context.Background(), "f1")
if err != nil {
t.Fatalf("resolve: %v", err)
}
if link.URL != "https://cdn.quark/x.mkv?sign=1" {
t.Fatalf("bad url: %s", link.URL)
}
if !link.Proxy {
t.Fatalf("quark should default to proxy mode")
}
if link.Headers["Cookie"] != "kps=abc" {
t.Fatalf("resolve must carry cookie header: %#v", link.Headers)
}
}
func TestQuarkListPaginates(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/file/sort" {
t.Fatalf("unexpected path %s", r.URL.Path)
}
page, _ := strconv.Atoi(r.URL.Query().Get("_page"))
w.Write([]byte(`{"status":200,"code":0,"data":{"list":[` + quarkPagePayload(page) + `]}}`))
}))
defer srv.Close()
p, err := New(TypeQuark, map[string]any{"cookie": "kps=abc", "base": srv.URL}, srv.Client())
if err != nil {
t.Fatal(err)
}
entries, err := p.List(context.Background(), "0")
if err != nil {
t.Fatalf("list: %v", err)
}
if len(entries) != 101 {
t.Fatalf("entries = %d, want 101", len(entries))
}
if entries[100].ID != "f100" || entries[100].Name != "Movie.100.mkv" {
t.Fatalf("last entry wrong: %#v", entries[100])
}
}
func quarkPagePayload(page int) string {
count := 100
offset := 0
if page > 1 {
count = 1
offset = 100
}
items := make([]string, 0, count)
for i := 0; i < count; i++ {
n := offset + i
items = append(items, fmt.Sprintf(`{"fid":"f%d","file_name":"Movie.%03d.mkv","dir":false,"size":%d}`, n, n, n))
}
return strings.Join(items, ",")
}
func TestDeprecatedProviderPlaybackOverrideKeysAreIgnored(t *testing.T) {
quark := newQuark(map[string]any{"cookie": "c", "force_302": "true"}, http.DefaultClient)
if !quark.proxy {
t.Fatalf("quark should keep safe proxy mode; force_302 is deprecated")
}
pan115 := new115(map[string]any{"cookie": "UID=1; CID=2", "force_proxy": "true"}, http.DefaultClient)
if pan115.proxy {
t.Fatalf("115 should keep safe direct mode; force_proxy is deprecated")
@@ -121,197 +21,6 @@ func TestDeprecatedProviderPlaybackOverrideKeysAreIgnored(t *testing.T) {
}
}
func Test115ListAndResolve(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/files":
if r.URL.Query().Get("cid") != "0" {
t.Errorf("bad cid %q", r.URL.Query().Get("cid"))
}
w.Write([]byte(`{"state":true,"data":[
{"cid":"100","n":"Movies","s":0},
{"fid":"200","n":"Inception.mkv","s":456,"pc":"pick200"}]}`))
default:
t.Errorf("unexpected path %s", r.URL.Path)
}
}))
defer srv.Close()
p, err := New(Type115, map[string]any{"cookie": "UID=1; CID=2", "base": srv.URL}, srv.Client())
if err != nil {
t.Fatal(err)
}
// The downurl endpoint is m115-encrypted end-to-end (the server side
// requires 115's private key), so stub the decrypted payload via the seam
// and assert the pickcode→URL extraction. The live crypto/transport path is
// exercised by integration testing against the real 115 API.
p115, ok := p.(*pan115Provider)
if !ok {
t.Fatalf("expected *pan115Provider, got %T", p)
}
p115.downURLPayload = func(ctx context.Context, pickcode string) ([]byte, error) {
if pickcode != "pick200" {
t.Errorf("bad pickcode %q", pickcode)
}
return []byte(`{"200":{"file_name":"Inception.mkv","file_size":"456","url":{"url":"https://cdn.115/x.mkv?t=1"}}}`), nil
}
entries, err := p.List(context.Background(), "")
if err != nil {
t.Fatalf("list: %v", err)
}
if len(entries) != 2 {
t.Fatalf("want 2 entries: %#v", entries)
}
if !entries[0].IsDir || entries[0].ID != "100" {
t.Fatalf("dir entry wrong: %#v", entries[0])
}
if entries[1].IsDir || entries[1].PickCode != "pick200" || entries[1].Size != 456 {
t.Fatalf("file entry wrong: %#v", entries[1])
}
link, err := p.Resolve(context.Background(), "pick200")
if err != nil {
t.Fatalf("resolve: %v", err)
}
if link.URL != "https://cdn.115/x.mkv?t=1" {
t.Fatalf("bad url: %s", link.URL)
}
if link.Proxy {
t.Fatalf("115 should default to 302 (no proxy)")
}
}
func Test115ListPaginates(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/files" {
t.Fatalf("unexpected path %s", r.URL.Path)
}
offset, _ := strconv.Atoi(r.URL.Query().Get("offset"))
count := 100
if offset > 0 {
count = 1
}
items := make([]string, 0, count)
for i := 0; i < count; i++ {
n := offset + i
items = append(items, fmt.Sprintf(`{"fid":"%d","n":"Movie.%03d.mkv","s":%d,"pc":"pick%d"}`, n, n, n, n))
}
w.Write([]byte(`{"state":true,"data":[` + strings.Join(items, ",") + `]}`))
}))
defer srv.Close()
p, err := New(Type115, map[string]any{"cookie": "UID=1; CID=2", "base": srv.URL}, srv.Client())
if err != nil {
t.Fatal(err)
}
entries, err := p.List(context.Background(), "0")
if err != nil {
t.Fatalf("list: %v", err)
}
if len(entries) != 101 {
t.Fatalf("entries = %d, want 101", len(entries))
}
if entries[100].ID != "100" || entries[100].PickCode != "pick100" {
t.Fatalf("last entry wrong: %#v", entries[100])
}
}
// Test115DownURLEndpointAndError exercises the live fetchDownURLPayload path:
// it must POST an m115-encrypted `data` body to /app/chrome/downurl?t=... and
// surface 115's error when state=false (no decryption needed for that branch).
func Test115DownURLEndpointAndError(t *testing.T) {
var gotData, gotT string
pro := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/app/chrome/downurl" {
t.Errorf("unexpected path %s", r.URL.Path)
}
gotT = r.URL.Query().Get("t")
_ = r.ParseForm()
gotData = r.PostFormValue("data")
w.Write([]byte(`{"state":false,"error":"not exist"}`))
}))
defer pro.Close()
p, err := New(Type115, map[string]any{"cookie": "UID=1", "pro_base": pro.URL}, pro.Client())
if err != nil {
t.Fatal(err)
}
_, err = p.Resolve(context.Background(), "pickX")
if err == nil || !strings.Contains(err.Error(), "not exist") {
t.Fatalf("want upstream error surfaced, got %v", err)
}
if gotT == "" {
t.Errorf("missing t query param")
}
if gotData == "" {
t.Errorf("missing encrypted data body")
}
if _, derr := base64.StdEncoding.DecodeString(gotData); derr != nil {
t.Errorf("data body is not base64: %v", derr)
}
}
func Test115QRFlow(t *testing.T) {
// status sequence: waiting → scanned → confirmed
calls := 0
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/api/1.0/web/1.0/token/":
w.Write([]byte(`{"state":1,"data":{"uid":"U1","time":1700,"sign":"S1"}}`))
case "/get/status/":
if r.URL.Query().Get("uid") != "U1" {
t.Errorf("bad uid %q", r.URL.Query().Get("uid"))
}
calls++
switch calls {
case 1:
w.Write([]byte(`{"state":1,"data":{"status":0}}`))
case 2:
w.Write([]byte(`{"state":1,"data":{"status":1}}`))
default:
w.Write([]byte(`{"state":1,"data":{"status":2}}`))
}
default:
t.Errorf("unexpected api path %s", r.URL.Path)
}
}))
defer api.Close()
passport := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/app/1.0/web/1.0/login/qrcode/" {
t.Errorf("unexpected passport path %s", r.URL.Path)
}
w.Write([]byte(`{"state":1,"data":{"cookie":{"UID":"u","CID":"c","SEID":"s"}}}`))
}))
defer passport.Close()
oldA, oldP := qr115APIBase, qr115PassportBase
qr115APIBase, qr115PassportBase = api.URL, passport.URL
defer func() { qr115APIBase, qr115PassportBase = oldA, oldP }()
ctx := context.Background()
sess, err := QRStart(ctx, api.Client())
if err != nil {
t.Fatalf("qr start: %v", err)
}
if sess.UID != "U1" || sess.QRImageURL == "" {
t.Fatalf("bad session: %#v", sess)
}
want := []string{"waiting", "scanned", "confirmed"}
for i, exp := range want {
st, err := QRPoll(ctx, api.Client(), sess)
if err != nil {
t.Fatalf("poll %d: %v", i, err)
}
if st.State != exp {
t.Fatalf("poll %d: want %s got %s", i, exp, st.State)
}
if exp == "confirmed" {
if st.Cookie == "" || !containsAll(st.Cookie, "UID=u", "SEID=s") {
t.Fatalf("confirmed must yield cookie: %q", st.Cookie)
}
}
}
}
func TestCloudDrive2WebDAVListAndResolve(t *testing.T) {
var gotAuth, gotDepth, gotRange string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -523,6 +232,144 @@ func TestOpenListListUsesAPIUsernamePasswordWithoutWebDAVFallback(t *testing.T)
}
}
func TestOpenListMutableProviderUsesAPI(t *testing.T) {
var mkdirPath, renamePath, renameName, moveSrcDir, moveDstDir string
var moveNames []string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch r.URL.Path {
case "/api/fs/mkdir":
var body map[string]string
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
t.Fatalf("decode mkdir body: %v", err)
}
mkdirPath = body["path"]
if r.Header.Get("Authorization") != "alist-token" {
t.Fatalf("mkdir Authorization = %q", r.Header.Get("Authorization"))
}
_, _ = w.Write([]byte(`{"code":200,"message":"success"}`))
case "/api/fs/rename":
var body map[string]string
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
t.Fatalf("decode rename body: %v", err)
}
renamePath = body["path"]
renameName = body["name"]
_, _ = w.Write([]byte(`{"code":200,"message":"success"}`))
case "/api/fs/move":
var body struct {
SrcDir string `json:"src_dir"`
DstDir string `json:"dst_dir"`
Names []string `json:"names"`
}
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
t.Fatalf("decode move body: %v", err)
}
moveSrcDir = body.SrcDir
moveDstDir = body.DstDir
moveNames = body.Names
_, _ = w.Write([]byte(`{"code":200,"message":"success"}`))
default:
t.Fatalf("unexpected path %s", r.URL.Path)
}
}))
defer srv.Close()
p, err := New(TypeOpenList, map[string]any{"server": srv.URL, "token": "alist-token"}, srv.Client())
if err != nil {
t.Fatal(err)
}
mutable, ok := p.(MutableProvider)
if !ok {
t.Fatal("openlist should support mutable provider")
}
created, err := mutable.Mkdir(context.Background(), "/电视剧", "欧美剧")
if err != nil {
t.Fatalf("mkdir: %v", err)
}
if mkdirPath != "/电视剧/欧美剧" || created.ID != "/电视剧/欧美剧" || !created.IsDir {
t.Fatalf("mkdir path=%q entry=%#v", mkdirPath, created)
}
renamed, err := mutable.Rename(context.Background(), "/电视剧/欧美剧", "美剧")
if err != nil {
t.Fatalf("rename: %v", err)
}
if renamePath != "/电视剧/欧美剧" || renameName != "美剧" || renamed.ID != "/电视剧/美剧" {
t.Fatalf("rename path=%q name=%q entry=%#v", renamePath, renameName, renamed)
}
moved, err := mutable.(MovableProvider).Move(context.Background(), "/待整理/Show.S01E01.mkv", "/动漫/国漫/Show/Season 01", "Show - S01E01.mkv")
if err != nil {
t.Fatalf("move: %v", err)
}
if moveSrcDir != "/待整理" || moveDstDir != "/动漫/国漫/Show/Season 01" || len(moveNames) != 1 || moveNames[0] != "Show.S01E01.mkv" {
t.Fatalf("move src=%q dst=%q names=%#v", moveSrcDir, moveDstDir, moveNames)
}
if renamePath != "/动漫/国漫/Show/Season 01/Show.S01E01.mkv" || renameName != "Show - S01E01.mkv" {
t.Fatalf("post-move rename path=%q name=%q", renamePath, renameName)
}
if moved.ID != "/动漫/国漫/Show/Season 01/Show - S01E01.mkv" {
t.Fatalf("moved entry = %#v", moved)
}
}
func TestCloudDrive2MutableProviderUsesWebDAV(t *testing.T) {
var mkcolSeen bool
var destinations []string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case r.Method == "MKCOL" && r.URL.Path == "/dav/TV":
mkcolSeen = true
w.WriteHeader(http.StatusCreated)
case r.Method == "MOVE" && r.URL.Path == "/dav/TV":
destinations = append(destinations, r.Header.Get("Destination"))
if r.Header.Get("Overwrite") != "F" {
t.Fatalf("Overwrite = %q, want F", r.Header.Get("Overwrite"))
}
w.WriteHeader(http.StatusCreated)
case r.Method == "MOVE" && r.URL.Path == "/dav/Inbox/Movie.mkv":
destinations = append(destinations, r.Header.Get("Destination"))
if r.Header.Get("Overwrite") != "F" {
t.Fatalf("Overwrite = %q, want F", r.Header.Get("Overwrite"))
}
w.WriteHeader(http.StatusCreated)
default:
t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path)
}
}))
defer srv.Close()
p, err := New(TypeCloudDrive2, map[string]any{"url": srv.URL + "/dav", "username": "u", "password": "p"}, srv.Client())
if err != nil {
t.Fatal(err)
}
mutable, ok := p.(MutableProvider)
if !ok {
t.Fatal("clouddrive2 should support mutable provider")
}
if _, err := mutable.Mkdir(context.Background(), "", "TV"); err != nil {
t.Fatalf("mkdir: %v", err)
}
if _, err := mutable.Rename(context.Background(), "/TV", "电视剧"); err != nil {
t.Fatalf("rename: %v", err)
}
moved, err := mutable.(MovableProvider).Move(context.Background(), "/Inbox/Movie.mkv", "/电影/外语电影/Movie (2026)", "Movie (2026).mkv")
if err != nil {
t.Fatalf("move: %v", err)
}
if !mkcolSeen || len(destinations) != 2 {
t.Fatalf("mkcol=%v destinations=%#v, want mkdir and two MOVE calls", mkcolSeen, destinations)
}
if destinations[0] != srv.URL+"/dav/%E7%94%B5%E8%A7%86%E5%89%A7" {
t.Fatalf("rename Destination = %q", destinations[0])
}
if destinations[1] != srv.URL+"/dav/%E7%94%B5%E5%BD%B1/%E5%A4%96%E8%AF%AD%E7%94%B5%E5%BD%B1/Movie%20%282026%29/Movie%20%282026%29.mkv" {
t.Fatalf("move Destination = %q", destinations[1])
}
if moved.ID != "/电影/外语电影/Movie (2026)/Movie (2026).mkv" {
t.Fatalf("moved entry = %#v", moved)
}
}
func TestOpenListListAPIFailureDoesNotFallbackToWebDAV(t *testing.T) {
var davSeen bool
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -804,22 +651,12 @@ func TestUnsupportedProvider(t *testing.T) {
if _, err := New("dropbox", nil, nil); err != ErrUnsupported {
t.Fatalf("want ErrUnsupported, got %v", err)
}
}
func containsAll(s string, subs ...string) bool {
for _, sub := range subs {
found := false
for i := 0; i+len(sub) <= len(s); i++ {
if s[i:i+len(sub)] == sub {
found = true
break
}
}
if !found {
return false
}
if _, err := New("quark", nil, nil); err != ErrUnsupported {
t.Fatalf("quark should be unsupported, got %v", err)
}
if IsCloudType("quark") {
t.Fatal("quark should not be an active cloud provider")
}
return true
}
var _ = time.Second