mirror of
https://github.com/truewhile/MeBox.git
synced 2026-09-29 19:36:36 +08:00
23d1fbfddb
Root cause: display-only cloud library filtering was reused by scan jobs, merged cloud mounts could be skipped, and OpenList listing relied on WebDAV/first-page behavior that could cap huge directories around 100 items. Cloud scans also let new items consume probe budget before existing rows with missing track/artwork metadata.\n\nChanges:\n- split display filtering from scannable cloud filtering so merged cloud mounts still scan\n- add OpenList API pagination with WebDAV fallback\n- prioritize existing cloud media missing metadata before new imports\n- restrict automatic cloud sync to one successful 19:00-21:00 daily window and keep manual scan immediate\n- disable startup cloud full-scan by default, with an explicit opt-in setting\n- improve Emby/cloud playback compatibility and cache/search/test coverage from the continued work
508 lines
14 KiB
Go
508 lines
14 KiB
Go
package cloud
|
||
|
||
import (
|
||
"bytes"
|
||
"context"
|
||
"encoding/base64"
|
||
"encoding/json"
|
||
"encoding/xml"
|
||
"fmt"
|
||
"io"
|
||
"net/http"
|
||
"net/url"
|
||
"path"
|
||
"strconv"
|
||
"strings"
|
||
)
|
||
|
||
// cloudDrive2Provider bridges CloudDrive2 through its WebDAV endpoint.
|
||
//
|
||
// CloudDrive2 already integrates many cloud disks (115 / 123 / Aliyun / Quark
|
||
// and more). Treating it as a WebDAV-backed cloud provider lets MediaStationGo
|
||
// browse, mount and upload to those disks without carrying every provider's
|
||
// private chunk-upload protocol in this project.
|
||
type cloudDrive2Provider struct {
|
||
typ string
|
||
name string
|
||
base *url.URL
|
||
username string
|
||
password string
|
||
token string
|
||
ua string
|
||
apiBase *url.URL
|
||
client *http.Client
|
||
proxy bool
|
||
}
|
||
|
||
func newCloudDrive2(cfg map[string]any, client *http.Client) *cloudDrive2Provider {
|
||
return newCloudDAVProvider(TypeCloudDrive2, "clouddrive2", cfg, client, "/dav")
|
||
}
|
||
|
||
func newOpenList(cfg map[string]any, client *http.Client) *cloudDrive2Provider {
|
||
return newCloudDAVProvider(TypeOpenList, "openlist", cfg, client, "/dav")
|
||
}
|
||
|
||
func newCloudDAVProvider(typ, name string, cfg map[string]any, client *http.Client, defaultDAVPath string) *cloudDrive2Provider {
|
||
rawURL := webDAVURLFromConfig(cfg, defaultDAVPath)
|
||
u, _ := url.Parse(strings.TrimRight(rawURL, "/"))
|
||
var apiBase *url.URL
|
||
if typ == TypeOpenList {
|
||
apiBase = openListAPIBaseFromConfig(cfg, rawURL, defaultDAVPath)
|
||
}
|
||
ua := str(cfg["ua"])
|
||
if ua == "" {
|
||
ua = defaultUA
|
||
}
|
||
proxy := true
|
||
if _, ok := cfg["force_302"]; ok && boolish(cfg["force_302"]) {
|
||
proxy = false
|
||
}
|
||
return &cloudDrive2Provider{
|
||
typ: typ,
|
||
name: name,
|
||
base: u,
|
||
username: str(cfg["username"]),
|
||
password: str(cfg["password"]),
|
||
token: str(cfg["token"]),
|
||
ua: ua,
|
||
apiBase: apiBase,
|
||
client: client,
|
||
proxy: proxy,
|
||
}
|
||
}
|
||
|
||
func (p *cloudDrive2Provider) Type() string { return p.typ }
|
||
|
||
func (p *cloudDrive2Provider) Ping(ctx context.Context) error {
|
||
_, err := p.List(ctx, "")
|
||
return err
|
||
}
|
||
|
||
func (p *cloudDrive2Provider) List(ctx context.Context, dir string) ([]FileEntry, error) {
|
||
if err := p.validate(); err != nil {
|
||
return nil, err
|
||
}
|
||
if p.typ == TypeOpenList && p.apiBase != nil && strings.TrimSpace(p.token) != "" {
|
||
if entries, err := p.listOpenListAPI(ctx, dir); err == nil {
|
||
return entries, nil
|
||
}
|
||
}
|
||
target := normalizeCloudDAVPath(dir)
|
||
req, err := http.NewRequestWithContext(ctx, "PROPFIND", p.urlFor(target), strings.NewReader(cloudDAVPropfindBody))
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
p.auth(req)
|
||
req.Header.Set("Depth", "1")
|
||
req.Header.Set("Content-Type", "application/xml; charset=utf-8")
|
||
req.Header.Set("Accept", "application/xml,text/xml,*/*")
|
||
resp, err := p.client.Do(req)
|
||
if err != nil {
|
||
return nil, decorateDAVTransportError(p.name, p.urlFor(target), err)
|
||
}
|
||
defer resp.Body.Close()
|
||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||
return nil, p.decorateDAVStatusError(resp, target)
|
||
}
|
||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
|
||
var multi cloudDAVMultiStatus
|
||
if err := xml.Unmarshal(body, &multi); err != nil {
|
||
return nil, fmt.Errorf("%s: decode webdav: %w", p.name, err)
|
||
}
|
||
basePath := strings.TrimRight(p.base.EscapedPath(), "/")
|
||
currentID := normalizeCloudDAVPath(target)
|
||
out := make([]FileEntry, 0, len(multi.Responses))
|
||
for _, item := range multi.Responses {
|
||
entryPath, err := p.entryIDFromHref(item.Href, basePath)
|
||
if err != nil || entryPath == "" || sameCloudDAVPath(entryPath, currentID) {
|
||
continue
|
||
}
|
||
name := firstNonEmpty(item.PropStat.Prop.DisplayName, path.Base(strings.TrimRight(entryPath, "/")))
|
||
if decoded, err := url.PathUnescape(name); err == nil {
|
||
name = decoded
|
||
}
|
||
if name == "" || name == "." || name == "/" {
|
||
continue
|
||
}
|
||
out = append(out, FileEntry{
|
||
ID: entryPath,
|
||
Name: name,
|
||
IsDir: item.PropStat.Prop.ResourceType.Collection != nil || strings.HasSuffix(item.Href, "/"),
|
||
Size: parseDAVSize(item.PropStat.Prop.ContentLength),
|
||
})
|
||
}
|
||
return out, nil
|
||
}
|
||
|
||
func (p *cloudDrive2Provider) listOpenListAPI(ctx context.Context, dir string) ([]FileEntry, error) {
|
||
const pageSize = 500
|
||
target := normalizeCloudDAVPath(dir)
|
||
out := make([]FileEntry, 0, pageSize)
|
||
for pageNum := 1; ; pageNum++ {
|
||
payload := map[string]any{
|
||
"path": target,
|
||
"password": "",
|
||
"page": pageNum,
|
||
"per_page": pageSize,
|
||
"refresh": false,
|
||
}
|
||
body, _ := json.Marshal(payload)
|
||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, p.openListAPIURL("/api/fs/list"), bytes.NewReader(body))
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
req.Header.Set("Content-Type", "application/json")
|
||
req.Header.Set("Accept", "application/json")
|
||
req.Header.Set("User-Agent", p.ua)
|
||
if p.token != "" {
|
||
req.Header.Set("Authorization", p.token)
|
||
}
|
||
resp, err := p.client.Do(req)
|
||
if err != nil {
|
||
return nil, decorateDAVTransportError(p.name, p.openListAPIURL("/api/fs/list"), err)
|
||
}
|
||
var decoded openListListResponse
|
||
decodeErr := json.NewDecoder(io.LimitReader(resp.Body, 32<<20)).Decode(&decoded)
|
||
resp.Body.Close()
|
||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||
return nil, fmt.Errorf("%s: api list %s returned http %d", p.name, target, resp.StatusCode)
|
||
}
|
||
if decodeErr != nil {
|
||
return nil, fmt.Errorf("%s: decode api list: %w", p.name, decodeErr)
|
||
}
|
||
if decoded.Code != 0 && decoded.Code != 200 {
|
||
msg := strings.TrimSpace(decoded.Message)
|
||
if msg == "" {
|
||
msg = fmt.Sprintf("code %d", decoded.Code)
|
||
}
|
||
return nil, fmt.Errorf("%s: api list %s failed: %s", p.name, target, msg)
|
||
}
|
||
for _, item := range decoded.Data.Content {
|
||
name := strings.TrimSpace(item.Name)
|
||
if name == "" || name == "." || name == "/" {
|
||
continue
|
||
}
|
||
out = append(out, FileEntry{
|
||
ID: joinOpenListAPIPath(target, name),
|
||
Name: name,
|
||
IsDir: item.IsDir,
|
||
Size: item.Size,
|
||
})
|
||
}
|
||
total := decoded.Data.Total
|
||
if total > 0 {
|
||
if len(out) >= total || len(decoded.Data.Content) == 0 {
|
||
break
|
||
}
|
||
continue
|
||
}
|
||
if len(decoded.Data.Content) == 0 || len(decoded.Data.Content) < pageSize {
|
||
break
|
||
}
|
||
}
|
||
return out, nil
|
||
}
|
||
|
||
func (p *cloudDrive2Provider) Resolve(ctx context.Context, fileRef string) (*DirectLink, error) {
|
||
if err := p.validate(); err != nil {
|
||
return nil, err
|
||
}
|
||
ref := normalizeCloudDAVPath(fileRef)
|
||
if ref == "/" {
|
||
return nil, fmt.Errorf("%s: file reference required", p.name)
|
||
}
|
||
headers := map[string]string{
|
||
"User-Agent": p.ua,
|
||
}
|
||
if p.token != "" {
|
||
headers["Authorization"] = p.token
|
||
} else if p.username != "" {
|
||
headers["Authorization"] = "Basic " + base64.StdEncoding.EncodeToString([]byte(p.username+":"+p.password))
|
||
}
|
||
return &DirectLink{URL: p.urlFor(ref), Headers: headers, Proxy: p.proxy}, nil
|
||
}
|
||
|
||
func (p *cloudDrive2Provider) validate() error {
|
||
if p.base == nil || p.base.Scheme == "" || p.base.Host == "" {
|
||
return fmt.Errorf("%s: missing WebDAV URL", p.name)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func (p *cloudDrive2Provider) auth(req *http.Request) {
|
||
req.Header.Set("User-Agent", p.ua)
|
||
if p.token != "" {
|
||
req.Header.Set("Authorization", p.token)
|
||
return
|
||
}
|
||
if p.username != "" {
|
||
req.SetBasicAuth(p.username, p.password)
|
||
}
|
||
}
|
||
|
||
func webDAVURLFromConfig(cfg map[string]any, defaultDAVPath string) string {
|
||
rawURL := str(cfg["url"])
|
||
if rawURL == "" {
|
||
rawURL = str(cfg["webdav_url"])
|
||
}
|
||
if rawURL != "" {
|
||
return ensureDefaultDAVPath(rawURL, defaultDAVPath)
|
||
}
|
||
return defaultWebDAVURL(str(cfg["server"]), defaultDAVPath)
|
||
}
|
||
|
||
func defaultWebDAVURL(server, defaultDAVPath string) string {
|
||
server = strings.TrimRight(strings.TrimSpace(server), "/")
|
||
if server == "" {
|
||
return ""
|
||
}
|
||
davPath := strings.TrimSpace(defaultDAVPath)
|
||
if davPath == "" {
|
||
return server
|
||
}
|
||
if !strings.HasPrefix(davPath, "/") {
|
||
davPath = "/" + davPath
|
||
}
|
||
return server + davPath
|
||
}
|
||
|
||
func openListAPIBaseFromConfig(cfg map[string]any, webDAVURL, defaultDAVPath string) *url.URL {
|
||
raw := str(cfg["server"])
|
||
if raw == "" {
|
||
raw = firstNonEmpty(str(cfg["api_url"]), webDAVURL)
|
||
}
|
||
raw = strings.TrimRight(strings.TrimSpace(raw), "/")
|
||
if raw == "" {
|
||
return nil
|
||
}
|
||
u, err := url.Parse(raw)
|
||
if err != nil || u.Scheme == "" || u.Host == "" {
|
||
return nil
|
||
}
|
||
davPath := strings.Trim(strings.TrimSpace(defaultDAVPath), "/")
|
||
if davPath != "" {
|
||
pathParts := strings.Split(strings.TrimRight(u.Path, "/"), "/")
|
||
if len(pathParts) > 0 && strings.EqualFold(pathParts[len(pathParts)-1], davPath) {
|
||
u.Path = strings.Join(pathParts[:len(pathParts)-1], "/")
|
||
if u.Path == "" {
|
||
u.Path = "/"
|
||
}
|
||
}
|
||
}
|
||
u.RawPath = ""
|
||
u.RawQuery = ""
|
||
u.Fragment = ""
|
||
return u
|
||
}
|
||
|
||
func (p *cloudDrive2Provider) openListAPIURL(apiPath string) string {
|
||
if p.apiBase == nil {
|
||
return ""
|
||
}
|
||
u := *p.apiBase
|
||
u.RawPath = ""
|
||
basePath := strings.TrimRight(u.Path, "/")
|
||
apiPath = "/" + strings.TrimLeft(apiPath, "/")
|
||
if basePath == "" || basePath == "/" {
|
||
u.Path = apiPath
|
||
} else {
|
||
u.Path = basePath + apiPath
|
||
}
|
||
return u.String()
|
||
}
|
||
|
||
func ensureDefaultDAVPath(rawURL, defaultDAVPath string) string {
|
||
rawURL = strings.TrimRight(strings.TrimSpace(rawURL), "/")
|
||
if rawURL == "" {
|
||
return ""
|
||
}
|
||
u, err := url.Parse(rawURL)
|
||
if err != nil || u.Scheme == "" || u.Host == "" {
|
||
return rawURL
|
||
}
|
||
if strings.TrimSpace(defaultDAVPath) == "" {
|
||
return rawURL
|
||
}
|
||
if u.Path == "" || u.Path == "/" {
|
||
davPath := strings.TrimSpace(defaultDAVPath)
|
||
if !strings.HasPrefix(davPath, "/") {
|
||
davPath = "/" + davPath
|
||
}
|
||
u.Path = davPath
|
||
u.RawPath = ""
|
||
return strings.TrimRight(u.String(), "/")
|
||
}
|
||
return rawURL
|
||
}
|
||
|
||
func (p *cloudDrive2Provider) decorateDAVStatusError(resp *http.Response, target string) error {
|
||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
|
||
detail := compactDAVErrorBody(string(body))
|
||
if detail == "" {
|
||
return fmt.Errorf("%s: list %s returned http %d", p.name, target, resp.StatusCode)
|
||
}
|
||
if resp.StatusCode == http.StatusMethodNotAllowed {
|
||
return fmt.Errorf("%s: list %s returned http %d:%s;请确认填写的是 WebDAV 地址(通常以 /dav 结尾),并且桥接网盘已在 OpenList/CloudDrive2 内完成登录或 Cookie 保存", p.name, target, resp.StatusCode, detail)
|
||
}
|
||
if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden {
|
||
return fmt.Errorf("%s: list %s returned http %d:%s;请检查 WebDAV 用户名/密码、Authorization Token,或先在 OpenList/CloudDrive2 中保存对应网盘 Cookie", p.name, target, resp.StatusCode, detail)
|
||
}
|
||
return fmt.Errorf("%s: list %s returned http %d:%s", p.name, target, resp.StatusCode, detail)
|
||
}
|
||
|
||
func compactDAVErrorBody(raw string) string {
|
||
raw = strings.TrimSpace(strings.ReplaceAll(raw, "\x00", ""))
|
||
if raw == "" {
|
||
return ""
|
||
}
|
||
raw = strings.Join(strings.Fields(raw), " ")
|
||
if len([]rune(raw)) > 180 {
|
||
return string([]rune(raw)[:180]) + "…"
|
||
}
|
||
return raw
|
||
}
|
||
|
||
func decorateDAVTransportError(name, target string, err error) error {
|
||
if err == nil {
|
||
return nil
|
||
}
|
||
message := err.Error()
|
||
if strings.Contains(message, "server gave HTTP response to HTTPS client") {
|
||
return fmt.Errorf("%s: %w;当前地址使用 https://,但服务端返回 HTTP。请改用 http:// 地址,例如 OpenList 默认 WebDAV 通常是 http://host:5244/dav/;如果必须使用 https,请在 OpenList 前配置反向代理和证书", name, err)
|
||
}
|
||
if strings.Contains(message, "first record does not look like a TLS handshake") {
|
||
return fmt.Errorf("%s: %w;疑似把 HTTP 服务配置成了 https://,请检查 %s 的协议头", name, err, target)
|
||
}
|
||
return err
|
||
}
|
||
|
||
func (p *cloudDrive2Provider) urlFor(remotePath string) string {
|
||
u := *p.base
|
||
u.RawPath = ""
|
||
basePath := strings.TrimRight(u.Path, "/")
|
||
remote := strings.Trim(normalizeCloudDAVPath(remotePath), "/")
|
||
switch {
|
||
case basePath == "" || basePath == "/":
|
||
if remote == "" {
|
||
u.Path = "/"
|
||
} else {
|
||
u.Path = "/" + remote
|
||
}
|
||
case remote == "":
|
||
u.Path = basePath
|
||
default:
|
||
u.Path = basePath + "/" + remote
|
||
}
|
||
return u.String()
|
||
}
|
||
|
||
func (p *cloudDrive2Provider) entryIDFromHref(href, basePath string) (string, error) {
|
||
if href == "" {
|
||
return "", nil
|
||
}
|
||
parsed, err := url.Parse(href)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
hrefPath := parsed.EscapedPath()
|
||
if hrefPath == "" {
|
||
hrefPath = href
|
||
}
|
||
if basePath != "" && basePath != "/" {
|
||
hrefPath = strings.TrimPrefix(hrefPath, basePath)
|
||
}
|
||
if decoded, err := url.PathUnescape(hrefPath); err == nil {
|
||
hrefPath = decoded
|
||
}
|
||
return normalizeCloudDAVPath(hrefPath), nil
|
||
}
|
||
|
||
const cloudDAVPropfindBody = `<?xml version="1.0" encoding="utf-8"?>
|
||
<d:propfind xmlns:d="DAV:">
|
||
<d:prop>
|
||
<d:displayname/>
|
||
<d:getcontentlength/>
|
||
<d:resourcetype/>
|
||
</d:prop>
|
||
</d:propfind>`
|
||
|
||
type cloudDAVMultiStatus struct {
|
||
Responses []cloudDAVResponse `xml:"response"`
|
||
}
|
||
|
||
type cloudDAVResponse struct {
|
||
Href string `xml:"href"`
|
||
PropStat cloudDAVPropStat `xml:"propstat"`
|
||
}
|
||
|
||
type cloudDAVPropStat struct {
|
||
Prop cloudDAVProp `xml:"prop"`
|
||
}
|
||
|
||
type cloudDAVProp struct {
|
||
DisplayName string `xml:"displayname"`
|
||
ContentLength string `xml:"getcontentlength"`
|
||
ResourceType cloudDAVResourceType `xml:"resourcetype"`
|
||
}
|
||
|
||
type cloudDAVResourceType struct {
|
||
Collection *struct{} `xml:"collection"`
|
||
}
|
||
|
||
type openListListResponse struct {
|
||
Code int `json:"code"`
|
||
Message string `json:"message"`
|
||
Data struct {
|
||
Content []openListListItem `json:"content"`
|
||
Total int `json:"total"`
|
||
} `json:"data"`
|
||
}
|
||
|
||
type openListListItem struct {
|
||
Name string `json:"name"`
|
||
Size int64 `json:"size"`
|
||
IsDir bool `json:"is_dir"`
|
||
}
|
||
|
||
func normalizeCloudDAVPath(p string) string {
|
||
p = strings.ReplaceAll(strings.TrimSpace(p), "\\", "/")
|
||
if p == "" || p == "." {
|
||
return "/"
|
||
}
|
||
if !strings.HasPrefix(p, "/") {
|
||
p = "/" + p
|
||
}
|
||
cleaned := path.Clean(p)
|
||
if cleaned == "." {
|
||
return "/"
|
||
}
|
||
return cleaned
|
||
}
|
||
|
||
func sameCloudDAVPath(a, b string) bool {
|
||
return strings.TrimRight(normalizeCloudDAVPath(a), "/") == strings.TrimRight(normalizeCloudDAVPath(b), "/")
|
||
}
|
||
|
||
func joinOpenListAPIPath(dir, name string) string {
|
||
dir = strings.TrimRight(normalizeCloudDAVPath(dir), "/")
|
||
name = strings.Trim(strings.ReplaceAll(name, "\\", "/"), "/")
|
||
if dir == "" || dir == "/" {
|
||
return normalizeCloudDAVPath(name)
|
||
}
|
||
return normalizeCloudDAVPath(dir + "/" + name)
|
||
}
|
||
|
||
func parseDAVSize(raw string) int64 {
|
||
n, _ := strconv.ParseInt(strings.TrimSpace(raw), 10, 64)
|
||
return n
|
||
}
|
||
|
||
func firstNonEmpty(values ...string) string {
|
||
for _, v := range values {
|
||
if strings.TrimSpace(v) != "" {
|
||
return strings.TrimSpace(v)
|
||
}
|
||
}
|
||
return ""
|
||
}
|