deploy: sync server CloudSearch 0.5.6
This commit is contained in:
@@ -2,10 +2,13 @@ FROM python:3.12-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
ENV PYTHONPATH=/app
|
||||
|
||||
COPY cloudsearch_transfer/requirements.txt ./requirements.txt
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY . .
|
||||
COPY cloudsearch_transfer ./cloudsearch_transfer
|
||||
RUN mkdir -p /data
|
||||
|
||||
ENV PORT=9528
|
||||
ENV TRANSFER_CONFIG_PATH=/data/transfer_config.json
|
||||
@@ -13,6 +16,6 @@ ENV TRANSFER_CONFIG_PATH=/data/transfer_config.json
|
||||
EXPOSE 9528
|
||||
|
||||
HEALTHCHECK --interval=30s --timeout=5s --retries=3 \
|
||||
CMD ["python", "server.py"]
|
||||
CMD python -c "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:%s/health' % os.getenv('PORT', '9528'), timeout=3)"
|
||||
|
||||
CMD ["python", "server.py"]
|
||||
CMD ["python", "-m", "cloudsearch_transfer.server"]
|
||||
|
||||
@@ -15,8 +15,8 @@ import logging
|
||||
from typing import List, Dict, Tuple, Optional
|
||||
|
||||
from ..base import BaseCloudDriveAdapter, FileInfo, match_url
|
||||
from ..config import PlatformConfig, TransferConfig
|
||||
from ..errors import TransferError, TransferErrorCode
|
||||
from ...config import PlatformConfig, TransferConfig
|
||||
from ...errors import TransferError, TransferErrorCode
|
||||
|
||||
from .credential import AliyunCredentialManager
|
||||
from .transfer import AliyunTransfer
|
||||
@@ -47,23 +47,29 @@ class AliyunAdapter(BaseCloudDriveAdapter):
|
||||
"Referer": "https://aliyundrive.com",
|
||||
}
|
||||
|
||||
def __init__(self, config: PlatformConfig, transfer_config: TransferConfig):
|
||||
super().__init__(config, transfer_config)
|
||||
capabilities: Dict[str, bool] = {
|
||||
**BaseCloudDriveAdapter.capabilities,
|
||||
"ensure_dir": True,
|
||||
"save_files": True,
|
||||
"rename": True,
|
||||
"move_files": True,
|
||||
"delete_files": True,
|
||||
}
|
||||
|
||||
# 创建凭证管理器（AliyunCredentialManager）
|
||||
def __init__(self, config: PlatformConfig, transfer_config: TransferConfig):
|
||||
# BaseCloudDriveAdapter.__init__ calls _setup_session(), so credential
|
||||
# state must exist before super().__init__.
|
||||
refresh_token = config.refresh_token or config.cookie or ""
|
||||
self._credential = AliyunCredentialManager(refresh_token=refresh_token)
|
||||
|
||||
# 初始化 drive_id
|
||||
self._drive_id = ""
|
||||
|
||||
# 创建子模块
|
||||
self._transfer: Optional[AliyunTransfer] = None
|
||||
self._cleanup: Optional[AliyunCleanup] = None
|
||||
super().__init__(config, transfer_config)
|
||||
|
||||
def _setup_session(self):
|
||||
"""初始化 session 和凭证"""
|
||||
if self._credential.refresh_token:
|
||||
refresh_token = getattr(self._credential, "refresh_token", "")
|
||||
if refresh_token:
|
||||
# 验证 refresh_token 并获取 drive_id
|
||||
if self._credential.validate():
|
||||
self._drive_id = self._credential.get_drive_id()
|
||||
@@ -164,8 +170,10 @@ class AliyunAdapter(BaseCloudDriveAdapter):
|
||||
platform=self.PLATFORM_KEY,
|
||||
)
|
||||
|
||||
# 确定目标目录
|
||||
# 确定目标目录：路径先解析/创建为 file_id。
|
||||
to_parent = save_dir if save_dir and save_dir != "/" else "root"
|
||||
if isinstance(to_parent, str) and to_parent.startswith("/"):
|
||||
to_parent = self.ensure_dir(to_parent)
|
||||
|
||||
transfer = self._get_transfer()
|
||||
new_ids = transfer._batch_copy(pwd_id, share_token, file_ids, to_parent)
|
||||
@@ -237,6 +245,74 @@ class AliyunAdapter(BaseCloudDriveAdapter):
|
||||
|
||||
# ─── 扩展功能 ──────────────────────────────────────────
|
||||
|
||||
|
||||
# ─── Optional Drive API capability methods ─────────────────────
|
||||
|
||||
def get_fids(self, file_paths: List[str]) -> List[Dict]:
|
||||
wanted = {p.rstrip("/") or "/" for p in file_paths}
|
||||
found: List[Dict] = []
|
||||
for path in wanted:
|
||||
if path == "/":
|
||||
found.append({"file_path": "/", "fid": "root"})
|
||||
continue
|
||||
parent = path.rsplit("/", 1)[0] or "/"
|
||||
name = path.rsplit("/", 1)[-1]
|
||||
for item in self.get_files(parent):
|
||||
if item.is_dir and item.name == name:
|
||||
found.append({"file_path": path, "fid": item.fid})
|
||||
break
|
||||
return found
|
||||
|
||||
def ensure_dir(self, dir_path: str) -> str:
|
||||
normalized = "/" + (dir_path or "/").strip("/")
|
||||
if normalized == "/":
|
||||
return "root"
|
||||
current = ""
|
||||
last_fid = "root"
|
||||
for part in [p for p in normalized.split("/") if p]:
|
||||
current = f"{current}/{part}" if current else f"/{part}"
|
||||
matches = self.get_fids([current])
|
||||
if matches:
|
||||
last_fid = matches[0].get("fid") or matches[0].get("file_id") or last_fid
|
||||
continue
|
||||
created = self.mkdir(current)
|
||||
data = created.get("data", created) if isinstance(created, dict) else {}
|
||||
last_fid = data.get("fid") or data.get("file_id") or data.get("id") or last_fid
|
||||
return last_fid
|
||||
|
||||
def mkdir(self, dir_path: str) -> Dict:
|
||||
parent = dir_path.rsplit("/", 1)[0] or "/"
|
||||
name = dir_path.rstrip("/").rsplit("/", 1)[-1]
|
||||
parent_fid = "root" if parent == "/" else self.ensure_dir(parent)
|
||||
url = "https://api.aliyundrive.com/adrive/v2/file/createWithFolders"
|
||||
body = {
|
||||
"drive_id": self._drive_id or self._credential.get_drive_id(),
|
||||
"parent_file_id": parent_fid,
|
||||
"name": name,
|
||||
"type": "folder",
|
||||
"check_name_mode": "refuse",
|
||||
}
|
||||
resp = self._post(url, json_data=body, headers=self._credential.get_headers())
|
||||
data = self._drive_api_json(resp, context="阿里云盘创建目录")
|
||||
return {"code": 0, "status": 200, "data": {"fid": data.get("file_id", ""), **data}}
|
||||
|
||||
def rename(self, fid: str, file_name: str) -> Dict:
|
||||
url = "https://api.aliyundrive.com/v3/file/update"
|
||||
body = {"drive_id": self._drive_id or self._credential.get_drive_id(), "file_id": fid, "name": file_name, "check_name_mode": "refuse"}
|
||||
data = self._drive_api_json(self._post(url, json_data=body, headers=self._credential.get_headers()), context="阿里云盘重命名")
|
||||
return {"code": 0, "status": 200, "data": data}
|
||||
|
||||
def move_files(self, fids: List[str], to_pdir_fid: str) -> Dict:
|
||||
if isinstance(to_pdir_fid, str) and to_pdir_fid.startswith("/"):
|
||||
to_pdir_fid = self.ensure_dir(to_pdir_fid)
|
||||
drive_id = self._drive_id or self._credential.get_drive_id()
|
||||
requests = [{"id": fid, "method": "POST", "url": "/file/move", "headers": {"Content-Type": "application/json"}, "body": {"drive_id": drive_id, "file_id": fid, "to_parent_file_id": to_pdir_fid}} for fid in fids]
|
||||
data = self._drive_api_json(self._post("https://api.aliyundrive.com/adrive/v4/batch", json_data={"requests": requests, "resource": "file"}, headers=self._credential.get_headers()), context="阿里云盘移动文件")
|
||||
return {"code": 0, "status": 200, "data": data}
|
||||
|
||||
def delete_files(self, fids: List[str]) -> Dict:
|
||||
return {"code": 0, "status": 200} if self.delete(fids) else {"code": -1, "status": 500}
|
||||
|
||||
def cleanup_files(self, file_ids: List[str]) -> Dict:
|
||||
"""
|
||||
清理文件（移入回收站），返回详细结果。
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import List, Tuple
|
||||
from typing import List, Tuple, Dict
|
||||
|
||||
from ..base import BaseCloudDriveAdapter, FileInfo
|
||||
from ...config import PlatformConfig, TransferConfig
|
||||
@@ -36,6 +36,15 @@ class BaiduAdapter(BaseCloudDriveAdapter):
|
||||
r'pan\.baidu\.com/s/1([A-Za-z0-9_-]+)',
|
||||
]
|
||||
|
||||
capabilities: Dict[str, bool] = {
|
||||
**BaseCloudDriveAdapter.capabilities,
|
||||
"ensure_dir": True,
|
||||
"save_files": True,
|
||||
"rename": True,
|
||||
"move_files": True,
|
||||
"delete_files": True,
|
||||
}
|
||||
|
||||
def __init__(self, config: PlatformConfig, transfer_config: TransferConfig):
|
||||
super().__init__(config, transfer_config)
|
||||
|
||||
@@ -123,11 +132,13 @@ class BaiduAdapter(BaseCloudDriveAdapter):
|
||||
fs_ids = detail["fs_ids"]
|
||||
filenames = detail.get("filenames", [])
|
||||
|
||||
target_dir = self.ensure_dir(save_dir) if save_dir and save_dir.startswith("/") else (save_dir or "/")
|
||||
|
||||
# ③ 转存
|
||||
self._transfer._transfer_files(shareid, uk, fs_ids, save_dir, bdstoken)
|
||||
self._transfer._transfer_files(shareid, uk, fs_ids, target_dir, bdstoken)
|
||||
|
||||
# ④ 列出目录匹配新 fs_id
|
||||
new_fs_ids = self._transfer._list_and_match(save_dir, filenames, bdstoken)
|
||||
new_fs_ids = self._transfer._list_and_match(target_dir, filenames, bdstoken)
|
||||
|
||||
# 暂存文件信息供 _filter_ads + _create_share 使用
|
||||
self._last_transfer_files = [
|
||||
@@ -248,6 +259,71 @@ class BaiduAdapter(BaseCloudDriveAdapter):
|
||||
|
||||
# ─── 扩展方法 ────────────────────────────────────────────
|
||||
|
||||
|
||||
# ─── Optional Drive API capability methods ─────────────────────
|
||||
|
||||
def ensure_dir(self, dir_path: str) -> str:
|
||||
normalized = "/" + (dir_path or "/").strip("/")
|
||||
if normalized == "/":
|
||||
return "/"
|
||||
current = ""
|
||||
for part in [p for p in normalized.split("/") if p]:
|
||||
parent = current or "/"
|
||||
current = f"{current}/{part}" if current else f"/{part}"
|
||||
exists = any(item.is_dir and item.name == part for item in self.get_files(parent))
|
||||
if not exists:
|
||||
self.mkdir(current)
|
||||
return normalized
|
||||
|
||||
def get_fids(self, file_paths: List[str]) -> List[Dict]:
|
||||
"""Resolve existing Baidu paths by listing their parent directories."""
|
||||
results: List[Dict] = []
|
||||
for path in file_paths:
|
||||
normalized = "/" + (path or "").strip("/")
|
||||
if normalized == "/":
|
||||
results.append({"file_path": path, "fid": "/", "path": "/"})
|
||||
continue
|
||||
parent, name = normalized.rsplit("/", 1)
|
||||
parent = parent or "/"
|
||||
match = next((item for item in self.get_files(parent) if item.name == name), None)
|
||||
if match:
|
||||
results.append({"file_path": path, "fid": match.fid, "path": normalized})
|
||||
return results
|
||||
|
||||
def mkdir(self, dir_path: str) -> Dict:
|
||||
bdstoken = self.credential.get_bdstoken()
|
||||
url = "https://pan.baidu.com/api/create"
|
||||
params = {"a": "commit", "bdstoken": bdstoken}
|
||||
data = {"path": dir_path, "isdir": 1, "block_list": "[]"}
|
||||
resp = self._post(url, data=data, params=params, headers=self.credential.get_headers())
|
||||
payload = self._drive_api_json(resp, context="百度网盘创建目录")
|
||||
errno = payload.get("errno", 0)
|
||||
if errno not in (0, -8):
|
||||
raise TransferError(TransferErrorCode.NETWORK_ERROR, message=f"百度创建目录失败 errno={errno}", platform=self.PLATFORM_KEY, details=payload)
|
||||
return {"code": 0, "status": 200, "data": {"path": dir_path, **payload}}
|
||||
|
||||
def rename(self, fid: str, file_name: str) -> Dict:
|
||||
return self._filemanager("rename", [{"path": fid, "newname": file_name}])
|
||||
|
||||
def move_files(self, fids: List[str], to_pdir_fid: str) -> Dict:
|
||||
target = self.ensure_dir(to_pdir_fid) if to_pdir_fid.startswith("/") else to_pdir_fid
|
||||
return self._filemanager("move", [{"path": fid, "dest": target} for fid in fids])
|
||||
|
||||
def delete_files(self, fids: List[str]) -> Dict:
|
||||
return {"code": 0, "status": 200} if self.delete(fids) else {"code": -1, "status": 500}
|
||||
|
||||
def _filemanager(self, opera: str, filelist: List[Dict]) -> Dict:
|
||||
import json
|
||||
bdstoken = self.credential.get_bdstoken()
|
||||
url = "https://pan.baidu.com/api/filemanager"
|
||||
params = {"opera": opera, "bdstoken": bdstoken}
|
||||
data = {"filelist": json.dumps(filelist, ensure_ascii=False)}
|
||||
payload = self._drive_api_json(self._post(url, data=data, params=params, headers=self.credential.get_headers()), context=f"百度网盘{opera}")
|
||||
errno = payload.get("errno", 0)
|
||||
if errno != 0:
|
||||
raise TransferError(TransferErrorCode.NETWORK_ERROR, message=f"百度文件操作失败 errno={errno}", platform=self.PLATFORM_KEY, details=payload)
|
||||
return {"code": 0, "status": 200, "data": payload}
|
||||
|
||||
def delete_paths(self, paths: List[str]) -> bool:
|
||||
"""便捷删除方法（直接调用 cleanup）"""
|
||||
return self._cleanup.delete_files(paths)
|
||||
|
||||
@@ -8,7 +8,7 @@ PLATFORM_KEY = 'xunlei'
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import List, Optional, Tuple
|
||||
from typing import List, Optional, Tuple, Dict
|
||||
|
||||
from ..base import (
|
||||
BaseCloudDriveAdapter,
|
||||
@@ -32,11 +32,24 @@ class XunleiAdapter(BaseCloudDriveAdapter):
|
||||
PLATFORM_KEY = "xunlei"
|
||||
URL_PATTERNS = [r"pan\.xunlei\.com/s/([A-Za-z0-9]+)"]
|
||||
|
||||
capabilities: Dict[str, bool] = {
|
||||
**BaseCloudDriveAdapter.capabilities,
|
||||
"ensure_dir": True,
|
||||
"save_files": True,
|
||||
"poll_task": True,
|
||||
"rename": True,
|
||||
"move_files": True,
|
||||
# batchDelete is permanent on Xunlei, so do not advertise it as generic delete_files.
|
||||
"delete_files": False,
|
||||
}
|
||||
|
||||
def __init__(self, config: PlatformConfig, transfer_config: TransferConfig):
|
||||
super().__init__(config, transfer_config)
|
||||
# BaseCloudDriveAdapter.__init__ calls _setup_session(), so credential
|
||||
# state must exist before super().__init__.
|
||||
self._credential = XunleiCredentialManager(config)
|
||||
self._transfer_engine: Optional[XunleiTransfer] = None
|
||||
self._cleanup = XunleiCleanup()
|
||||
self._cleanup = XunleiCleanup(self._credential)
|
||||
super().__init__(config, transfer_config)
|
||||
|
||||
def _setup_session(self):
|
||||
"""初始化 session 认证头"""
|
||||
@@ -54,27 +67,35 @@ class XunleiAdapter(BaseCloudDriveAdapter):
|
||||
"""懒加载转存引擎"""
|
||||
if self._transfer_engine is None:
|
||||
self._transfer_engine = XunleiTransfer(
|
||||
self.session,
|
||||
self._credential,
|
||||
self.config,
|
||||
self.transfer_config,
|
||||
credential=self._credential,
|
||||
timeout=self.transfer_config.request_timeout,
|
||||
poll_interval=self.transfer_config.task_poll_interval,
|
||||
poll_max_attempts=self.transfer_config.task_poll_max_attempts,
|
||||
)
|
||||
self._transfer_engine.session = self.session
|
||||
return self._transfer_engine
|
||||
|
||||
# ─── 抽象方法实现 ──────────────────────────────
|
||||
|
||||
def _get_share_detail(self, pwd_id: str, passcode: str = "") -> dict:
|
||||
self._ensure_auth()
|
||||
return self._transfer.get_share_info(pwd_id, passcode)
|
||||
return self._transfer._get_share_info(pwd_id, passcode)
|
||||
|
||||
def _save_files(self, pwd_id: str, detail: dict, save_dir: str) -> List[str]:
|
||||
self._ensure_auth()
|
||||
return self._transfer.save_files(pwd_id, detail, save_dir)
|
||||
target = self.ensure_dir(save_dir) if save_dir and save_dir.startswith("/") else (save_dir or "")
|
||||
files = detail.get("files", [])
|
||||
file_ids = [f.get("file_id") or f.get("fid") or f.get("id") for f in files if f.get("file_id") or f.get("fid") or f.get("id")]
|
||||
if not file_ids:
|
||||
raise RuntimeError("无法从分享中提取文件ID")
|
||||
task_id = self._transfer._restore_files(pwd_id, detail.get("pass_code_token", ""), file_ids, parent_id=target)
|
||||
mapping = self._transfer._poll_restore_task(task_id)
|
||||
return [mapping.get(fid, "") for fid in file_ids if mapping.get(fid, "")]
|
||||
|
||||
def _create_share(self, file_ids: List[str], title: str,
|
||||
password: str = "") -> Tuple[str, str]:
|
||||
self._ensure_auth()
|
||||
return self._transfer.create_share(file_ids, title, password)
|
||||
return self._transfer._create_share(file_ids, password=password)
|
||||
|
||||
def _extract_file_list(self, detail: dict) -> List[FileInfo]:
|
||||
files = detail.get("files", [])
|
||||
@@ -94,13 +115,80 @@ class XunleiAdapter(BaseCloudDriveAdapter):
|
||||
|
||||
def get_files(self, parent_fid: str = "0") -> List[FileInfo]:
|
||||
self._ensure_auth()
|
||||
return self._transfer.list_files(parent_fid)
|
||||
url = "https://api-pan.xunlei.com/drive/v1/files"
|
||||
params = {"parent_id": "" if parent_fid in ("0", "/") else parent_fid}
|
||||
data = self._drive_api_json(self._get(url, params=params, headers=self._credential.get_headers()), context="迅雷网盘列目录")
|
||||
items = data.get("files", data.get("list", []))
|
||||
return [FileInfo(fid=item.get("id") or item.get("file_id", ""), name=item.get("name", ""), size=item.get("size", 0), is_dir=item.get("kind") == "drive#folder" or item.get("is_dir", False)) for item in items]
|
||||
|
||||
def ensure_dir(self, dir_path: str) -> str:
|
||||
normalized = "/" + (dir_path or "/").strip("/")
|
||||
if normalized == "/":
|
||||
return ""
|
||||
parent_id = ""
|
||||
current = ""
|
||||
for part in [p for p in normalized.split("/") if p]:
|
||||
current = f"{current}/{part}" if current else f"/{part}"
|
||||
existing = next((item for item in self.get_files(parent_id or "0") if item.is_dir and item.name == part), None)
|
||||
if existing:
|
||||
parent_id = existing.fid
|
||||
continue
|
||||
result = self.mkdir(current if not parent_id else part, parent_id=parent_id)
|
||||
parent_id = result.get("data", {}).get("fid", parent_id)
|
||||
return parent_id
|
||||
|
||||
def get_fids(self, file_paths: List[str]) -> List[Dict]:
|
||||
"""Resolve existing file/directory paths without creating anything."""
|
||||
results: List[Dict] = []
|
||||
for path in file_paths:
|
||||
normalized = "/" + (path or "").strip("/")
|
||||
if normalized == "/":
|
||||
results.append({"file_path": path, "fid": ""})
|
||||
continue
|
||||
parent_id = ""
|
||||
found: Optional[FileInfo] = None
|
||||
missing = False
|
||||
parts = [p for p in normalized.split("/") if p]
|
||||
for index, part in enumerate(parts):
|
||||
found = next((item for item in self.get_files(parent_id or "0") if item.name == part), None)
|
||||
if not found:
|
||||
missing = True
|
||||
break
|
||||
if index < len(parts) - 1 and not found.is_dir:
|
||||
missing = True
|
||||
break
|
||||
parent_id = found.fid
|
||||
if not missing and found:
|
||||
results.append({"file_path": path, "fid": found.fid})
|
||||
return results
|
||||
|
||||
def mkdir(self, dir_path: str, parent_id: str = "") -> Dict:
|
||||
name = dir_path.rstrip("/").rsplit("/", 1)[-1]
|
||||
body = {"kind": "drive#folder", "name": name, "parent_id": parent_id or ""}
|
||||
data = self._drive_api_json(self._post("https://api-pan.xunlei.com/drive/v1/files", json_data=body, headers=self._credential.get_headers_with_captcha(action="mkdir")), context="迅雷网盘创建目录")
|
||||
return {"code": 0, "status": 200, "data": {"fid": data.get("id") or data.get("file_id", ""), **data}}
|
||||
|
||||
def rename(self, fid: str, file_name: str) -> Dict:
|
||||
data = self._drive_api_json(self._post(f"https://api-pan.xunlei.com/drive/v1/files/{fid}", json_data={"name": file_name}, headers=self._credential.get_headers()), context="迅雷网盘重命名")
|
||||
return {"code": 0, "status": 200, "data": data}
|
||||
|
||||
def move_files(self, fids: List[str], to_pdir_fid: str) -> Dict:
|
||||
target = self.ensure_dir(to_pdir_fid) if to_pdir_fid.startswith("/") else to_pdir_fid
|
||||
data = self._drive_api_json(self._post("https://api-pan.xunlei.com/drive/v1/files:batchMove", json_data={"ids": fids, "parent_id": target}, headers=self._credential.get_headers()), context="迅雷网盘移动文件")
|
||||
return {"code": 0, "status": 200, "data": data}
|
||||
|
||||
def query_task(self, task_id: str) -> Dict:
|
||||
return self.poll_task(task_id)
|
||||
|
||||
def poll_task(self, task_id: str) -> Dict:
|
||||
return {"code": 0, "status": 200, "data": self._transfer._poll_restore_task(task_id)}
|
||||
|
||||
def delete_files(self, fids: List[str]) -> Dict:
|
||||
return {"code": 0, "status": 200} if self.delete(fids) else {"code": -1, "status": 500}
|
||||
|
||||
def delete(self, file_ids: List[str]) -> bool:
|
||||
self._ensure_auth()
|
||||
return self._cleanup.delete_files(
|
||||
self.session, self._credential, file_ids
|
||||
)
|
||||
return self._cleanup.delete_files(file_ids)
|
||||
|
||||
def _get_banned_keywords(self) -> List[str]:
|
||||
return self.config.banned_keywords or self.transfer_config.default_banned_keywords
|
||||
|
||||
@@ -70,7 +70,7 @@ class XunleiTransfer:
|
||||
|
||||
# ─── 步骤 ①：获取分享详情 ─────────────────────────────────────
|
||||
|
||||
def _get_share_info(self, share_id: str) -> Dict[str, Any]:
|
||||
def _get_share_info(self, share_id: str, passcode: str = "") -> Dict[str, Any]:
|
||||
"""步骤①：获取分享详情。
|
||||
|
||||
GET /drive/v1/share?share_id=<share_id>
|
||||
@@ -88,6 +88,8 @@ class XunleiTransfer:
|
||||
"""
|
||||
url = f"{XUNLEI_PAN_API}/drive/v1/share"
|
||||
params: Dict[str, str] = {"share_id": share_id}
|
||||
if passcode:
|
||||
params["pass_code"] = passcode
|
||||
headers = self.credential.get_headers()
|
||||
|
||||
logger.info("[XunleiTransfer] ① Fetching share info for share_id=%s", share_id)
|
||||
@@ -303,6 +305,7 @@ class XunleiTransfer:
|
||||
def _create_share(
|
||||
self,
|
||||
file_ids: List[str],
|
||||
password: str = "",
|
||||
expiration_days: str = "-1",
|
||||
) -> Tuple[str, str]:
|
||||
"""步骤④：创建新分享链接。
|
||||
@@ -331,6 +334,8 @@ class XunleiTransfer:
|
||||
"file_ids": file_ids,
|
||||
"expiration_days": expiration_days,
|
||||
}
|
||||
if password:
|
||||
body["pass_code"] = password
|
||||
# share 操作可能需要 captcha_token
|
||||
headers = self.credential.get_headers_with_captcha(action="share")
|
||||
headers.setdefault("Content-Type", "application/json")
|
||||
@@ -367,7 +372,7 @@ class XunleiTransfer:
|
||||
share_url,
|
||||
pass_code,
|
||||
)
|
||||
return share_url, pass_code
|
||||
return share_url, pass_code or password
|
||||
|
||||
# ─── 公开入口 ─────────────────────────────────────────────────
|
||||
|
||||
|
||||
@@ -150,8 +150,8 @@ class ConfigManager:
|
||||
"platforms": {
|
||||
name: {
|
||||
"enabled": cfg.enabled,
|
||||
"cookie": cfg.cookie[:20] + "..." if cfg.cookie else "",
|
||||
"refresh_token": cfg.refresh_token[:20] + "..." if cfg.refresh_token else "",
|
||||
"cookie": cfg.cookie,
|
||||
"refresh_token": cfg.refresh_token,
|
||||
"account_name": cfg.account_name,
|
||||
"save_dir": cfg.save_dir,
|
||||
"share_password": cfg.share_password,
|
||||
|
||||
@@ -7,8 +7,8 @@ import os
|
||||
import uuid
|
||||
import logging
|
||||
from flask import Flask, request, jsonify
|
||||
from config import ConfigManager
|
||||
from orchestration.transfer import TransferOrchestrator
|
||||
from cloudsearch_transfer.config import ConfigManager
|
||||
from cloudsearch_transfer.orchestration.transfer import TransferOrchestrator
|
||||
|
||||
# ─── 初始化 ────────────────────────────────────────────
|
||||
|
||||
@@ -149,6 +149,18 @@ def stats():
|
||||
return jsonify(orchestrator.get_stats())
|
||||
|
||||
|
||||
|
||||
def require_config_auth():
|
||||
token = os.getenv("TRANSFER_API_TOKEN", "")
|
||||
if not token:
|
||||
return jsonify({"error": "TRANSFER_API_TOKEN is required"}), 401
|
||||
supplied = request.headers.get("X-Transfer-Token") or request.headers.get("Authorization", "")
|
||||
if supplied.startswith("Bearer "):
|
||||
supplied = supplied[7:]
|
||||
if supplied != token:
|
||||
return jsonify({"error": "unauthorized"}), 401
|
||||
return None
|
||||
|
||||
# ─── 配置管理 ──────────────────────────────────────────
|
||||
|
||||
@app.route("/api/config/platforms", methods=["GET"])
|
||||
@@ -169,14 +181,17 @@ def get_platforms():
|
||||
@app.route("/api/config/platforms/<name>", methods=["PUT"])
|
||||
def update_platform(name):
|
||||
"""更新平台配置"""
|
||||
auth_error = require_config_auth()
|
||||
if auth_error:
|
||||
return auth_error
|
||||
data = request.get_json() or {}
|
||||
if name not in config.platforms:
|
||||
from config import PlatformConfig
|
||||
from cloudsearch_transfer.config import PlatformConfig
|
||||
config.platforms[name] = PlatformConfig()
|
||||
|
||||
cfg = config.platforms[name]
|
||||
if "enabled" in data:
|
||||
cfg.enabled = data["enabled"]
|
||||
cfg.enabled = bool(data["enabled"])
|
||||
if "cookie" in data:
|
||||
cfg.cookie = data["cookie"]
|
||||
if "refresh_token" in data:
|
||||
|
||||
@@ -0,0 +1,198 @@
|
||||
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from cloudsearch_transfer.adapter.aliyun import AliyunAdapter
|
||||
from cloudsearch_transfer.adapter.baidu import BaiduAdapter
|
||||
from cloudsearch_transfer.adapter.xunlei import XunleiAdapter
|
||||
from cloudsearch_transfer.config import PlatformConfig, TransferConfig
|
||||
from cloudsearch_transfer.adapter.base import FileInfo
|
||||
|
||||
|
||||
class DummyCredential:
|
||||
cookie = "k=xxxxxxxx"
|
||||
refresh_token = "rt"
|
||||
def validate(self):
|
||||
return True
|
||||
def get_bdstoken(self):
|
||||
return "bdstoken"
|
||||
def get_drive_id(self):
|
||||
return "drive-id"
|
||||
def get_headers(self):
|
||||
return {}
|
||||
def get_auth_headers(self):
|
||||
return {}
|
||||
|
||||
|
||||
def make_config(**kwargs):
|
||||
data = dict(enabled=True, cookie="k=" + "x" * 80, refresh_token="rt", account_name="test", save_dir="/")
|
||||
data.update(kwargs)
|
||||
return PlatformConfig(**data)
|
||||
|
||||
|
||||
def make_transfer_config():
|
||||
return TransferConfig(request_timeout=1, max_retries=0, ad_filter_enabled=False, task_poll_interval=0, task_poll_max_attempts=2)
|
||||
|
||||
|
||||
class P1DriveApiCapabilityTests(unittest.TestCase):
|
||||
def make_aliyun(self):
|
||||
with patch("cloudsearch_transfer.adapter.aliyun.AliyunCredentialManager", return_value=DummyCredential()):
|
||||
return AliyunAdapter(make_config(), make_transfer_config())
|
||||
|
||||
def make_baidu(self):
|
||||
with patch("cloudsearch_transfer.adapter.baidu.BaiduCredentialManager", return_value=DummyCredential()):
|
||||
return BaiduAdapter(make_config(), make_transfer_config())
|
||||
|
||||
def make_xunlei(self):
|
||||
with patch("cloudsearch_transfer.adapter.xunlei.XunleiCredentialManager", return_value=DummyCredential()):
|
||||
return XunleiAdapter(make_config(), make_transfer_config())
|
||||
|
||||
def test_p1_adapters_declare_callable_core_drive_api_methods(self):
|
||||
expectations = {
|
||||
"aliyun": (self.make_aliyun(), ["ensure_dir", "save_files", "rename", "move_files", "delete_files"]),
|
||||
"baidu": (self.make_baidu(), ["ensure_dir", "save_files", "rename", "move_files", "delete_files"]),
|
||||
"xunlei": (self.make_xunlei(), ["ensure_dir", "save_files", "poll_task", "rename", "move_files"]),
|
||||
}
|
||||
for platform, (adapter, required) in expectations.items():
|
||||
with self.subTest(platform=platform):
|
||||
for capability in required:
|
||||
self.assertTrue(adapter.capabilities.get(capability), f"{platform}.{capability} capability not declared")
|
||||
for method in ["ensure_dir", "mkdir", "rename", "move_files"]:
|
||||
self.assertNotEqual(getattr(type(adapter), method), getattr(adapter.__class__.__mro__[1], method, None), f"{platform}.{method} not overridden")
|
||||
if adapter.capabilities.get("delete_files"):
|
||||
self.assertNotEqual(getattr(type(adapter), "delete_files"), getattr(adapter.__class__.__mro__[1], "delete_files", None), f"{platform}.delete_files not overridden")
|
||||
|
||||
def test_aliyun_save_files_resolves_path_to_file_id_before_copy(self):
|
||||
adapter = self.make_aliyun()
|
||||
adapter.ensure_dir = lambda path: "target-folder-id"
|
||||
captured = {}
|
||||
transfer = adapter._get_transfer()
|
||||
def fake_batch_copy(share_id, share_token, file_ids, to_parent):
|
||||
captured["to_parent"] = to_parent
|
||||
return ["new-file"]
|
||||
transfer._batch_copy = fake_batch_copy
|
||||
detail = {"share_token": "share-token", "files": [{"file_id": "src-file"}]}
|
||||
self.assertEqual(adapter._save_files("share-id", detail, "/Media"), ["new-file"])
|
||||
self.assertEqual(captured["to_parent"], "target-folder-id")
|
||||
|
||||
def test_aliyun_ensure_dir_creates_nested_paths_progressively(self):
|
||||
adapter = self.make_aliyun()
|
||||
existing = {"/A": "fid-A"}
|
||||
created = []
|
||||
adapter.get_fids = lambda paths: [{"file_path": p, "fid": existing[p]} for p in paths if p in existing]
|
||||
def fake_mkdir(path):
|
||||
created.append(path)
|
||||
existing[path] = "fid-" + path.rsplit("/", 1)[-1]
|
||||
return {"code": 0, "data": {"fid": existing[path]}}
|
||||
adapter.mkdir = fake_mkdir
|
||||
self.assertEqual(adapter.ensure_dir("/A/B"), "fid-B")
|
||||
self.assertEqual(created, ["/A/B"])
|
||||
|
||||
def test_baidu_save_files_ensures_path_before_transfer_and_matching(self):
|
||||
adapter = self.make_baidu()
|
||||
adapter.ensure_dir = lambda path: "/CloudSearch/Media"
|
||||
captured = {}
|
||||
adapter.credential.get_bdstoken = lambda: "bdstoken"
|
||||
def fake_transfer_files(shareid, uk, fs_ids, save_dir, token):
|
||||
captured["transfer_dir"] = save_dir
|
||||
adapter._transfer._transfer_files = fake_transfer_files
|
||||
def fake_list_and_match(save_dir, filenames, token):
|
||||
captured["list_dir"] = save_dir
|
||||
return ["1001"]
|
||||
adapter._transfer._list_and_match = fake_list_and_match
|
||||
detail = {"shareid": "sid", "uk": "uk", "fs_ids": ["old"], "filenames": ["demo.mkv"]}
|
||||
self.assertEqual(adapter._save_files("pwd", detail, "/CloudSearch/Media"), ["1001"])
|
||||
self.assertEqual(captured["transfer_dir"], "/CloudSearch/Media")
|
||||
self.assertEqual(captured["list_dir"], "/CloudSearch/Media")
|
||||
|
||||
def test_baidu_ensure_dir_creates_nested_paths_progressively(self):
|
||||
adapter = self.make_baidu()
|
||||
dirs = {"/A"}
|
||||
created = []
|
||||
adapter.get_files = lambda parent="/": [FileInfo(fid=parent.rstrip("/") + "/A", name="A", is_dir=True)] if parent == "/" else []
|
||||
def fake_mkdir(path):
|
||||
created.append(path)
|
||||
dirs.add(path)
|
||||
return {"code": 0, "data": {"path": path}}
|
||||
adapter.mkdir = fake_mkdir
|
||||
self.assertEqual(adapter.ensure_dir("/A/B"), "/A/B")
|
||||
self.assertEqual(created, ["/A/B"])
|
||||
|
||||
def test_xunlei_save_files_resolves_path_and_polls_restore_task(self):
|
||||
adapter = self.make_xunlei()
|
||||
adapter.ensure_dir = lambda path: "target-parent-id"
|
||||
transfer = adapter._transfer
|
||||
transfer._restore_files = lambda share_id, token, fids, parent_id="": "task-1" if parent_id == "target-parent-id" else (_ for _ in ()).throw(AssertionError(parent_id))
|
||||
transfer._poll_restore_task = lambda task_id: {"old-1": "new-1"}
|
||||
detail = {"share_id": "share", "pass_code_token": "pct", "files": [{"id": "old-1", "name": "demo"}]}
|
||||
self.assertEqual(adapter._save_files("share", detail, "/Media"), ["new-1"])
|
||||
|
||||
def test_xunlei_query_task_delegates_to_restore_poll(self):
|
||||
adapter = self.make_xunlei()
|
||||
adapter._transfer._poll_restore_task = lambda task_id: {"old": "new"}
|
||||
self.assertEqual(adapter.query_task("task-1"), {"code": 0, "status": 200, "data": {"old": "new"}})
|
||||
|
||||
def test_xunlei_does_not_advertise_permanent_delete_as_drive_delete_capability(self):
|
||||
adapter = self.make_xunlei()
|
||||
self.assertFalse(adapter.capabilities.get("delete_files"), "迅雷 batchDelete is permanent and must not be advertised as generic delete_files")
|
||||
|
||||
def test_xunlei_get_share_detail_passes_extraction_code_to_transfer_engine(self):
|
||||
adapter = self.make_xunlei()
|
||||
captured = {}
|
||||
def fake_get_share_info(share_id, passcode=""):
|
||||
captured["share_id"] = share_id
|
||||
captured["passcode"] = passcode
|
||||
return {"title": "demo", "files": [{"id": "old"}], "pass_code_token": "pct"}
|
||||
adapter._transfer._get_share_info = fake_get_share_info
|
||||
detail = adapter._get_share_detail("share-id", "abcd")
|
||||
self.assertEqual(detail["title"], "demo")
|
||||
self.assertEqual(captured, {"share_id": "share-id", "passcode": "abcd"})
|
||||
|
||||
def test_xunlei_create_share_forwards_requested_share_password(self):
|
||||
adapter = self.make_xunlei()
|
||||
captured = {}
|
||||
def fake_create_share(file_ids, password=""):
|
||||
captured["file_ids"] = file_ids
|
||||
captured["password"] = password
|
||||
return ("https://pan.xunlei.com/s/new", password)
|
||||
adapter._transfer._create_share = fake_create_share
|
||||
self.assertEqual(adapter._create_share(["fid-1"], "demo", "xy12"), ("https://pan.xunlei.com/s/new", "xy12"))
|
||||
self.assertEqual(captured, {"file_ids": ["fid-1"], "password": "xy12"})
|
||||
|
||||
def test_xunlei_poll_task_alias_matches_declared_capability(self):
|
||||
adapter = self.make_xunlei()
|
||||
adapter._transfer._poll_restore_task = lambda task_id: {"old": "new"}
|
||||
self.assertEqual(adapter.poll_task("task-1"), {"code": 0, "status": 200, "data": {"old": "new"}})
|
||||
|
||||
def test_xunlei_get_fids_is_read_only_and_does_not_create_missing_paths(self):
|
||||
adapter = self.make_xunlei()
|
||||
calls = []
|
||||
def fake_get_files(parent="0"):
|
||||
calls.append(("get_files", parent))
|
||||
if parent in ("0", ""):
|
||||
return [FileInfo(fid="fid-A", name="A", is_dir=True)]
|
||||
if parent == "fid-A":
|
||||
return [FileInfo(fid="fid-B", name="B", is_dir=False)]
|
||||
return []
|
||||
def fail_mkdir(*args, **kwargs):
|
||||
raise AssertionError("get_fids must not create directories")
|
||||
adapter.get_files = fake_get_files
|
||||
adapter.mkdir = fail_mkdir
|
||||
self.assertEqual(adapter.get_fids(["/A/B", "/A/Missing"]), [{"file_path": "/A/B", "fid": "fid-B"}])
|
||||
self.assertEqual(calls, [("get_files", "0"), ("get_files", "fid-A"), ("get_files", "0"), ("get_files", "fid-A")])
|
||||
|
||||
def test_baidu_get_fids_checks_existing_parent_listing(self):
|
||||
adapter = self.make_baidu()
|
||||
calls = []
|
||||
def fake_get_files(parent="/"):
|
||||
calls.append(parent)
|
||||
if parent == "/A":
|
||||
return [FileInfo(fid="1001", name="movie.mkv", is_dir=False)]
|
||||
return []
|
||||
adapter.get_files = fake_get_files
|
||||
self.assertEqual(adapter.get_fids(["/A/movie.mkv", "/A/missing.mkv"]), [{"file_path": "/A/movie.mkv", "fid": "1001", "path": "/A/movie.mkv"}])
|
||||
self.assertEqual(calls, ["/A", "/A"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,135 @@
|
||||
import subprocess
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class TransferServerRuntimeTests(unittest.TestCase):
|
||||
def test_transfer_image_imports_server_module_from_app_workdir(self):
|
||||
result = subprocess.run(
|
||||
[
|
||||
"docker", "run", "--rm",
|
||||
"-e", "TRANSFER_CONFIG_PATH=/tmp/transfer_config.json",
|
||||
"cloudsearch-transfer:test",
|
||||
"python", "-c",
|
||||
"import cloudsearch_transfer.server as s; print(hasattr(s, 'app'))",
|
||||
],
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
timeout=20,
|
||||
)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertIn("True", result.stdout)
|
||||
|
||||
def test_update_platform_creates_config_with_package_import(self):
|
||||
result = subprocess.run(
|
||||
[
|
||||
"docker", "run", "--rm",
|
||||
"-e", "TRANSFER_CONFIG_PATH=/tmp/transfer_config_server_test.json",
|
||||
"cloudsearch-transfer:test",
|
||||
"python", "-c",
|
||||
"""
|
||||
import os
|
||||
os.environ['TRANSFER_API_TOKEN'] = 'test-token'
|
||||
from cloudsearch_transfer.server import app
|
||||
client = app.test_client()
|
||||
resp = client.put('/api/config/platforms/newplatform', headers={'X-Transfer-Token': 'test-token'}, json={'enabled': True, 'save_dir': '/Media'})
|
||||
print(resp.status_code)
|
||||
print(resp.get_data(as_text=True))
|
||||
""",
|
||||
],
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
timeout=20,
|
||||
)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertIn("200", result.stdout)
|
||||
self.assertIn("newplatform", result.stdout)
|
||||
|
||||
def test_dockerfile_healthcheck_uses_http_health_probe(self):
|
||||
dockerfile = Path(__file__).resolve().parents[1] / "Dockerfile"
|
||||
content = dockerfile.read_text()
|
||||
healthcheck_block = content.split("HEALTHCHECK", 1)[1].split("\n\nCMD", 1)[0]
|
||||
self.assertIn("/health", healthcheck_block)
|
||||
self.assertNotIn('python -m cloudsearch_transfer.server', healthcheck_block)
|
||||
|
||||
def test_config_update_requires_api_token_when_configured(self):
|
||||
script = """
|
||||
import os
|
||||
os.environ['TRANSFER_API_TOKEN'] = 'secret-token'
|
||||
os.environ['TRANSFER_CONFIG_PATH'] = '/tmp/transfer_config_auth_test.json'
|
||||
from cloudsearch_transfer.server import app
|
||||
client = app.test_client()
|
||||
unauth = client.put('/api/config/platforms/newplatform', json={'enabled': True})
|
||||
auth = client.put('/api/config/platforms/newplatform', headers={'X-Transfer-Token': 'secret-token'}, json={'enabled': True, 'save_dir': '/Media'})
|
||||
print(unauth.status_code)
|
||||
print(auth.status_code)
|
||||
"""
|
||||
result = subprocess.run(
|
||||
[
|
||||
"docker", "run", "--rm",
|
||||
"-e", "TRANSFER_CONFIG_PATH=/tmp/transfer_config_auth_test.json",
|
||||
"cloudsearch-transfer:test",
|
||||
"python", "-c", script,
|
||||
],
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
timeout=20,
|
||||
)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertIn("401", result.stdout)
|
||||
self.assertIn("200", result.stdout)
|
||||
|
||||
def test_config_update_requires_token_by_default(self):
|
||||
script = """
|
||||
import os
|
||||
os.environ.pop('TRANSFER_API_TOKEN', None)
|
||||
os.environ['TRANSFER_CONFIG_PATH'] = '/tmp/transfer_config_default_auth_test.json'
|
||||
from cloudsearch_transfer.server import app
|
||||
client = app.test_client()
|
||||
resp = client.put('/api/config/platforms/newplatform', json={'enabled': True})
|
||||
print(resp.status_code)
|
||||
"""
|
||||
result = subprocess.run(
|
||||
[
|
||||
"docker", "run", "--rm",
|
||||
"-e", "TRANSFER_CONFIG_PATH=/tmp/transfer_config_default_auth_test.json",
|
||||
"cloudsearch-transfer:test",
|
||||
"python", "-c", script,
|
||||
],
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
timeout=20,
|
||||
)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertIn("401", result.stdout)
|
||||
|
||||
def test_config_save_persists_full_credentials_for_runtime_reload(self):
|
||||
script = """
|
||||
from cloudsearch_transfer.config import ConfigManager, PlatformConfig
|
||||
path = '/tmp/transfer_config_persist_test.json'
|
||||
cfg = ConfigManager(path)
|
||||
cfg.platforms['quark'] = PlatformConfig(enabled=True, cookie='cookie-value-that-is-longer-than-twenty-characters', refresh_token='refresh-token-that-is-longer-than-twenty-characters')
|
||||
cfg.save()
|
||||
reloaded = ConfigManager(path)
|
||||
print(reloaded.platforms['quark'].cookie)
|
||||
print(reloaded.platforms['quark'].refresh_token)
|
||||
"""
|
||||
result = subprocess.run(
|
||||
["docker", "run", "--rm", "cloudsearch-transfer:test", "python", "-c", script],
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
timeout=20,
|
||||
)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertIn("cookie-value-that-is-longer-than-twenty-characters", result.stdout)
|
||||
self.assertIn("refresh-token-that-is-longer-than-twenty-characters", result.stdout)
|
||||
self.assertNotIn("...", result.stdout)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user