timxx/CloudSearch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

deploy: sync server CloudSearch 0.5.6

Browse Source
This commit is contained in:
admin
2026-06-25 02:08:39 +08:00
parent 75f5d26964
commit ce3f95b8b9
53 changed files with 2740 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cloudsearch_transfer/adapter/aliyun/__init__.py
+87 -11
View File
@@ -15,8 +15,8 @@ import logging
from typing import List, Dict, Tuple, Optional
from ..base import BaseCloudDriveAdapter, FileInfo, match_url
from ..config import PlatformConfig, TransferConfig
from ..errors import TransferError, TransferErrorCode
from ...config import PlatformConfig, TransferConfig
from ...errors import TransferError, TransferErrorCode
from .credential import AliyunCredentialManager
from .transfer import AliyunTransfer
@@ -47,23 +47,29 @@ class AliyunAdapter(BaseCloudDriveAdapter):
"Referer": "https://aliyundrive.com",
}
def __init__(self, config: PlatformConfig, transfer_config: TransferConfig):
super().__init__(config, transfer_config)
capabilities: Dict[str, bool] = {
**BaseCloudDriveAdapter.capabilities,
"ensure_dir": True,
"save_files": True,
"rename": True,
"move_files": True,
"delete_files": True,
}
# 创建凭证管理器（AliyunCredentialManager）
def __init__(self, config: PlatformConfig, transfer_config: TransferConfig):
# BaseCloudDriveAdapter.__init__ calls _setup_session(), so credential
# state must exist before super().__init__.
refresh_token = config.refresh_token or config.cookie or ""
self._credential = AliyunCredentialManager(refresh_token=refresh_token)
# 初始化 drive_id
self._drive_id = ""
# 创建子模块
self._transfer: Optional[AliyunTransfer] = None
self._cleanup: Optional[AliyunCleanup] = None
super().__init__(config, transfer_config)
def _setup_session(self):
"""初始化 session 和凭证"""
if self._credential.refresh_token:
refresh_token = getattr(self._credential, "refresh_token", "")
if refresh_token:
# 验证 refresh_token 并获取 drive_id
if self._credential.validate():
self._drive_id = self._credential.get_drive_id()
@@ -164,8 +170,10 @@ class AliyunAdapter(BaseCloudDriveAdapter):
platform=self.PLATFORM_KEY,
)
# 确定目标目录
# 确定目标目录：路径先解析/创建为 file_id。
to_parent = save_dir if save_dir and save_dir != "/" else "root"
if isinstance(to_parent, str) and to_parent.startswith("/"):
to_parent = self.ensure_dir(to_parent)
transfer = self._get_transfer()
new_ids = transfer._batch_copy(pwd_id, share_token, file_ids, to_parent)
@@ -237,6 +245,74 @@ class AliyunAdapter(BaseCloudDriveAdapter):
# ─── 扩展功能 ──────────────────────────────────────────
# ─── Optional Drive API capability methods ─────────────────────
def get_fids(self, file_paths: List[str]) -> List[Dict]:
wanted = {p.rstrip("/") or "/" for p in file_paths}
found: List[Dict] = []
for path in wanted:
if path == "/":
found.append({"file_path": "/", "fid": "root"})
continue
parent = path.rsplit("/", 1)[0] or "/"
name = path.rsplit("/", 1)[-1]
for item in self.get_files(parent):
if item.is_dir and item.name == name:
found.append({"file_path": path, "fid": item.fid})
break
return found
def ensure_dir(self, dir_path: str) -> str:
normalized = "/" + (dir_path or "/").strip("/")
if normalized == "/":
return "root"
current = ""
last_fid = "root"
for part in [p for p in normalized.split("/") if p]:
current = f"{current}/{part}" if current else f"/{part}"
matches = self.get_fids([current])
if matches:
last_fid = matches[0].get("fid") or matches[0].get("file_id") or last_fid
continue
created = self.mkdir(current)
data = created.get("data", created) if isinstance(created, dict) else {}
last_fid = data.get("fid") or data.get("file_id") or data.get("id") or last_fid
return last_fid
def mkdir(self, dir_path: str) -> Dict:
parent = dir_path.rsplit("/", 1)[0] or "/"
name = dir_path.rstrip("/").rsplit("/", 1)[-1]
parent_fid = "root" if parent == "/" else self.ensure_dir(parent)
url = "https://api.aliyundrive.com/adrive/v2/file/createWithFolders"
body = {
"drive_id": self._drive_id or self._credential.get_drive_id(),
"parent_file_id": parent_fid,
"name": name,
"type": "folder",
"check_name_mode": "refuse",
}
resp = self._post(url, json_data=body, headers=self._credential.get_headers())
data = self._drive_api_json(resp, context="阿里云盘创建目录")
return {"code": 0, "status": 200, "data": {"fid": data.get("file_id", ""), **data}}
def rename(self, fid: str, file_name: str) -> Dict:
url = "https://api.aliyundrive.com/v3/file/update"
body = {"drive_id": self._drive_id or self._credential.get_drive_id(), "file_id": fid, "name": file_name, "check_name_mode": "refuse"}
data = self._drive_api_json(self._post(url, json_data=body, headers=self._credential.get_headers()), context="阿里云盘重命名")
return {"code": 0, "status": 200, "data": data}
def move_files(self, fids: List[str], to_pdir_fid: str) -> Dict:
if isinstance(to_pdir_fid, str) and to_pdir_fid.startswith("/"):
to_pdir_fid = self.ensure_dir(to_pdir_fid)
drive_id = self._drive_id or self._credential.get_drive_id()
requests = [{"id": fid, "method": "POST", "url": "/file/move", "headers": {"Content-Type": "application/json"}, "body": {"drive_id": drive_id, "file_id": fid, "to_parent_file_id": to_pdir_fid}} for fid in fids]
data = self._drive_api_json(self._post("https://api.aliyundrive.com/adrive/v4/batch", json_data={"requests": requests, "resource": "file"}, headers=self._credential.get_headers()), context="阿里云盘移动文件")
return {"code": 0, "status": 200, "data": data}
def delete_files(self, fids: List[str]) -> Dict:
return {"code": 0, "status": 200} if self.delete(fids) else {"code": -1, "status": 500}
def cleanup_files(self, file_ids: List[str]) -> Dict:
"""
清理文件（移入回收站），返回详细结果。
cloudsearch_transfer/adapter/baidu/__init__.py
+79 -3
View File
@@ -6,7 +6,7 @@
"""
import logging
from typing import List, Tuple
from typing import List, Tuple, Dict
from ..base import BaseCloudDriveAdapter, FileInfo
from ...config import PlatformConfig, TransferConfig
@@ -36,6 +36,15 @@ class BaiduAdapter(BaseCloudDriveAdapter):
r'pan\.baidu\.com/s/1([A-Za-z0-9_-]+)',
]
capabilities: Dict[str, bool] = {
**BaseCloudDriveAdapter.capabilities,
"ensure_dir": True,
"save_files": True,
"rename": True,
"move_files": True,
"delete_files": True,
}
def __init__(self, config: PlatformConfig, transfer_config: TransferConfig):
super().__init__(config, transfer_config)
@@ -123,11 +132,13 @@ class BaiduAdapter(BaseCloudDriveAdapter):
fs_ids = detail["fs_ids"]
filenames = detail.get("filenames", [])
target_dir = self.ensure_dir(save_dir) if save_dir and save_dir.startswith("/") else (save_dir or "/")
# ③ 转存
self._transfer._transfer_files(shareid, uk, fs_ids, save_dir, bdstoken)
self._transfer._transfer_files(shareid, uk, fs_ids, target_dir, bdstoken)
# ④ 列出目录匹配新 fs_id
new_fs_ids = self._transfer._list_and_match(save_dir, filenames, bdstoken)
new_fs_ids = self._transfer._list_and_match(target_dir, filenames, bdstoken)
# 暂存文件信息供 _filter_ads + _create_share 使用
self._last_transfer_files = [
@@ -248,6 +259,71 @@ class BaiduAdapter(BaseCloudDriveAdapter):
# ─── 扩展方法 ────────────────────────────────────────────
# ─── Optional Drive API capability methods ─────────────────────
def ensure_dir(self, dir_path: str) -> str:
normalized = "/" + (dir_path or "/").strip("/")
if normalized == "/":
return "/"
current = ""
for part in [p for p in normalized.split("/") if p]:
parent = current or "/"
current = f"{current}/{part}" if current else f"/{part}"
exists = any(item.is_dir and item.name == part for item in self.get_files(parent))
if not exists:
self.mkdir(current)
return normalized
def get_fids(self, file_paths: List[str]) -> List[Dict]:
"""Resolve existing Baidu paths by listing their parent directories."""
results: List[Dict] = []
for path in file_paths:
normalized = "/" + (path or "").strip("/")
if normalized == "/":
results.append({"file_path": path, "fid": "/", "path": "/"})
continue
parent, name = normalized.rsplit("/", 1)
parent = parent or "/"
match = next((item for item in self.get_files(parent) if item.name == name), None)
if match:
results.append({"file_path": path, "fid": match.fid, "path": normalized})
return results
def mkdir(self, dir_path: str) -> Dict:
bdstoken = self.credential.get_bdstoken()
url = "https://pan.baidu.com/api/create"
params = {"a": "commit", "bdstoken": bdstoken}
data = {"path": dir_path, "isdir": 1, "block_list": "[]"}
resp = self._post(url, data=data, params=params, headers=self.credential.get_headers())
payload = self._drive_api_json(resp, context="百度网盘创建目录")
errno = payload.get("errno", 0)
if errno not in (0, -8):
raise TransferError(TransferErrorCode.NETWORK_ERROR, message=f"百度创建目录失败 errno={errno}", platform=self.PLATFORM_KEY, details=payload)
return {"code": 0, "status": 200, "data": {"path": dir_path, **payload}}
def rename(self, fid: str, file_name: str) -> Dict:
return self._filemanager("rename", [{"path": fid, "newname": file_name}])
def move_files(self, fids: List[str], to_pdir_fid: str) -> Dict:
target = self.ensure_dir(to_pdir_fid) if to_pdir_fid.startswith("/") else to_pdir_fid
return self._filemanager("move", [{"path": fid, "dest": target} for fid in fids])
def delete_files(self, fids: List[str]) -> Dict:
return {"code": 0, "status": 200} if self.delete(fids) else {"code": -1, "status": 500}
def _filemanager(self, opera: str, filelist: List[Dict]) -> Dict:
import json
bdstoken = self.credential.get_bdstoken()
url = "https://pan.baidu.com/api/filemanager"
params = {"opera": opera, "bdstoken": bdstoken}
data = {"filelist": json.dumps(filelist, ensure_ascii=False)}
payload = self._drive_api_json(self._post(url, data=data, params=params, headers=self.credential.get_headers()), context=f"百度网盘{opera}")
errno = payload.get("errno", 0)
if errno != 0:
raise TransferError(TransferErrorCode.NETWORK_ERROR, message=f"百度文件操作失败 errno={errno}", platform=self.PLATFORM_KEY, details=payload)
return {"code": 0, "status": 200, "data": payload}
def delete_paths(self, paths: List[str]) -> bool:
"""便捷删除方法（直接调用 cleanup）"""
return self._cleanup.delete_files(paths)
cloudsearch_transfer/adapter/xunlei/__init__.py
+102 -14
View File
@@ -8,7 +8,7 @@ PLATFORM_KEY = 'xunlei'
from __future__ import annotations
import logging
from typing import List, Optional, Tuple
from typing import List, Optional, Tuple, Dict
from ..base import (
BaseCloudDriveAdapter,
@@ -32,11 +32,24 @@ class XunleiAdapter(BaseCloudDriveAdapter):
PLATFORM_KEY = "xunlei"
URL_PATTERNS = [r"pan\.xunlei\.com/s/([A-Za-z0-9]+)"]
capabilities: Dict[str, bool] = {
**BaseCloudDriveAdapter.capabilities,
"ensure_dir": True,
"save_files": True,
"poll_task": True,
"rename": True,
"move_files": True,
# batchDelete is permanent on Xunlei, so do not advertise it as generic delete_files.
"delete_files": False,
}
def __init__(self, config: PlatformConfig, transfer_config: TransferConfig):
super().__init__(config, transfer_config)
# BaseCloudDriveAdapter.__init__ calls _setup_session(), so credential
# state must exist before super().__init__.
self._credential = XunleiCredentialManager(config)
self._transfer_engine: Optional[XunleiTransfer] = None
self._cleanup = XunleiCleanup()
self._cleanup = XunleiCleanup(self._credential)
super().__init__(config, transfer_config)
def _setup_session(self):
"""初始化 session 认证头"""
@@ -54,27 +67,35 @@ class XunleiAdapter(BaseCloudDriveAdapter):
"""懒加载转存引擎"""
if self._transfer_engine is None:
self._transfer_engine = XunleiTransfer(
self.session,
self._credential,
self.config,
self.transfer_config,
credential=self._credential,
timeout=self.transfer_config.request_timeout,
poll_interval=self.transfer_config.task_poll_interval,
poll_max_attempts=self.transfer_config.task_poll_max_attempts,
)
self._transfer_engine.session = self.session
return self._transfer_engine
# ─── 抽象方法实现 ──────────────────────────────
def _get_share_detail(self, pwd_id: str, passcode: str = "") -> dict:
self._ensure_auth()
return self._transfer.get_share_info(pwd_id, passcode)
return self._transfer._get_share_info(pwd_id, passcode)
def _save_files(self, pwd_id: str, detail: dict, save_dir: str) -> List[str]:
self._ensure_auth()
return self._transfer.save_files(pwd_id, detail, save_dir)
target = self.ensure_dir(save_dir) if save_dir and save_dir.startswith("/") else (save_dir or "")
files = detail.get("files", [])
file_ids = [f.get("file_id") or f.get("fid") or f.get("id") for f in files if f.get("file_id") or f.get("fid") or f.get("id")]
if not file_ids:
raise RuntimeError("无法从分享中提取文件ID")
task_id = self._transfer._restore_files(pwd_id, detail.get("pass_code_token", ""), file_ids, parent_id=target)
mapping = self._transfer._poll_restore_task(task_id)
return [mapping.get(fid, "") for fid in file_ids if mapping.get(fid, "")]
def _create_share(self, file_ids: List[str], title: str,
password: str = "") -> Tuple[str, str]:
self._ensure_auth()
return self._transfer.create_share(file_ids, title, password)
return self._transfer._create_share(file_ids, password=password)
def _extract_file_list(self, detail: dict) -> List[FileInfo]:
files = detail.get("files", [])
@@ -94,13 +115,80 @@ class XunleiAdapter(BaseCloudDriveAdapter):
def get_files(self, parent_fid: str = "0") -> List[FileInfo]:
self._ensure_auth()
return self._transfer.list_files(parent_fid)
url = "https://api-pan.xunlei.com/drive/v1/files"
params = {"parent_id": "" if parent_fid in ("0", "/") else parent_fid}
data = self._drive_api_json(self._get(url, params=params, headers=self._credential.get_headers()), context="迅雷网盘列目录")
items = data.get("files", data.get("list", []))
return [FileInfo(fid=item.get("id") or item.get("file_id", ""), name=item.get("name", ""), size=item.get("size", 0), is_dir=item.get("kind") == "drive#folder" or item.get("is_dir", False)) for item in items]
def ensure_dir(self, dir_path: str) -> str:
normalized = "/" + (dir_path or "/").strip("/")
if normalized == "/":
return ""
parent_id = ""
current = ""
for part in [p for p in normalized.split("/") if p]:
current = f"{current}/{part}" if current else f"/{part}"
existing = next((item for item in self.get_files(parent_id or "0") if item.is_dir and item.name == part), None)
if existing:
parent_id = existing.fid
continue
result = self.mkdir(current if not parent_id else part, parent_id=parent_id)
parent_id = result.get("data", {}).get("fid", parent_id)
return parent_id
def get_fids(self, file_paths: List[str]) -> List[Dict]:
"""Resolve existing file/directory paths without creating anything."""
results: List[Dict] = []
for path in file_paths:
normalized = "/" + (path or "").strip("/")
if normalized == "/":
results.append({"file_path": path, "fid": ""})
continue
parent_id = ""
found: Optional[FileInfo] = None
missing = False
parts = [p for p in normalized.split("/") if p]
for index, part in enumerate(parts):
found = next((item for item in self.get_files(parent_id or "0") if item.name == part), None)
if not found:
missing = True
break
if index < len(parts) - 1 and not found.is_dir:
missing = True
break
parent_id = found.fid
if not missing and found:
results.append({"file_path": path, "fid": found.fid})
return results
def mkdir(self, dir_path: str, parent_id: str = "") -> Dict:
name = dir_path.rstrip("/").rsplit("/", 1)[-1]
body = {"kind": "drive#folder", "name": name, "parent_id": parent_id or ""}
data = self._drive_api_json(self._post("https://api-pan.xunlei.com/drive/v1/files", json_data=body, headers=self._credential.get_headers_with_captcha(action="mkdir")), context="迅雷网盘创建目录")
return {"code": 0, "status": 200, "data": {"fid": data.get("id") or data.get("file_id", ""), **data}}
def rename(self, fid: str, file_name: str) -> Dict:
data = self._drive_api_json(self._post(f"https://api-pan.xunlei.com/drive/v1/files/{fid}", json_data={"name": file_name}, headers=self._credential.get_headers()), context="迅雷网盘重命名")
return {"code": 0, "status": 200, "data": data}
def move_files(self, fids: List[str], to_pdir_fid: str) -> Dict:
target = self.ensure_dir(to_pdir_fid) if to_pdir_fid.startswith("/") else to_pdir_fid
data = self._drive_api_json(self._post("https://api-pan.xunlei.com/drive/v1/files:batchMove", json_data={"ids": fids, "parent_id": target}, headers=self._credential.get_headers()), context="迅雷网盘移动文件")
return {"code": 0, "status": 200, "data": data}
def query_task(self, task_id: str) -> Dict:
return self.poll_task(task_id)
def poll_task(self, task_id: str) -> Dict:
return {"code": 0, "status": 200, "data": self._transfer._poll_restore_task(task_id)}
def delete_files(self, fids: List[str]) -> Dict:
return {"code": 0, "status": 200} if self.delete(fids) else {"code": -1, "status": 500}
def delete(self, file_ids: List[str]) -> bool:
self._ensure_auth()
return self._cleanup.delete_files(
self.session, self._credential, file_ids
)
return self._cleanup.delete_files(file_ids)
def _get_banned_keywords(self) -> List[str]:
return self.config.banned_keywords or self.transfer_config.default_banned_keywords
cloudsearch_transfer/adapter/xunlei/transfer.py
+7 -2
View File
@@ -70,7 +70,7 @@ class XunleiTransfer:
# ─── 步骤 ①：获取分享详情 ─────────────────────────────────────
def _get_share_info(self, share_id: str) -> Dict[str, Any]:
def _get_share_info(self, share_id: str, passcode: str = "") -> Dict[str, Any]:
"""步骤①：获取分享详情。
GET /drive/v1/share?share_id=<share_id>
@@ -88,6 +88,8 @@ class XunleiTransfer:
"""
url = f"{XUNLEI_PAN_API}/drive/v1/share"
params: Dict[str, str] = {"share_id": share_id}
if passcode:
params["pass_code"] = passcode
headers = self.credential.get_headers()
logger.info("[XunleiTransfer] ① Fetching share info for share_id=%s", share_id)
@@ -303,6 +305,7 @@ class XunleiTransfer:
def _create_share(
self,
file_ids: List[str],
password: str = "",
expiration_days: str = "-1",
) -> Tuple[str, str]:
"""步骤④：创建新分享链接。
@@ -331,6 +334,8 @@ class XunleiTransfer:
"file_ids": file_ids,
"expiration_days": expiration_days,
}
if password:
body["pass_code"] = password
# share 操作可能需要 captcha_token
headers = self.credential.get_headers_with_captcha(action="share")
headers.setdefault("Content-Type", "application/json")
@@ -367,7 +372,7 @@ class XunleiTransfer:
share_url,
pass_code,
)
return share_url, pass_code
return share_url, pass_code or password
# ─── 公开入口 ─────────────────────────────────────────────────