Files
CloudSearch/packages/backend/src/routes/search.routes.ts

630 lines
23 KiB
TypeScript

import { Router, Request, Response } from 'express';
// Native fetch available in Node 20+
import { searchLimiter, saveLimiter } from '../middleware/rate-limit';
import { detectIntent } from '../intent/intent.service';
import { search, applyTitleFilter } from '../search/search.service';
import { getRankings, getHotKeywords, getCategorizedRankings } from '../search/rankings.service';
import { parseVideo } from '../video/video.service';
import { saveFromShare } from '../cloud/cloud.service';
import { getEnabledCloudTypeSet } from '../cloud/cloud-types.service';
import { getSystemConfig } from '../admin/system-config.service';
import { verifyToken } from '../admin/auth.service';
import { LinkValidator } from '../validation/link-validator.service';
import { getContentInfo } from '../content/content.service';
import { detectCloudType } from '../config/cloud-labels';
import { CLOUD_LABELS, CLOUD_COLORS } from '../config/cloud-labels';
import { getDb } from '../database/database';
const router = Router();
// ============ Search & Query ============
/**
* POST /api/query
* Intent recognition + execution
*/
router.post('/query', searchLimiter, async (req: Request, res: Response) => {
try {
const { input, q } = req.body;
const query = input || q;
if (!query || typeof query !== 'string') {
res.status(400).json({ error: 'Input is required' });
return;
}
const intent = detectIntent(query);
const ip = req.ip || req.socket.remoteAddress || '';
switch (intent.type) {
case 'SEARCH': {
const result = await search(intent.cleanInput, 1, ip);
// Pass through: use all results, group by cloud type
const allResults = result.results || [];
// Transform to frontend-friendly format
let formatted = (allResults || []).map((item: any, idx: number) => ({
id: `search_${idx}`,
title: filterTitle(item.title || item.content || ''),
description: item.content || '',
share_url: item.url || '',
cloud_type: detectCloudType(item.url || ''),
file_size: '',
update_time: item.datetime || '',
source: item.source || '',
file_id: '',
cover: Array.isArray(item.images) && item.images.length > 0 ? item.images[0] : '',
password: item.password || '',
}));
// Filter out expired/invalid links
formatted = formatted.filter(r => !r.share_url || !isExpiredShareLink(r.share_url));
// Filter by enabled cloud types (admin-configurable per-type toggle)
// Skip filter if search_all_channels is enabled
const searchAllChannels = getSystemConfig('search_all_channels') === 'true';
if (!searchAllChannels) {
const enabledSet = getEnabledCloudTypeSet();
formatted = formatted.filter(r => !r.cloud_type || enabledSet.has(r.cloud_type));
}
const contentQuery = intent.cleanInput || query;
const contentInfo = await getContentInfo(contentQuery).catch(() => null);
const extractedTags = extractTagsFromResults(formatted, contentQuery);
const linkValidationEnabled = getSystemConfig('link_validation_enabled') !== 'false';
// Set up streaming response (NDJSON)
res.setHeader('Content-Type', 'application/x-ndjson');
res.setHeader('X-Accel-Buffering', 'no');
res.setHeader('Cache-Control', 'no-cache');
res.setHeader('Connection', 'keep-alive');
// 0. Send searching signal immediately so frontend shows feedback
res.write(JSON.stringify({ type: 'searching' }) + '\n');
// 0.5 Query local DB for previously saved resources matching keyword
const savedResults = getSavedResources(intent.cleanInput);
if (savedResults.length > 0) {
res.write(JSON.stringify({
type: 'saved',
results: savedResults,
total: savedResults.length,
}) + '\n');
}
// 1. Send stats immediately
const fallbackImage = getSystemConfig('search_fallback_image') || '';
const siteLogo = getSystemConfig('site_logo') || '';
const siteNameInStats = getSystemConfig('site_name') || 'CloudSearch';
const siteDisclaimer = getSystemConfig('site_disclaimer') || '';
const siteMarquee = getSystemConfig('site_marquee') || '';
const statsPayload = {
type: 'stats',
total: formatted.length,
channels: groupResultsByChannel(formatted, (item: any) => item.cloud_type),
content_info: contentInfo,
content_tags: extractedTags,
link_validation: linkValidationEnabled,
fallback_image: fallbackImage,
site_logo: siteLogo,
site_name: siteNameInStats,
site_disclaimer: siteDisclaimer,
site_marquee: siteMarquee,
};
res.write(JSON.stringify(statsPayload) + '\n');
// 2. Validate links — per-type grouping, newest-first, per-type cap from config
if (linkValidationEnabled) {
const validator = new LinkValidator();
const resultLimit = parseInt(getSystemConfig('search_result_limit') || '10', 10);
const MAX_VALID_PER_TYPE = Math.min(100, Math.max(1, resultLimit)); // configurable, 1-100
const MAX_TOTAL_VALID = MAX_VALID_PER_TYPE * 6; // up to 6 cloud types
const pool = validator['pool']; // concurrency: 10
// Group formatted results by cloud_type, then sort each group by time desc
const byType: Record<string, any[]> = {};
for (const item of formatted) {
const ct = item.cloud_type || 'others';
if (!byType[ct]) byType[ct] = [];
byType[ct].push(item);
}
// Sort each group by update_time descending (newest first)
for (const ct of Object.keys(byType)) {
byType[ct].sort((a: any, b: any) => {
const ta = a.update_time || '';
const tb = b.update_time || '';
if (!ta && !tb) return 0;
if (!ta) return 1;
if (!tb) return -1;
return tb.localeCompare(ta);
});
}
// Build a round-robin validation queue: interleave items from each type
// to give fair priority across all cloud types
const typeOrder = ['quark', 'baidu', 'aliyun', '115', 'tianyi', '123pan', 'uc', 'xunlei', 'pikpak', 'magnet', 'ed2k', 'others'];
const sortedTypes = typeOrder.filter(ct => byType[ct] && byType[ct].length > 0);
// Sort by total count descending so types with more results get more validation slots
sortedTypes.sort((a, b) => (byType[b]?.length || 0) - (byType[a]?.length || 0));
const validationQueue: { item: any; type: string }[] = [];
const maxLen = Math.max(...sortedTypes.map(ct => byType[ct].length), 0);
for (let i = 0; i < maxLen; i++) {
for (const ct of sortedTypes) {
if (i < byType[ct].length) {
validationQueue.push({ item: byType[ct][i], type: ct });
}
}
}
const validResults: any[] = [];
const perTypeValid: Record<string, number> = {};
let totalValid = 0;
let totalInvalid = 0;
let totalChecked = 0;
const unknownItemIds: number[] = []; // IDs that got 'unknown' from PanSou
// Pass 1: PanSou-only validation
const tasks = validationQueue.map(({ item, type }) => pool.run(async () => {
// Stop if we've hit overall cap or per-type cap
if (totalValid >= MAX_TOTAL_VALID) return;
if ((perTypeValid[type] || 0) >= MAX_VALID_PER_TYPE) return;
totalChecked++;
try {
const vr = await validator.validate(item.share_url, item.cloud_type);
// 'unknown' = PanSou couldn't determine → treat as valid for now
if (vr.status === 'valid' || vr.status === 'unknown') {
if (vr.status === 'unknown') {
unknownItemIds.push(item.id);
}
if (totalValid < MAX_TOTAL_VALID && (perTypeValid[type] || 0) < MAX_VALID_PER_TYPE) {
validResults.push(item);
perTypeValid[type] = (perTypeValid[type] || 0) + 1;
totalValid++;
res.write(JSON.stringify({ type: 'result', id: item.id, valid: true, message: vr.message }) + '\n');
}
} else {
totalInvalid++;
res.write(JSON.stringify({ type: 'result', id: item.id, valid: false, message: vr.message }) + '\n');
}
} catch {
if (totalValid < MAX_TOTAL_VALID && (perTypeValid[type] || 0) < MAX_VALID_PER_TYPE) {
validResults.push(item);
perTypeValid[type] = (perTypeValid[type] || 0) + 1;
totalValid++;
}
res.write(JSON.stringify({ type: 'result', id: item.id, valid: true }) + '\n');
}
}));
await Promise.all(tasks);
// Pass 2: If PanSou didn't provide enough valid results, validate
// uncertain items with local fallback (external API calls)
if (totalValid < MAX_TOTAL_VALID && unknownItemIds.length > 0) {
const unknownItems = validationQueue.filter(({ item }) => unknownItemIds.includes(item.id));
for (const { item, type } of unknownItems) {
if (totalValid >= MAX_TOTAL_VALID) break;
if ((perTypeValid[type] || 0) >= MAX_VALID_PER_TYPE) break;
try {
const vr = await validator.validateWithLocalFallback(item.share_url, item.cloud_type);
if (vr.status === 'valid') {
// Already in validResults from pass 1, just count it again
perTypeValid[type] = (perTypeValid[type] || 0) + 1;
totalValid++;
res.write(JSON.stringify({ type: 'result', id: item.id, valid: true, message: vr.message + ' (本地确认)' }) + '\n');
} else if (vr.status === 'invalid') {
// Remove from validResults — was previously included as unknown
const idx = validResults.findIndex(r => r.id === item.id);
if (idx >= 0) {
validResults.splice(idx, 1);
perTypeValid[type] = Math.max(0, (perTypeValid[type] || 1) - 1);
totalValid--;
}
totalInvalid++;
res.write(JSON.stringify({ type: 'result', id: item.id, valid: false, message: vr.message + ' (本地确认失效)' }) + '\n');
}
} catch {
// Keep as-is (already treated as valid from pass 1)
}
}
}
const skippedCount = validationQueue.length - totalChecked;
res.write(JSON.stringify({
type: 'complete',
results: validResults,
channels: groupResultsByChannel(validResults, (item: any) => item.cloud_type),
total: validResults.length,
filtered: totalInvalid,
per_type: perTypeValid,
skipped: skippedCount,
}) + '\n');
} else {
// No validation - just send all results
res.write(JSON.stringify({
type: 'complete',
results: formatted,
channels: groupResultsByChannel(formatted, (item: any) => item.cloud_type),
total: formatted.length,
filtered: 0,
}) + '\n');
}
res.end();
break;
}
case 'VIDEO_PARSE': {
const videoInfo = await parseVideo(intent.cleanInput);
res.json({ intent: intent.type, platform: intent.platform, data: videoInfo });
break;
}
case 'CLOUD_SAVE': {
const result = await saveFromShare(intent.cleanInput, intent.platform || '', undefined, req.ip);
res.json({ intent: intent.type, platform: intent.platform, ...result });
break;
}
default:
res.status(400).json({ error: 'Unknown intent type' });
}
} catch (err: any) {
console.error('[Query] Error:', err);
res.status(500).json({ error: err.message || 'Internal server error' });
}
});
/**
* GET /api/search
* Search with optional link validation filtering
*/
router.get('/search', searchLimiter, async (req: Request, res: Response) => {
try {
const keyword = (req.query.q || req.query.kw) as string;
const page = parseInt(req.query.page as string || '1', 10);
const ip = req.ip || req.socket.remoteAddress || '';
if (!keyword) {
res.status(400).json({ error: 'Query parameter "q" is required' });
return;
}
const result = await search(keyword, page, ip);
// Pass through: use all results
const allResults = result.results || [];
// Transform to frontend format
let formatted = (allResults || []).map((item: any) => ({
id: item.id || '',
title: filterTitle(item.title || item.content || ''),
description: item.content || item.snippet || '',
share_url: item.url || '',
cloud_type: detectCloudType(item.url || ''),
file_size: '',
source: item.source || '',
datetime: item.datetime || '',
cover: Array.isArray(item.images) && item.images.length > 0 ? item.images[0] : '',
password: item.password || '',
}));
// Filter out expired/invalid links
const expiredCount = formatted.filter(r => r.share_url && isExpiredShareLink(r.share_url)).length;
formatted = formatted.filter(r => !r.share_url || !isExpiredShareLink(r.share_url));
// Filter by enabled cloud types (admin-configurable per-type toggle)
const enabledSet = getEnabledCloudTypeSet();
formatted = formatted.filter(r => !r.cloud_type || enabledSet.has(r.cloud_type));
// Return results immediately without blocking validation
const channels = groupResultsByChannel(formatted, (item: any) =>
detectCloudType(item.url || '')
);
res.json({
results: formatted,
channels,
total: formatted.length,
filtered: expiredCount,
link_validation: false,
});
} catch (err: any) {
console.error('[Search] Error:', err);
res.status(500).json({ error: err.message || 'Internal server error' });
}
});
/**
* Load title filter rules from DB and apply to a title.
*/
function filterTitle(title: string): string {
const rules = getSystemConfig('title_filter_rules') || '';
return applyTitleFilter(title, rules);
}
// detectCloudType is imported from config/cloud-labels
// 检测失效的分享链接(支持多种模式)
function isExpiredShareLink(url: string): boolean {
if (!url) return false;
// 空链接/纯片段(无实际链接内容)
if (url.startsWith('#') || url.length < 10) return true;
// PanSou 有时返回残缺链接如 "/s/xxx" 或只有 "#/list/share"
if (url.startsWith('/') && !url.startsWith('//') && !url.startsWith('http')) return true;
// 夸克链接格式校验
if (url.includes('pan.quark.cn')) {
const baseUrl = url.split('#')[0]; // 去掉 hash 路由片段
// 有效格式必须是 pan.quark.cn/s/xxxxxx
if (!/pan\.quark\.cn\/s\/[a-zA-Z0-9]+/.test(baseUrl)) return true;
}
// 百度网盘常见失效格式
if (url.includes('pan.baidu.com') && /share\/init\?surl=$/.test(url)) return true;
// 阿里云盘失效格式(短到异常的链接)
if (url.includes('aliyundrive.com') && url.length < 30) return true;
return false;
}
/**
* Group search results into channels by cloud type.
* Each channel: { cloud_type, label, color, count, items }
*/
function groupResultsByChannel(results: any[], getCloudType?: (item: any) => string): any[] {
const groups: Record<string, any[]> = {};
const order: Record<string, number> = {
quark: 1, baidu: 2, aliyun: 3, '115': 4,
tianyi: 5, '123pan': 6, uc: 7, xunlei: 8,
pikpak: 9, magnet: 10, ed2k: 11, others: 12,
};
for (const item of results) {
const ct = getCloudType ? getCloudType(item) : (item.source || detectCloudType(item.url || '') || 'others');
if (!groups[ct]) groups[ct] = [];
groups[ct].push(item);
}
return Object.entries(groups)
.sort((a, b) => (order[a[0]] ?? 99) - (order[b[0]] ?? 99))
.map(([cloud_type, items]) => ({
cloud_type,
label: (CLOUD_LABELS as any)[cloud_type] || cloud_type,
color: (CLOUD_COLORS as any)[cloud_type] || '#95a5a6',
count: items.length,
items,
}));
}
// ============ Video ============
// ============ Video ============
/**
* POST /api/video/parse
* Parse a video URL
*/
router.post('/video/parse', async (req: Request, res: Response) => {
try {
const { url } = req.body;
if (!url) {
res.status(400).json({ error: 'URL is required' });
return;
}
const videoInfo = await parseVideo(url);
res.json(videoInfo);
} catch (err: any) {
console.error('[Video] Parse error:', err);
res.status(500).json({ error: err.message || 'Failed to parse video' });
}
});
// ============ Cloud Save ============
// ============ Cloud Save ============
/**
* POST /api/save
* Save a share link to a specific cloud
*/
router.post('/save', saveLimiter, async (req: Request, res: Response) => {
try {
// Support both formats:
// 1. Backend-style: { url, cloudType }
// 2. Frontend-style: { source: { share_url }, target_cloud }
const url = req.body.url || req.body.source?.share_url || req.body.source?.url;
const cloudType = req.body.cloudType || req.body.target_cloud || (req.body.source as any)?.cloud_type;
const sourceTitle = req.body.source_title || req.body.source?.title || req.body.title;
if (!url || !cloudType) {
res.status(400).json({ error: 'URL and cloudType/cloud_type are required' });
return;
}
const ip = req.ip || (req.headers['x-forwarded-for'] as string)?.split(',')[0]?.trim() || '';
const result = await saveFromShare(url, cloudType, sourceTitle, ip);
res.json(result);
} catch (err: any) {
console.error('[Save] Error:', err);
res.status(500).json({ error: err.message || 'Failed to save to cloud' });
}
});
/**
* POST /api/video/save-to-cloud
* Save a video to cloud
*/
router.post('/video/save-to-cloud', saveLimiter, async (req: Request, res: Response) => {
try {
const { videoUrl, cloudType, title } = req.body;
if (!videoUrl || !cloudType) {
res.status(400).json({ error: 'videoUrl and cloudType are required' });
return;
}
const ip = req.ip || (req.headers['x-forwarded-for'] as string)?.split(',')[0]?.trim() || '';
const result = await saveFromShare(videoUrl, cloudType, title, ip);
res.json(result);
} catch (err: any) {
console.error('[Video] Save-to-cloud error:', err);
res.status(500).json({ error: err.message || 'Failed to save video to cloud' });
}
});
// ============ Rankings ============
// ============ Rankings ============
/**
* GET /api/rankings
* Get search rankings
*/
router.get('/rankings', async (_req: Request, res: Response) => {
try {
const rankings = await getRankings();
res.json(rankings);
} catch (err: any) {
console.error('[Rankings] Error:', err);
res.status(500).json({ error: err.message || 'Internal server error' });
}
});
/**
* GET /api/rankings/hot
* Get hot keywords
*/
router.get('/rankings/hot', async (_req: Request, res: Response) => {
try {
const keywords = await getHotKeywords();
res.json(keywords);
} catch (err: any) {
console.error('[Hot] Error:', err);
res.status(500).json({ error: err.message || 'Internal server error' });
}
});
/**
* GET /api/rankings/categorized
* Get categorized rankings (hot + newest per category), cached for 12h
*/
router.get('/rankings/categorized', async (_req: Request, res: Response) => {
try {
const data = await getCategorizedRankings();
res.json(data);
} catch (err: any) {
console.error('[Categorized] Error:', err);
res.status(500).json({ error: err.message || 'Internal server error' });
}
});
/**
* GET /api/site-config
* Public site configuration (no auth required).
*/
router.get('/site-config', (_req: Request, res: Response) => {
try {
const siteLogo = getSystemConfig('site_logo') || '';
const siteName = getSystemConfig('site_name') || 'CloudSearch';
const fallbackImage = getSystemConfig('search_fallback_image') || '';
const siteDisclaimer = getSystemConfig('site_disclaimer') || '';
const siteMarquee = getSystemConfig('site_marquee') || '';
res.json({ site_logo: siteLogo, site_name: siteName, search_fallback_image: fallbackImage, site_disclaimer: siteDisclaimer, site_marquee: siteMarquee });
} catch (err: any) {
res.status(500).json({ error: err.message || 'Internal server error' });
}
});
/**
* GET /api/me
* Get current user info from token (public, no auth middleware).
*/
router.get('/me', (req: Request, res: Response) => {
try {
const authHeader = req.headers.authorization;
if (!authHeader || !authHeader.startsWith('Bearer ')) {
res.json({ loggedIn: false });
return;
}
const token = authHeader.split(' ')[1];
const payload = verifyToken(token);
if (!payload) {
res.json({ loggedIn: false });
return;
}
res.json({ loggedIn: true, id: payload.id, username: payload.username });
} catch (err: any) {
res.json({ loggedIn: false });
}
});
// ============ Admin ============
/**
* Extract genre tags from search result titles.
*/
function extractTagsFromResults(results: any[], keyword: string): string[] {
const tags: string[] = [];
if (keyword) tags.push(keyword);
const genreKeywords: Record<string, string> = {
'动画': '动画', '动漫': '动画', '国漫': '国漫',
'剧场版': '剧场版', '年番': '年番',
'动作': '动作', '奇幻': '奇幻', '玄幻': '玄幻',
'仙侠': '仙侠', '古装': '古装', '爱情': '爱情',
'科幻': '科幻', '喜剧': '喜剧', '悬疑': '悬疑',
'恐怖': '恐怖', '惊悚': '惊悚', '剧情': '剧情',
'冒险': '冒险', '战争': '战争', '武侠': '武侠',
'纪录': '纪录片', '真人': '真人秀', '短片': '短片',
};
const seen = new Set<string>();
for (const r of results) {
const title = (r.title || r.note || '') as string;
for (const [key, val] of Object.entries(genreKeywords)) {
if (title.includes(key) && !seen.has(val)) {
seen.add(val);
tags.push(val);
}
}
}
return tags;
}
/**
* Query DB for previously saved resources that match the keyword.
* Returns formatted results for immediate streaming before external API call.
*/
function getSavedResources(keyword: string): any[] {
try {
const db = getDb();
const rows = db.prepare(`
SELECT source_url, source_title, target_cloud, share_url, created_at
FROM save_records
WHERE status = 'success'
AND (source_title LIKE ? OR source_url LIKE ?)
ORDER BY created_at DESC
LIMIT 20
`).all(`%${keyword}%`, `%${keyword}%`) as any[];
return rows.map((row: any, idx: number) => ({
id: `saved_${idx}`,
title: row.source_title || row.source_url || '',
description: '',
share_url: row.share_url || row.source_url || '',
cloud_type: detectCloudType(row.share_url || row.source_url || ''),
file_size: '',
update_time: row.created_at || '',
source: 'local',
file_id: '',
cover: '',
password: '',
}));
} catch (err) {
console.error('[SavedResources] DB query error:', err);
return [];
}
}
export default router;