chore: initial commit - CloudSearch v0.0.2
This commit is contained in:
351
packages/backend/src/search/rankings.service.ts
Executable file
351
packages/backend/src/search/rankings.service.ts
Executable file
@@ -0,0 +1,351 @@
|
||||
// Native fetch available in Node 20+
|
||||
import { getDb } from '../database/database';
|
||||
import { getTimezone, formatLocalDateTime } from '../utils/time';
|
||||
|
||||
export interface RankingItem {
|
||||
keyword: string;
|
||||
searchCount: number;
|
||||
updatedAt: string;
|
||||
rating?: number;
|
||||
}
|
||||
|
||||
export interface CategorizedRanking {
|
||||
category: string;
|
||||
label: string;
|
||||
hot: RankingItem[];
|
||||
newest: RankingItem[];
|
||||
}
|
||||
|
||||
export interface CategorizedResponse {
|
||||
fetchedAt: string;
|
||||
categories: CategorizedRanking[];
|
||||
}
|
||||
|
||||
// ===== Bilibili PGC 排行榜配置 =====
|
||||
interface BiliPgcDef {
|
||||
category: string;
|
||||
label: string;
|
||||
season_type: number; // 1=番剧, 2=电影, 3=纪录片, 4=国创, 5=电视剧, 7=综艺
|
||||
}
|
||||
|
||||
const BILI_PGC_CATEGORIES: BiliPgcDef[] = [
|
||||
// 国创:凡人修仙传、灵笼、斗破苍穹等官方国产动画
|
||||
{ category: 'donghua', label: '国产动漫', season_type: 4 },
|
||||
// 番剧:日漫等全球动画
|
||||
{ category: 'global_anime', label: '热门动漫', season_type: 1 },
|
||||
];
|
||||
|
||||
// ===== 百度热搜榜配置 =====
|
||||
interface BaiduBoardDef {
|
||||
category: string;
|
||||
label: string;
|
||||
tab: string; // movie=电影热搜, teleplay=电视剧热搜
|
||||
}
|
||||
|
||||
const BAIDU_BOARDS: BaiduBoardDef[] = [
|
||||
// 百度电影热搜:实时反映国内电影热度
|
||||
{ category: 'movie', label: '国内电影', tab: 'movie' },
|
||||
// 百度电视剧热搜:国内剧集热度
|
||||
{ category: 'tv', label: '热门剧集', tab: 'teleplay' },
|
||||
];
|
||||
|
||||
// ===== TMDB 分类配置(保留欧美和冷门内容)=====
|
||||
interface TmdbCategoryDef {
|
||||
category: string;
|
||||
label: string;
|
||||
hotUrl: string;
|
||||
newestUrl: string;
|
||||
}
|
||||
|
||||
const TMDB_CATEGORIES: TmdbCategoryDef[] = [
|
||||
{
|
||||
category: 'western_movie', label: '欧美电影',
|
||||
hotUrl: 'https://api.themoviedb.org/3/discover/movie?with_origin_country=US&sort_by=vote_average.desc&vote_count.gte=10',
|
||||
newestUrl: 'https://api.themoviedb.org/3/discover/movie?with_origin_country=US&sort_by=release_date.desc&vote_count.gte=1',
|
||||
},
|
||||
{
|
||||
category: 'western_tv', label: '欧美剧集',
|
||||
hotUrl: 'https://api.themoviedb.org/3/discover/tv?with_origin_country=US&sort_by=vote_average.desc&vote_count.gte=10',
|
||||
newestUrl: 'https://api.themoviedb.org/3/discover/tv?with_origin_country=US&sort_by=first_air_date.desc&vote_count.gte=10',
|
||||
},
|
||||
{
|
||||
category: 'niche', label: '冷门佳片',
|
||||
hotUrl: 'https://api.themoviedb.org/3/discover/movie?sort_by=vote_average.desc&vote_count.gte=10&vote_count.lte=500',
|
||||
newestUrl: 'https://api.themoviedb.org/3/discover/movie?sort_by=release_date.desc&vote_count.gte=1&vote_count.lte=500',
|
||||
},
|
||||
];
|
||||
|
||||
// ===== 显示顺序 =====
|
||||
const CATEGORY_ORDER: Record<string, number> = {
|
||||
donghua: 1,
|
||||
movie: 2,
|
||||
tv: 3,
|
||||
global_anime: 4,
|
||||
western_movie: 5,
|
||||
western_tv: 6,
|
||||
niche: 7,
|
||||
hotsite: 8,
|
||||
};
|
||||
|
||||
// ===== 12小时缓存 =====
|
||||
let cache: { data: CategorizedResponse; time: number } | null = null;
|
||||
const CACHE_TTL = 12 * 60 * 60 * 1000;
|
||||
|
||||
function isCacheValid(): boolean {
|
||||
return cache !== null && (Date.now() - cache.time) < CACHE_TTL;
|
||||
}
|
||||
|
||||
// ===== Bilibili PGC API =====
|
||||
|
||||
/**
|
||||
* 抓取 Bilibili PGC 排行榜(番剧/国创)
|
||||
*/
|
||||
async function fetchFromBiliPgc(season_type: number): Promise<RankingItem[]> {
|
||||
try {
|
||||
const url = `https://api.bilibili.com/pgc/web/rank/list?season_type=${season_type}&day=7`;
|
||||
const resp = await fetch(url, {
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
'Referer': 'https://www.bilibili.com/',
|
||||
'Accept': 'application/json, text/plain, */*',
|
||||
'Accept-Language': 'zh-CN,zh;q=0.9',
|
||||
},
|
||||
signal: AbortSignal.timeout(8000),
|
||||
});
|
||||
if (!resp.ok) {
|
||||
console.error(`[BiliPGC] HTTP ${resp.status} for season_type=${season_type}`);
|
||||
return [];
|
||||
}
|
||||
const json = await resp.json() as any;
|
||||
if (json.code !== 0 || !json.result?.list) {
|
||||
console.error(`[BiliPGC] API error code=${json.code} for season_type=${season_type}`);
|
||||
return [];
|
||||
}
|
||||
return json.result.list.slice(0, 20).map((item: any) => {
|
||||
const stat = item.stat || {};
|
||||
const viewCount = stat.view || 0;
|
||||
const followCount = stat.follow || 0;
|
||||
const searchCount = viewCount > 0 ? viewCount : followCount;
|
||||
|
||||
let rating = 0;
|
||||
if (item.rating) {
|
||||
const m = String(item.rating).match(/([\d.]+)/);
|
||||
if (m) rating = parseFloat(m[1]);
|
||||
}
|
||||
|
||||
return {
|
||||
keyword: item.title || '',
|
||||
searchCount,
|
||||
updatedAt: item.new_ep?.index_show || item.new_ep?.cover || '',
|
||||
rating,
|
||||
};
|
||||
});
|
||||
} catch (err) {
|
||||
console.error(`[BiliPGC] Fetch error for season_type=${season_type}:`, (err as Error).message);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
// ===== 百度热搜榜 API =====
|
||||
|
||||
/**
|
||||
* 抓取百度热搜榜
|
||||
* tab: movie=电影, teleplay=电视剧
|
||||
*/
|
||||
async function fetchFromBaidu(tab: string): Promise<RankingItem[]> {
|
||||
try {
|
||||
const url = `https://top.baidu.com/api/board?tab=${tab}`;
|
||||
const resp = await fetch(url, {
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
'Referer': 'https://top.baidu.com/board',
|
||||
},
|
||||
signal: AbortSignal.timeout(8000),
|
||||
});
|
||||
if (!resp.ok) {
|
||||
console.error(`[Baidu] HTTP ${resp.status} for tab=${tab}`);
|
||||
return [];
|
||||
}
|
||||
const json = await resp.json() as any;
|
||||
if (!json.success || !json.data?.cards) {
|
||||
console.error(`[Baidu] API error for tab=${tab}`);
|
||||
return [];
|
||||
}
|
||||
|
||||
const results: RankingItem[] = [];
|
||||
for (const card of json.data.cards) {
|
||||
for (const item of (card.content || [])) {
|
||||
results.push({
|
||||
keyword: item.word || '',
|
||||
// hotScore can be like "96438", parse as number
|
||||
searchCount: parseInt(item.hotScore || '0', 10) || 0,
|
||||
updatedAt: item.desc || '',
|
||||
rating: 0,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return results.slice(0, 20);
|
||||
} catch (err) {
|
||||
console.error(`[Baidu] Fetch error for tab=${tab}:`, (err as Error).message);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
// ===== TMDB =====
|
||||
|
||||
function getTmdbToken(): string {
|
||||
const db = getDb();
|
||||
return (db.prepare('SELECT value FROM system_configs WHERE key = ?').get('tmdb_api_token') as any)?.value || '';
|
||||
}
|
||||
|
||||
function tmdbResultToRanking(item: any): RankingItem {
|
||||
const title = item.title || item.name || '';
|
||||
const date = item.release_date || item.first_air_date || '';
|
||||
const rating = item.vote_average ? Math.round(item.vote_average * 10) / 10 : 0;
|
||||
return {
|
||||
keyword: title,
|
||||
searchCount: item.vote_count || 0,
|
||||
updatedAt: date,
|
||||
rating,
|
||||
};
|
||||
}
|
||||
|
||||
async function tmdbFetch(url: string, token: string): Promise<any[]> {
|
||||
const fullUrl = `${url}${url.includes('?') ? '&' : '?'}language=zh-CN`;
|
||||
try {
|
||||
const resp = await fetch(fullUrl, {
|
||||
headers: { 'Authorization': `Bearer ${token}` },
|
||||
signal: AbortSignal.timeout(10000),
|
||||
});
|
||||
if (!resp.ok) {
|
||||
console.error(`[TMDB] HTTP ${resp.status} for ${url}`);
|
||||
return [];
|
||||
}
|
||||
const data = await resp.json() as any;
|
||||
return (data.results || []).slice(0, 20);
|
||||
} catch (err) {
|
||||
console.error(`[TMDB] Fetch error for ${url}:`, err);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
// ===== 主流程 =====
|
||||
|
||||
async function fetchRankings(): Promise<CategorizedResponse> {
|
||||
const fetchedAt = formatLocalDateTime();
|
||||
|
||||
// 1. 并行抓取 Bilibili PGC 数据(国漫、番剧)
|
||||
const biliPromises = BILI_PGC_CATEGORIES.map(async (cat) => {
|
||||
const results = await fetchFromBiliPgc(cat.season_type);
|
||||
const mid = Math.ceil(results.length / 2);
|
||||
return {
|
||||
category: cat.category,
|
||||
label: cat.label,
|
||||
hot: results.slice(0, mid),
|
||||
newest: results.slice(mid),
|
||||
};
|
||||
});
|
||||
|
||||
// 2. 并行抓取百度热搜数据(电影、电视剧)
|
||||
// 百度只有热榜没有最新榜,全部放 hot
|
||||
const baiduPromises = BAIDU_BOARDS.map(async (board) => {
|
||||
const results = await fetchFromBaidu(board.tab);
|
||||
return {
|
||||
category: board.category,
|
||||
label: board.label,
|
||||
hot: results,
|
||||
newest: [],
|
||||
};
|
||||
});
|
||||
|
||||
// 3. 并行抓取 TMDB 数据(欧美观影、剧集、冷门)
|
||||
const token = getTmdbToken();
|
||||
let tmdbResults: CategorizedRanking[] = [];
|
||||
if (token) {
|
||||
const tmdbPromises = TMDB_CATEGORIES.map(async (cat) => {
|
||||
const [hotResults, newestResults] = await Promise.all([
|
||||
tmdbFetch(cat.hotUrl, token),
|
||||
tmdbFetch(cat.newestUrl, token),
|
||||
]);
|
||||
return {
|
||||
category: cat.category,
|
||||
label: cat.label,
|
||||
hot: hotResults.map(tmdbResultToRanking),
|
||||
newest: newestResults.map(tmdbResultToRanking),
|
||||
};
|
||||
});
|
||||
tmdbResults = await Promise.all(tmdbPromises);
|
||||
}
|
||||
|
||||
// 4. 本站热搜
|
||||
const db = getDb();
|
||||
const rows = db.prepare(
|
||||
'SELECT keyword, search_count as searchCount, updated_at as updatedAt FROM hot_keywords ORDER BY search_count DESC LIMIT 20'
|
||||
).all() as RankingItem[];
|
||||
const newestRows = db.prepare(
|
||||
'SELECT keyword, search_count as searchCount, updated_at as updatedAt FROM hot_keywords ORDER BY updated_at DESC LIMIT 20'
|
||||
).all() as RankingItem[];
|
||||
|
||||
const hotsiteCategory: CategorizedRanking = {
|
||||
category: 'hotsite',
|
||||
label: '本站热搜',
|
||||
hot: rows,
|
||||
newest: newestRows,
|
||||
};
|
||||
|
||||
// 5. 合并所有结果
|
||||
const [biliResults, baiduResults] = await Promise.all([
|
||||
Promise.all(biliPromises),
|
||||
Promise.all(baiduPromises),
|
||||
]);
|
||||
const allCategories = [...biliResults, ...baiduResults, ...tmdbResults, hotsiteCategory];
|
||||
|
||||
// 按 CATEGORY_ORDER 排序
|
||||
allCategories.sort((a, b) => (CATEGORY_ORDER[a.category] || 99) - (CATEGORY_ORDER[b.category] || 99));
|
||||
|
||||
return { fetchedAt, categories: allCategories };
|
||||
}
|
||||
|
||||
export async function getCategorizedRankings(): Promise<CategorizedResponse> {
|
||||
if (isCacheValid()) {
|
||||
return cache!.data;
|
||||
}
|
||||
|
||||
try {
|
||||
const data = await fetchRankings();
|
||||
cache = { data, time: Date.now() };
|
||||
return data;
|
||||
} catch (err) {
|
||||
console.error('[Rankings] Fetch error:', err);
|
||||
if (cache) return cache.data;
|
||||
const db = getDb();
|
||||
const rows = db.prepare(
|
||||
'SELECT keyword, search_count as searchCount, updated_at as updatedAt FROM hot_keywords ORDER BY search_count DESC LIMIT 20'
|
||||
).all() as RankingItem[];
|
||||
return {
|
||||
fetchedAt: formatLocalDateTime(),
|
||||
categories: [{
|
||||
category: 'hotsite', label: '本站热搜',
|
||||
hot: rows,
|
||||
newest: [...rows].sort((a, b) => b.updatedAt.localeCompare(a.updatedAt)).slice(0, 20),
|
||||
}],
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export async function getRankings(): Promise<RankingItem[]> {
|
||||
const db = getDb();
|
||||
const rows = db.prepare(
|
||||
'SELECT keyword, search_count as searchCount, updated_at as updatedAt FROM hot_keywords ORDER BY search_count DESC LIMIT 20'
|
||||
).all() as RankingItem[];
|
||||
return rows;
|
||||
}
|
||||
|
||||
export async function getHotKeywords(): Promise<string[]> {
|
||||
const db = getDb();
|
||||
const rows = db.prepare(
|
||||
'SELECT keyword FROM hot_keywords ORDER BY search_count DESC LIMIT 20'
|
||||
).all() as { keyword: string }[];
|
||||
return rows.map(r => r.keyword);
|
||||
}
|
||||
125
packages/backend/src/search/search-optimizer.ts
Executable file
125
packages/backend/src/search/search-optimizer.ts
Executable file
@@ -0,0 +1,125 @@
|
||||
/**
|
||||
* Search Results Optimizer
|
||||
*
|
||||
* For each cloud type, keep only the top N most relevant results.
|
||||
* Order groups by priority: real cloud storage > other providers > magnet/others.
|
||||
*
|
||||
* Goal: give users a manageable, high-quality result set instead of overwhelming them
|
||||
* with hundreds of results dominated by magnet links.
|
||||
*/
|
||||
|
||||
import { detectCloudType } from '../config/cloud-labels';
|
||||
|
||||
/** Minimal result shape the optimizer needs */
|
||||
interface OptimizableResult {
|
||||
title?: string;
|
||||
url?: string;
|
||||
source?: string;
|
||||
score?: number;
|
||||
[key: string]: any;
|
||||
}
|
||||
|
||||
/** Priority tiers for result ordering */
|
||||
const CLOUD_PRIORITY: Record<string, number> = {
|
||||
// Tier 1: Major cloud storage (most useful for save-to-cloud feature)
|
||||
baidu: 10,
|
||||
quark: 10,
|
||||
aliyun: 10,
|
||||
// Tier 2: Other cloud storage
|
||||
'115': 20,
|
||||
tianyi: 20,
|
||||
'123pan': 20,
|
||||
uc: 20,
|
||||
xunlei: 20,
|
||||
pikpak: 20,
|
||||
// Tier 3: Mobile/app links (not very useful)
|
||||
mobile: 50,
|
||||
// Tier 4: Direct links (lowest utility for cloud saving)
|
||||
magnet: 100,
|
||||
ed2k: 100,
|
||||
others: 100,
|
||||
};
|
||||
|
||||
const DEFAULT_PRIORITY = 50;
|
||||
|
||||
/** Get cloud type for a result, with an extra check for tracker URLs */
|
||||
function getCloudType(result: OptimizableResult): string {
|
||||
const url = result.url;
|
||||
// Check for tracker/private-site URLs not covered by shared detection
|
||||
if (url && /mteam|hdarea|hdsky/i.test(url)) return 'others';
|
||||
return detectCloudType(url);
|
||||
}
|
||||
|
||||
function getPriority(cloudType: string): number {
|
||||
return CLOUD_PRIORITY[cloudType] ?? DEFAULT_PRIORITY;
|
||||
}
|
||||
|
||||
export interface OptimizationResult {
|
||||
results: OptimizableResult[];
|
||||
/** Per-type stats for display */
|
||||
perType: Array<{ type: string; count: number; total: number }>;
|
||||
/** How many items were kept vs filtered */
|
||||
keptCount: number;
|
||||
filteredCount: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Optimize search results:
|
||||
* 1. Group by cloud type
|
||||
* 2. Sort by score descending within each group
|
||||
* 3. Keep only top `maxPerType` results per type
|
||||
* 4. Order groups by priority (cloud storage first)
|
||||
*/
|
||||
export function optimizeSearchResults(
|
||||
items: OptimizableResult[],
|
||||
maxPerType: number = 20
|
||||
): OptimizationResult {
|
||||
// Step 1: Group by cloud type
|
||||
const grouped: Record<string, OptimizableResult[]> = {};
|
||||
const typeTotals: Record<string, number> = {};
|
||||
|
||||
for (const item of items) {
|
||||
const ct = getCloudType(item);
|
||||
if (!grouped[ct]) {
|
||||
grouped[ct] = [];
|
||||
}
|
||||
grouped[ct].push(item);
|
||||
typeTotals[ct] = (typeTotals[ct] || 0) + 1;
|
||||
}
|
||||
|
||||
// Step 2 & 3: Sort each group by score desc, take top N
|
||||
const kept: OptimizableResult[] = [];
|
||||
const perType: Array<{ type: string; count: number; total: number }> = [];
|
||||
|
||||
for (const [ct, groupItems] of Object.entries(grouped)) {
|
||||
// Sort by score descending (higher score = more relevant)
|
||||
groupItems.sort((a, b) => (b.score || 0) - (a.score || 0));
|
||||
|
||||
const top = groupItems.slice(0, maxPerType);
|
||||
kept.push(...top);
|
||||
|
||||
perType.push({
|
||||
type: ct,
|
||||
count: top.length,
|
||||
total: typeTotals[ct],
|
||||
});
|
||||
}
|
||||
|
||||
// Step 4: Sort kept results by cloud priority, then by score within same priority
|
||||
kept.sort((a, b) => {
|
||||
const pa = getPriority(getCloudType(a));
|
||||
const pb = getPriority(getCloudType(b));
|
||||
if (pa !== pb) return pa - pb;
|
||||
return (b.score || 0) - (a.score || 0);
|
||||
});
|
||||
|
||||
const keptCount = kept.length;
|
||||
const filteredCount = items.length - keptCount;
|
||||
|
||||
return {
|
||||
results: kept,
|
||||
perType,
|
||||
keptCount,
|
||||
filteredCount,
|
||||
};
|
||||
}
|
||||
348
packages/backend/src/search/search.service.ts
Executable file
348
packages/backend/src/search/search.service.ts
Executable file
@@ -0,0 +1,348 @@
|
||||
// Native fetch available in Node 20+
|
||||
import config from '../config';
|
||||
import { getDb } from '../database/database';
|
||||
import { localTimestamp } from '../utils/time';
|
||||
import { proxiedFetch } from '../utils/proxy-agent';
|
||||
|
||||
export interface SearchResult {
|
||||
title: string;
|
||||
url: string;
|
||||
content: string;
|
||||
score?: number;
|
||||
source?: string;
|
||||
password?: string;
|
||||
datetime?: string;
|
||||
responseTimeMs?: number;
|
||||
}
|
||||
|
||||
export interface SearchResponse {
|
||||
results: SearchResult[];
|
||||
total: number;
|
||||
page: number;
|
||||
pageSize: number;
|
||||
}
|
||||
|
||||
export interface ApiSearchSource {
|
||||
name: string;
|
||||
url: string;
|
||||
method?: string; // GET | POST (default POST)
|
||||
headers?: Record<string, string>;
|
||||
body?: string; // JSON body template, supports {keyword} {page}
|
||||
resultPath: string; // dot-notation path to results array (e.g. "data.list")
|
||||
fieldMap: { // maps SearchResult fields to JSON response fields
|
||||
title?: string;
|
||||
url?: string;
|
||||
content?: string;
|
||||
password?: string;
|
||||
datetime?: string;
|
||||
};
|
||||
timeout?: number; // per-source timeout (ms), default 10000
|
||||
}
|
||||
|
||||
/** Simple dot/bracket notation JSON path accessor. */
|
||||
function jsonPathGet(obj: any, path: string): any {
|
||||
if (!obj || !path) return undefined;
|
||||
const parts = path
|
||||
.replace(/\[(\d+)\]/g, '.$1') // items[0] → items.0
|
||||
.split('.')
|
||||
.filter(Boolean);
|
||||
let current = obj;
|
||||
for (const part of parts) {
|
||||
if (current == null) return undefined;
|
||||
current = current[part];
|
||||
}
|
||||
return current;
|
||||
}
|
||||
|
||||
/** Parse configured API search sources from system config. */
|
||||
function getApiSearchSources(): ApiSearchSource[] {
|
||||
try {
|
||||
const db = getDb();
|
||||
const raw = (db.prepare("SELECT value FROM system_configs WHERE key = 'api_search_sources'").get() as any)?.value;
|
||||
if (!raw) return [];
|
||||
const parsed = JSON.parse(raw);
|
||||
if (!Array.isArray(parsed)) return [];
|
||||
return parsed.filter((s: any) => s.url && s.resultPath);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/** Query a single API search source and return results with timing. */
|
||||
async function queryApiSource(
|
||||
source: ApiSearchSource,
|
||||
keyword: string,
|
||||
page: number,
|
||||
proxyUrl?: string,
|
||||
): Promise<{ source: string; results: SearchResult[]; responseTimeMs: number; error?: string }> {
|
||||
const startTime = Date.now();
|
||||
const timeout = source.timeout || 10000;
|
||||
const method = (source.method || 'POST').toUpperCase();
|
||||
|
||||
try {
|
||||
let url = source.url;
|
||||
const headers: Record<string, string> = { ...source.headers };
|
||||
|
||||
let body: string | undefined;
|
||||
if (source.body) {
|
||||
body = source.body
|
||||
.replace(/\{keyword\}/g, encodeURIComponent(keyword))
|
||||
.replace(/\{page\}/g, String(page));
|
||||
}
|
||||
|
||||
// For GET requests, append query params; for POST, use body
|
||||
const fetchOptions: RequestInit = {
|
||||
method,
|
||||
headers: { 'Content-Type': 'application/json', ...headers },
|
||||
signal: AbortSignal.timeout(timeout),
|
||||
};
|
||||
|
||||
// For GET requests, append query params; for POST, use body
|
||||
if (method === 'GET') {
|
||||
// Parse body as JSON and convert to query string
|
||||
if (body) {
|
||||
try {
|
||||
const params = JSON.parse(body);
|
||||
const qs = new URLSearchParams(params).toString();
|
||||
url += (url.includes('?') ? '&' : '?') + qs;
|
||||
} catch {
|
||||
// If body is not JSON, append as raw query
|
||||
url += (url.includes('?') ? '&' : '?') + body;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
(fetchOptions as any).body = body || JSON.stringify({ keyword, page });
|
||||
}
|
||||
|
||||
const response = await proxiedFetch(url, fetchOptions, proxyUrl);
|
||||
const responseTimeMs = Date.now() - startTime;
|
||||
|
||||
if (!response.ok) {
|
||||
return { source: source.name, results: [], responseTimeMs, error: `HTTP ${response.status}` };
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
const resultTimeMs = Date.now() - startTime;
|
||||
|
||||
// Extract results array using JSONPath
|
||||
const items = jsonPathGet(data, source.resultPath);
|
||||
if (!Array.isArray(items)) {
|
||||
return { source: source.name, results: [], responseTimeMs: resultTimeMs, error: 'resultPath not found or not an array' };
|
||||
}
|
||||
|
||||
// Map fields
|
||||
const fm = source.fieldMap || {};
|
||||
const results: SearchResult[] = items.map((item: any) => ({
|
||||
title: (fm.title ? item[fm.title] : item.title) || item.name || '',
|
||||
url: (fm.url ? item[fm.url] : item.url) || item.link || '',
|
||||
content: (fm.content ? item[fm.content] : item.content) || item.snippet || '',
|
||||
password: (fm.password ? item[fm.password] : item.password) || '',
|
||||
datetime: (fm.datetime ? item[fm.datetime] : item.datetime) || item.date || '',
|
||||
source: source.name,
|
||||
responseTimeMs: resultTimeMs,
|
||||
}));
|
||||
|
||||
return { source: source.name, results, responseTimeMs: resultTimeMs };
|
||||
} catch (err: any) {
|
||||
const responseTimeMs = Date.now() - startTime;
|
||||
return { source: source.name, results: [], responseTimeMs, error: err.message };
|
||||
}
|
||||
}
|
||||
|
||||
/** Query all configured API search sources in parallel. */
|
||||
async function searchApiSources(keyword: string, page: number, proxyUrl?: string): Promise<{
|
||||
results: SearchResult[];
|
||||
sourceStats: { name: string; count: number; responseTimeMs: number; error?: string }[];
|
||||
}> {
|
||||
const sources = getApiSearchSources();
|
||||
if (sources.length === 0) return { results: [], sourceStats: [] };
|
||||
|
||||
const promises = sources.map(s => queryApiSource(s, keyword, page, proxyUrl));
|
||||
const allResults = await Promise.all(promises);
|
||||
|
||||
const sourceStats = allResults.map(r => ({
|
||||
name: r.source,
|
||||
count: r.results.length,
|
||||
responseTimeMs: r.responseTimeMs,
|
||||
error: r.error,
|
||||
}));
|
||||
|
||||
// Merge all results, tag with source name, sort by response time (fastest first)
|
||||
const results = allResults
|
||||
.flatMap(r => r.results)
|
||||
.sort((a, b) => (a.responseTimeMs || 99999) - (b.responseTimeMs || 99999));
|
||||
|
||||
return { results, sourceStats };
|
||||
}
|
||||
|
||||
export async function search(keyword: string, page: number = 1, ip?: string): Promise<SearchResponse> {
|
||||
const db = getDb();
|
||||
const pansouUrl = (db.prepare('SELECT value FROM system_configs WHERE key = ?').get('pansou_url') as any)?.value || config.pansouUrl;
|
||||
const proxyEnabled = (db.prepare('SELECT value FROM system_configs WHERE key = ?').get('search_proxy_enabled') as any)?.value === 'true';
|
||||
const proxyUrl = (db.prepare('SELECT value FROM system_configs WHERE key = ?').get('search_proxy_url') as any)?.value || '';
|
||||
const effectiveProxy = proxyEnabled ? proxyUrl : undefined;
|
||||
|
||||
// ── Run PanSou and API sources in parallel ──
|
||||
const pansouPromise = (async () => {
|
||||
const url = `${pansouUrl}/api/search`;
|
||||
const fetchOptions: RequestInit = {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ kw: keyword, page }),
|
||||
signal: AbortSignal.timeout(10000),
|
||||
};
|
||||
const pansouStart = Date.now();
|
||||
const response = await proxiedFetch(url, fetchOptions, effectiveProxy);
|
||||
if (!response.ok) throw new Error(`PanSou API error: ${response.status}`);
|
||||
const data = await response.json() as any;
|
||||
return { data, responseTimeMs: Date.now() - pansouStart };
|
||||
})();
|
||||
|
||||
const apiSourcesPromise = searchApiSources(keyword, page, effectiveProxy);
|
||||
|
||||
const [pansouResult, apiSourcesResult] = await Promise.all([pansouPromise, apiSourcesPromise]);
|
||||
|
||||
const { data, responseTimeMs: pansouTime } = pansouResult;
|
||||
|
||||
// ── Parse PanSou results ──
|
||||
let items: any[] = [];
|
||||
let total = 0;
|
||||
|
||||
if (data.data?.merged_by_type) {
|
||||
for (const [cloudType, cloudItems] of Object.entries(data.data.merged_by_type)) {
|
||||
if (Array.isArray(cloudItems)) {
|
||||
items.push(...cloudItems.map((item: any) => ({
|
||||
...item,
|
||||
_cloud_type: cloudType,
|
||||
})));
|
||||
}
|
||||
}
|
||||
total = data.data.total || items.length;
|
||||
} else if (Array.isArray(data.data)) {
|
||||
items = data.data;
|
||||
total = data.total || items.length;
|
||||
} else if (Array.isArray(data.results)) {
|
||||
items = data.results;
|
||||
total = data.total || items.length;
|
||||
}
|
||||
|
||||
const pansouResults: SearchResult[] = items.map((item: any) => ({
|
||||
title: item.note || item.title || '',
|
||||
url: item.url || item.link || '',
|
||||
content: item.content || item.snippet || item.note || '',
|
||||
score: item.score || 0,
|
||||
source: item.source || item._cloud_type || 'pansou',
|
||||
password: item.password || '',
|
||||
datetime: item.datetime || '',
|
||||
responseTimeMs: pansouTime,
|
||||
images: item.images || [],
|
||||
}));
|
||||
|
||||
// ── Merge PanSou + API sources, sort by response time (fastest first) ──
|
||||
const allResults = [...apiSourcesResult.results, ...pansouResults]
|
||||
.sort((a, b) => (a.responseTimeMs || 99999) - (b.responseTimeMs || 99999));
|
||||
|
||||
// Deduplicate by URL within merged results
|
||||
const seenUrls = new Set<string>();
|
||||
const results: SearchResult[] = [];
|
||||
for (const r of allResults) {
|
||||
if (r.url && !seenUrls.has(r.url)) {
|
||||
seenUrls.add(r.url);
|
||||
results.push(r);
|
||||
} else if (!r.url) {
|
||||
results.push(r); // keep results without URLs (unlikely but safe)
|
||||
}
|
||||
}
|
||||
|
||||
total = results.length;
|
||||
|
||||
// Sort by datetime descending as secondary sort (preserve response-time groups)
|
||||
results.sort((a: any, b: any) => {
|
||||
const ta = a.datetime || '';
|
||||
const tb = b.datetime || '';
|
||||
if (!ta && !tb) return 0;
|
||||
if (!ta) return 1;
|
||||
if (!tb) return -1;
|
||||
return tb.localeCompare(ta);
|
||||
});
|
||||
|
||||
// Record search statistics
|
||||
recordSearchStats(keyword, results.length, ip);
|
||||
|
||||
// Update hot keywords
|
||||
updateHotKeywords(keyword);
|
||||
|
||||
return {
|
||||
results,
|
||||
total,
|
||||
page,
|
||||
pageSize: data.pageSize || 10,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply title filter rules to clean up search result titles.
|
||||
* Rules format (one per line):
|
||||
* # comment lines are ignored (hash must be followed by space)
|
||||
* /pattern/flags → regex: matched content is deleted from title
|
||||
* plain text → literal text: exact text is deleted from title wherever it appears
|
||||
*/
|
||||
export function applyTitleFilter(title: string, rules: string): string {
|
||||
if (!title || !rules) return title;
|
||||
const lines = rules.split('\n');
|
||||
let result = title;
|
||||
for (const rawLine of lines) {
|
||||
const line = rawLine.trim();
|
||||
if (!line || line.startsWith('# ')) continue;
|
||||
try {
|
||||
if (line.startsWith('/') && line.lastIndexOf('/') > 0) {
|
||||
const lastSlashIdx = line.lastIndexOf('/');
|
||||
const pattern = line.substring(1, lastSlashIdx);
|
||||
const flags = line.substring(lastSlashIdx + 1);
|
||||
const anchored = pattern.startsWith('^') ? pattern : '^' + pattern;
|
||||
const re = new RegExp(anchored, flags);
|
||||
const match = re.exec(result);
|
||||
if (match && match.index === 0) {
|
||||
result = result.slice(match[0].length);
|
||||
}
|
||||
} else {
|
||||
if (result.startsWith(line)) {
|
||||
result = result.slice(line.length);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
return result.trim();
|
||||
}
|
||||
|
||||
function recordSearchStats(keyword: string, resultCount: number, ip?: string): void {
|
||||
try {
|
||||
const db = getDb();
|
||||
db.prepare(
|
||||
'INSERT INTO search_stats (keyword, intent, result_count, ip_address, created_at) VALUES (?, ?, ?, ?, ?)'
|
||||
).run(keyword, 'SEARCH', resultCount, ip || '', localTimestamp());
|
||||
} catch (err) {
|
||||
console.error('[Search] Failed to record stats:', err);
|
||||
}
|
||||
}
|
||||
|
||||
function updateHotKeywords(keyword: string): void {
|
||||
try {
|
||||
const db = getDb();
|
||||
const existing = db.prepare('SELECT id FROM hot_keywords WHERE keyword = ?').get(keyword) as any;
|
||||
|
||||
if (existing) {
|
||||
db.prepare(
|
||||
"UPDATE hot_keywords SET search_count = search_count + 1, updated_at = ? WHERE keyword = ?"
|
||||
).run(localTimestamp(), keyword);
|
||||
} else {
|
||||
db.prepare(
|
||||
"INSERT INTO hot_keywords (keyword, search_count, updated_at) VALUES (?, 1, ?)"
|
||||
).run(keyword, localTimestamp());
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('[Search] Failed to update hot keywords:', err);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user