Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 69 additions & 6 deletions clis/douban/utils.js
Original file line number Diff line number Diff line change
Expand Up @@ -563,22 +563,60 @@ export async function searchDouban(page, type, keyword, limit) {
const inferDoubanSearchResultType = ${inferDoubanSearchResultTypeSource};
const normalize = (value) => (value || '').replace(/\\s+/g, ' ').trim();
const seen = new Set();
const results = [];
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
const rawItems = Array.isArray(window.__DATA__?.items) ? window.__DATA__.items : [];
const hasPageDataItems = rawItems.length > 0;
const rawItemsById = new Map(
rawItems
.map((item) => [String(item?.id || '').trim(), item])
.filter(([id]) => id),
);
const normalizeRawRating = (item) => {
const rating = item?.rating;
if (typeof rating === 'number') return Number.isFinite(rating) ? rating : 0;
if (typeof rating === 'string') return parseFloat(rating) || 0;
if (rating && typeof rating === 'object') {
return parseFloat(String(rating.value || rating.rating || rating.average || '0')) || 0;
}
return 0;
};
const normalizeRawUrl = (item, id) => {
const rawUrl = normalize(item?.url || item?.uri || item?.link);
if (rawUrl) return rawUrl.startsWith('http') ? rawUrl : new URL(rawUrl, location.origin).toString();
if (!id) return '';
const domain = type === 'book' ? 'book.douban.com' : type === 'music' ? 'music.douban.com' : 'movie.douban.com';
return 'https://' + domain + '/subject/' + id + '/';
};
const normalizeRawAbstract = (item) => normalize(item?.abstract || item?.abstract_2 || item?.description || '');
const normalizeRawCover = (item) => normalize(item?.cover_url || item?.cover || item?.pic?.normal || item?.pic?.large || '');
const appendRawItemResult = (item) => {
const id = String(item?.id || '').trim();
const title = normalize(item?.title || item?.name);
const url = normalizeRawUrl(item, id);
if (!title || !url || !url.includes('/subject/') || seen.has(url)) return;
seen.add(url);
const abstract = normalizeRawAbstract(item);
results.push({
rank: results.length + 1,
id: id || (url.match(/subject\\/(\\d+)/)?.[1] || ''),
type: inferDoubanSearchResultType(type, item),
title,
rating: normalizeRawRating(item),
abstract: abstract.slice(0, 100) + (abstract.length > 100 ? '...' : ''),
url,
cover: normalizeRawCover(item),
});
};

for (let i = 0; i < 20; i += 1) {
if (document.querySelector('.item-root .title-text, .item-root .title a')) break;
await sleep(300);
}

const items = Array.from(document.querySelectorAll('.item-root, .result-list .result-item'));
const hasRenderedItems = items.length > 0;

const results = [];
for (const el of items) {
const titleEl = el.querySelector('.title-text, .title a, .title h3 a, h3 a, a[title]');
const title = normalize(titleEl?.textContent) || normalize(titleEl?.getAttribute('title'));
Expand All @@ -593,23 +631,48 @@ export async function searchDouban(page, type, keyword, limit) {
const abstract = normalize(
el.querySelector('.meta.abstract, .meta, .abstract, .subject-abstract, p')?.textContent,
);
const effectiveAbstract = abstract || normalizeRawAbstract(rawItem);
results.push({
rank: results.length + 1,
id,
type: inferDoubanSearchResultType(type, rawItem),
title,
rating: ratingText.includes('.') ? parseFloat(ratingText) : 0,
abstract: abstract.slice(0, 100) + (abstract.length > 100 ? '...' : ''),
rating: ratingText.includes('.') ? parseFloat(ratingText) : normalizeRawRating(rawItem),
abstract: effectiveAbstract.slice(0, 100) + (effectiveAbstract.length > 100 ? '...' : ''),
url,
cover: el.querySelector('img')?.getAttribute('src') || '',
cover: el.querySelector('img')?.getAttribute('src') || normalizeRawCover(rawItem),
});
if (results.length >= ${safeLimit}) break;
}
return results;
if (results.length === 0) {
for (const rawItem of rawItems) {
appendRawItemResult(rawItem);
if (results.length >= ${safeLimit}) break;
}
}
return {
results,
hasRenderedItems,
hasPageDataItems,
};
})()
`);
});
return Array.isArray(data) ? data : [];
const results = Array.isArray(data)
? data
: Array.isArray(data?.results)
? data.results
: [];
const hasRenderedItems = !Array.isArray(data) && Boolean(data?.hasRenderedItems);
const hasPageDataItems = !Array.isArray(data) && Boolean(data?.hasPageDataItems);
if (results.length === 0 && !hasRenderedItems && !hasPageDataItems) {
const keywordText = String(keyword || '').trim();
const queryHint = keywordText ? ` for "${keywordText}"` : '';
const hint = `No search result DOM items or window.__DATA__.items were found${queryHint}. `
+ 'The Douban search page may have changed or blocked the browser session.';
throw new EmptyResultError('douban search', hint);
}
return results;
}
/**
* Get current user's Douban ID from movie.douban.com/mine page
Expand Down
63 changes: 63 additions & 0 deletions clis/douban/utils.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,69 @@ describe('douban utils', () => {
]);
});

it('falls back to window data items when search result DOM is not rendered', async () => {
const rawItems = [
{
id: 2026281,
title: '大棋局 : 美国的首要地位及其地缘战略',
url: 'https://book.douban.com/subject/2026281/',
abstract: '兹比格纽·布热津斯基 / 中国国际问题研究所 / 上海人民出版社 / 2007-1 / 23.00元',
cover_url: 'https://img1.doubanio.com/view/subject/m/public/s2552669.jpg',
rating: { count: 5563, value: 8.7 },
tpl_name: 'search_subject',
},
{
id: 35284951,
title: '大棋局 : 美国的首要地位及其地缘战略',
url: 'https://book.douban.com/subject/35284951/',
abstract: '[美]兹比格纽•布热津斯基 著 / 上海人民出版社 / 2021-1 / 52',
cover_url: 'https://img2.doubanio.com/view/subject/m/public/s33779181.jpg',
rating: { count: 462, value: 8.5 },
tpl_name: 'search_subject',
},
];
const page = {
goto: vi.fn().mockResolvedValue(undefined),
wait: vi.fn().mockResolvedValue(undefined),
evaluate: vi.fn()
.mockResolvedValueOnce({ blocked: false, title: '大棋局 - 读书 - 豆瓣搜索', href: 'https://search.douban.com/book/subject_search?search_text=%E5%A4%A7%E6%A3%8B%E5%B1%80&cat=1001' })
.mockImplementationOnce((script) => runSearchEvaluate(script, rawItems, [])),
};

await expect(searchDouban(page, 'book', '大棋局', 3)).resolves.toMatchObject([
{
rank: 1,
id: '2026281',
type: 'book',
title: '大棋局 : 美国的首要地位及其地缘战略',
rating: 8.7,
url: 'https://book.douban.com/subject/2026281/',
cover: 'https://img1.doubanio.com/view/subject/m/public/s2552669.jpg',
},
{
rank: 2,
id: '35284951',
type: 'book',
rating: 8.5,
},
]);
});

it('throws when the search page exposes neither DOM result items nor page data items', async () => {
const page = {
goto: vi.fn().mockResolvedValue(undefined),
wait: vi.fn().mockResolvedValue(undefined),
evaluate: vi.fn()
.mockResolvedValueOnce({ blocked: false, title: '大棋局 - 读书 - 豆瓣搜索', href: 'https://search.douban.com/book/subject_search?search_text=%E5%A4%A7%E6%A3%8B%E5%B1%80&cat=1001' })
.mockImplementationOnce((script) => runSearchEvaluate(script, [], [])),
};

await expect(searchDouban(page, 'book', '大棋局', 3)).rejects.toMatchObject({
code: 'EMPTY_RESULT',
hint: expect.stringContaining('No search result DOM items or window.__DATA__.items were found for "大棋局"'),
});
});

it('normalizes douban book subject raw data into structured fields', () => {
const normalized = normalizeDoubanBookSubject({
id: '2567698',
Expand Down
119 changes: 118 additions & 1 deletion src/browser/cdp.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ vi.mock('ws', () => ({
WebSocket: MockWebSocket,
}));

import { CDPBridge } from './cdp.js';
import { CDPBridge, __test__ } from './cdp.js';

describe('CDPBridge cookies', () => {
beforeEach(() => {
Expand Down Expand Up @@ -64,3 +64,120 @@ describe('CDPBridge cookies', () => {
]);
});
});

describe('CDP target reuse', () => {
beforeEach(() => {
vi.unstubAllEnvs();
});

it('derives a stable tab name from the browser workspace', () => {
expect(__test__.buildCDPTabName('site:douban', {})).toBe('opencli:site:douban');
expect(__test__.buildCDPTabName(undefined, {})).toBe('opencli:default');
});

it('does not enable default tab reuse for registered Electron apps', () => {
expect(__test__.buildCDPTabName('site:cursor', {})).toBeUndefined();
expect(__test__.buildCDPTabName('site:codex', {})).toBeUndefined();
expect(__test__.buildCDPTabName('site:cursor', { OPENCLI_CDP_TAB_NAME: 'cursor-fixed' })).toBe('cursor-fixed');
});

it('allows explicit tab names and opt-out via environment', () => {
expect(__test__.buildCDPTabName('site:douban', { OPENCLI_CDP_TAB_NAME: 'douban-fixed' })).toBe('douban-fixed');
expect(__test__.buildCDPTabName('site:douban', { OPENCLI_CDP_REUSE_TAB: 'false' })).toBeUndefined();
});

it('selects an existing CDP target by persistent window.name', async () => {
const targets = [
{
id: 'a',
type: 'page',
title: '普通标签页',
url: 'https://www.douban.com/',
webSocketDebuggerUrl: 'ws://127.0.0.1/a',
},
{
id: 'b',
type: 'page',
title: '大棋局 - 读书 - 豆瓣搜索',
url: 'https://search.douban.com/book/subject_search?search_text=x',
webSocketDebuggerUrl: 'ws://127.0.0.1/b',
},
];

const selected = await __test__.selectNamedCDPTarget(
targets,
'opencli:site:douban',
async (target) => target.id === 'b' ? 'opencli:site:douban' : '',
);

expect(selected?.id).toBe('b');
});

it('creates a new target instead of reusing arbitrary tabs when a named target is absent', async () => {
const targets = [
{
id: 'a',
type: 'page',
title: 'User Tab',
url: 'https://example.com/',
webSocketDebuggerUrl: 'ws://127.0.0.1/a',
},
];

const selected = await __test__.resolveCDPTarget(
targets,
'http://127.0.0.1:9222',
'opencli:site:douban',
async () => '',
async () => ({
id: 'new',
type: 'page',
title: '',
url: 'about:blank',
webSocketDebuggerUrl: 'ws://127.0.0.1/new',
}),
);

expect(selected?.id).toBe('new');
});

it('keeps the ranked fallback for unnamed CDP target selection', async () => {
const targets = [
{
id: 'a',
type: 'page',
title: '',
url: 'about:blank',
webSocketDebuggerUrl: 'ws://127.0.0.1/a',
},
{
id: 'b',
type: 'page',
title: 'Local App',
url: 'http://localhost:3000/',
webSocketDebuggerUrl: 'ws://127.0.0.1/b',
},
];

const selected = await __test__.resolveCDPTarget(
targets,
'http://127.0.0.1:9222',
undefined,
async () => '',
async () => {
throw new Error('should not create a target');
},
);

expect(selected?.id).toBe('b');
});

it('does not pick Chrome internal popup targets', () => {
expect(__test__.scoreCDPTarget({
type: 'page',
title: 'Omnibox Popup',
url: 'chrome://omnibox-popup.top-chrome/',
webSocketDebuggerUrl: 'ws://127.0.0.1/omnibox',
})).toBe(Number.NEGATIVE_INFINITY);
});
});
Loading
Loading