diff --git a/actors/apify_rag-web-browser/README.md b/actors/apify_rag-web-browser/README.md index 94d7551..fde51e6 100644 --- a/actors/apify_rag-web-browser/README.md +++ b/actors/apify_rag-web-browser/README.md @@ -16,6 +16,7 @@ The extracted text can then be injected into prompts and retrieval augmented gen - 🕷 Automatically **bypasses anti-scraping protections** using proxies and browser fingerprints - 📝 Output formats include **Markdown**, plain text, and HTML - 🔗 **Links are converted to absolute URLs**, so they stay valid outside of the page they came from +- 🪗 **Collapsed sections are expanded** in Browser mode, so their content is not missing from the output - 🔌 Supports **OpenAPI and MCP** for easy integration - 🪟 It's **open source**, so you can review and modify it @@ -234,6 +235,19 @@ Media files carry no text for the LLM, so the Actor never downloads them: - Search results (and a `query` that is a URL) pointing directly to a media file, e.g. `https://example.com/video.mp4`, are not crawled at all. Such a result is returned with an empty text and `Skipped media file` as the HTTP status message. +### Collapsed content + +Some web pages keep parts of their content hidden until the reader expands them, e.g. accordions or +FAQ sections, and add it to the page only when it's clicked. To capture such content, the Actor clicks +the collapsed elements of the page, i.e. those matching the `[aria-expanded="false"]` CSS selector, +in Browser mode (`scrapingTool=browser-playwright`) before it extracts the content. + +Elements linking to another page are not clicked, so that the Actor stays on the page it was asked +to extract. + +Note that clicking costs about half a second on pages that have such elements, and that content of +expanded navigation menus can end up in the output as well. + ### Reducing response time For low-latency applications, it's recommended to run the RAG Web Browser in Standby mode diff --git a/actors/apify_url-to-markdown/README.md b/actors/apify_url-to-markdown/README.md index 95bd910..fdd5b30 100644 --- a/actors/apify_url-to-markdown/README.md +++ b/actors/apify_url-to-markdown/README.md @@ -88,6 +88,9 @@ Relative links such as `/docs` are resolved into absolute URLs against the page ### What happens with media files? Media files carry no text to convert, so they are never downloaded. Images, audio, video, and fonts of the converted page are blocked in the Browser mode, and a URL pointing directly to a media file, e.g. `https://example.com/video.mp4`, is not fetched at all — the output contains no content and `Skipped media file` as the HTTP status message. +### What happens with collapsed content? +Content that a page adds only when the reader expands it, e.g. accordions or FAQ sections, would be missing from the markdown. To capture it, the Browser mode clicks the collapsed elements of the page, i.e. those matching the `[aria-expanded="false"]` CSS selector, before converting it. Elements linking to another page are not clicked, so that the Actor stays on the page it was asked to convert. + ### Can I use URL to Markdown with the Apify API? The Apify API gives you programmatic access to the Apify platform. The API is organized around RESTful HTTP endpoints that enable you to manage, schedule, and run Apify Actors. The API also lets you access any datasets, monitor Actor performance, fetch results, create and update versions, and more. diff --git a/src/request-handler.ts b/src/request-handler.ts index 69044a4..e505023 100644 --- a/src/request-handler.ts +++ b/src/request-handler.ts @@ -11,6 +11,11 @@ import { addTimeMeasureEvent, isActorStandby, transformTimeMeasuresToRelative } import { processHtml } from './website-content-crawler/html-processing.js'; import { htmlToMarkdown } from './website-content-crawler/markdown.js'; +/** Same as the default of the `clickElementsCssSelector` input of Website Content Crawler. */ +const CLICK_ELEMENTS_CSS_SELECTOR = '[aria-expanded="false"]'; + +const CLICK_RENDER_WAIT_MS = 500; + let ACTOR_TIMEOUT_AT: number | undefined; try { ACTOR_TIMEOUT_AT = process.env.ACTOR_TIMEOUT_AT ? new Date(process.env.ACTOR_TIMEOUT_AT).getTime() : undefined; @@ -70,6 +75,34 @@ export async function waitForDynamicContent(context: PlaywrightCrawlingContext, } } +/** + * Tries to expand collapsed content by clicking on it, so that its text is included in the extracted + * content, e.g. https://www.checkout.com/docs/support/reporting (adapted from: Website Content Crawler). + * + * A click handler can still navigate the page with JavaScript, and then the content is extracted from + * the page it navigated to, the same as in Website Content Crawler. + */ +async function expandClickableElements(page: PlaywrightCrawlingContext['page'], cssSelector: string) { + const clickedCount = await page.evaluate((selector) => { + // only click on items that don't have `href` attribute or they lead to the current page + const elements = [...document.querySelectorAll(selector)].filter((el) => { + const href = el.getAttribute('href'); + return (!href || href.startsWith('#')) && typeof (el as HTMLElement).click === 'function'; + }); + + for (const el of elements) { + (el as HTMLElement).click(); + } + + return elements.length; + }, cssSelector); + + if (clickedCount > 0) { + log.debug(`Clicked ${clickedCount} element(s) matching \`${cssSelector}\``); + await sleep(CLICK_RENDER_WAIT_MS); + } +} + type ContentCrawlingContext = PlaywrightCrawlingContext | CheerioCrawlingContext; function isValidContentType(contentType: string | undefined) { @@ -227,6 +260,11 @@ export async function requestHandlerPlaywright( addTimeMeasureEvent(request.userData, 'playwright-remove-cookie'); } + if (page) { + await expandClickableElements(page, CLICK_ELEMENTS_CSS_SELECTOR); + addTimeMeasureEvent(request.userData, 'playwright-expand-clickable-elements'); + } + // Parsing the page after the dynamic content has been loaded / cookie warnings removed const $ = await context.parseWithCheerio(); addTimeMeasureEvent(request.userData, 'playwright-parse-with-cheerio'); diff --git a/src/types.ts b/src/types.ts index 2938e1f..ac6fcae 100644 --- a/src/types.ts +++ b/src/types.ts @@ -69,6 +69,7 @@ export interface TimeMeasure { | 'error' | 'playwright-request-start' | 'playwright-wait-dynamic-content' + | 'playwright-expand-clickable-elements' | 'playwright-parse-with-cheerio' | 'playwright-process-html' | 'playwright-remove-cookie' diff --git a/tests/helpers/html/clickable.html b/tests/helpers/html/clickable.html new file mode 100644 index 0000000..797e2b0 --- /dev/null +++ b/tests/helpers/html/clickable.html @@ -0,0 +1,41 @@ + + + + + + Clickable Test Page + + +

always visible content

+ + +
+ + Show more +
+ + Link to another page + + + + + diff --git a/tests/helpers/server.ts b/tests/helpers/server.ts index f549e21..2e11cb3 100644 --- a/tests/helpers/server.ts +++ b/tests/helpers/server.ts @@ -30,6 +30,10 @@ export function createTestServer() { sendHtml('basic.html', res); }); + app.get('/clickable', (_req, res) => { + sendHtml('clickable.html', res); + }); + app.get('/with-image', (_req, res) => { sendHtml('with-image.html', res); }); diff --git a/tests/playwright-crawler.content.test.ts b/tests/playwright-crawler.content.test.ts index 920edc9..309d69e 100644 --- a/tests/playwright-crawler.content.test.ts +++ b/tests/playwright-crawler.content.test.ts @@ -7,7 +7,7 @@ import { firefox } from 'playwright'; import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; import { requestHandlerPlaywright } from '../src/request-handler.js'; -import type { ContentCrawlerUserData } from '../src/types.js'; +import type { ContentCrawlerUserData, ContentScraperSettings, Output } from '../src/types.js'; import { createRequest } from '../src/utils.js'; import { startTestServer, stopTestServer } from './helpers/server.js'; @@ -27,9 +27,12 @@ describe('Playwright Crawler Content Tests', () => { await stopTestServer(testServer); }); - it('test basic content extraction with playwright', async () => { + /** + * Scrapes a single URL and returns the results the request handler pushed to the dataset. + */ + async function scrapeWithPlaywright(url: string, settings: Partial = {}) { + const results: Output[] = []; const failedUrls = new Set(); - const successUrls = new Set(); // Create memory storage and request queue const client = new MemoryStorage({ persistStorage: false }); @@ -38,14 +41,10 @@ describe('Playwright Crawler Content Tests', () => { const crawler = new PlaywrightCrawler({ requestQueue, requestHandler: async (context) => { - const pushDataSpy = vi.spyOn(context, 'pushData').mockResolvedValue(undefined); + vi.spyOn(context, 'pushData').mockImplementation(async (data) => { + results.push(data as Output); + }); await requestHandlerPlaywright(context as unknown as PlaywrightCrawlingContext); - - expect(pushDataSpy).toHaveBeenCalledTimes(1); - expect(pushDataSpy).toHaveBeenCalledWith(expect.objectContaining({ - text: expect.stringContaining('hello world'), - })); - successUrls.add(context.request.url); }, failedRequestHandler: async ({ request }, error) => { log.error(`Request ${request.url} failed with error: ${error.message}`); @@ -65,7 +64,7 @@ describe('Playwright Crawler Content Tests', () => { const r = createRequest( 'query', { - url: `${baseUrl}/basic`, + url, description: 'Test request', rank: 1, title: 'Test title', @@ -76,6 +75,7 @@ describe('Playwright Crawler Content Tests', () => { outputFormats: ['text'], maxHtmlCharsToProcess: 100000, dynamicContentWaitSecs: 20, + ...settings, }, [], ); @@ -85,7 +85,30 @@ describe('Playwright Crawler Content Tests', () => { await crawler.run(); + return { results, failedUrls }; + } + + it('test basic content extraction with playwright', async () => { + const { results, failedUrls } = await scrapeWithPlaywright(`${baseUrl}/basic`); + + expect(failedUrls.size).toBe(0); + expect(results).toHaveLength(1); + expect(results[0].text).toContain('hello world'); + }); + + it('expands clickable elements to extract collapsed content', async () => { + const { results, failedUrls } = await scrapeWithPlaywright(`${baseUrl}/clickable`, { + dynamicContentWaitSecs: 2, + }); + expect(failedUrls.size).toBe(0); - expect(successUrls.size).toBe(1); + expect(results).toHaveLength(1); + expect(results[0].text).toContain('always visible content'); + // Content of every collapsed element, which is added to the page only once it's clicked + expect(results[0].text).toContain('collapsed panel content'); + expect(results[0].text).toContain('anchor panel content'); + // The link leading to another page must not be clicked, not even to its fragment + expect(results[0].text).not.toContain('link panel content'); + expect(results[0].text).not.toContain('hello world'); }); }); diff --git a/vitest.config.ts b/vitest.config.ts index af08605..3526d8d 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -2,7 +2,8 @@ import { defineConfig } from 'vitest/config'; export default defineConfig({ test: { - testTimeout: 15000, + // Generous timeout because most of the tests launch a browser, which is slow on a cold start + testTimeout: 30000, globals: true, environment: 'node', include: ['tests/**/*.test.ts'],