feat: expand clickable elements

actors/apify_rag-web-browser/README.md

@@ -16,6 +16,7 @@ The extracted text can then be injected into prompts and retrieval augmented gen
 - 🕷 Automatically **bypasses anti-scraping protections** using proxies and browser fingerprints
 - 📝 Output formats include **Markdown**, plain text, and HTML
 - 🔗 **Links are converted to absolute URLs**, so they stay valid outside of the page they came from
+- 🪗 **Collapsed sections are expanded** in Browser mode, so their content is not missing from the output
 - 🔌 Supports **OpenAPI and MCP** for easy integration
 - 🪟 It's **open source**, so you can review and modify it
 
@@ -234,6 +235,19 @@ Media files carry no text for the LLM, so the Actor never downloads them:
 - Search results (and a `query` that is a URL) pointing directly to a media file, e.g. `https://example.com/video.mp4`,
   are not crawled at all. Such a result is returned with an empty text and `Skipped media file` as the HTTP status message.
 
+### Collapsed content
+
+Some web pages keep parts of their content hidden until the reader expands them, e.g. accordions or
+FAQ sections, and add it to the page only when it's clicked. To capture such content, the Actor clicks
+the collapsed elements of the page, i.e. those matching the `[aria-expanded="false"]` CSS selector,
+in Browser mode (`scrapingTool=browser-playwright`) before it extracts the content.
+
+Elements linking to another page are not clicked, so that the Actor stays on the page it was asked
+to extract.
+
+Note that clicking costs about half a second on pages that have such elements, and that content of
+expanded navigation menus can end up in the output as well.
+
 ### Reducing response time
 
 For low-latency applications, it's recommended to run the RAG Web Browser in Standby mode

actors/apify_url-to-markdown/README.md

@@ -88,6 +88,9 @@ Relative links such as `/docs` are resolved into absolute URLs against the page
 ### What happens with media files?
 Media files carry no text to convert, so they are never downloaded. Images, audio, video, and fonts of the converted page are blocked in the Browser mode, and a URL pointing directly to a media file, e.g. `https://example.com/video.mp4`, is not fetched at all — the output contains no content and `Skipped media file` as the HTTP status message.
 
+### What happens with collapsed content?
+Content that a page adds only when the reader expands it, e.g. accordions or FAQ sections, would be missing from the markdown. To capture it, the Browser mode clicks the collapsed elements of the page, i.e. those matching the `[aria-expanded="false"]` CSS selector, before converting it. Elements linking to another page are not clicked, so that the Actor stays on the page it was asked to convert.
+
 ### Can I use URL to Markdown with the Apify API?
 The Apify API gives you programmatic access to the Apify platform. The API is organized around RESTful HTTP endpoints that enable you to manage, schedule, and run Apify Actors. The API also lets you access any datasets, monitor Actor performance, fetch results, create and update versions, and more.
 

src/request-handler.ts

@@ -11,6 +11,11 @@ import { addTimeMeasureEvent, isActorStandby, transformTimeMeasuresToRelative }
 import { processHtml } from './website-content-crawler/html-processing.js';
 import { htmlToMarkdown } from './website-content-crawler/markdown.js';
 
+/** Same as the default of the `clickElementsCssSelector` input of Website Content Crawler. */
+const CLICK_ELEMENTS_CSS_SELECTOR = '[aria-expanded="false"]';
+
+const CLICK_RENDER_WAIT_MS = 500;
+
 let ACTOR_TIMEOUT_AT: number | undefined;
 try {
     ACTOR_TIMEOUT_AT = process.env.ACTOR_TIMEOUT_AT ? new Date(process.env.ACTOR_TIMEOUT_AT).getTime() : undefined;
@@ -70,6 +75,34 @@ export async function waitForDynamicContent(context: PlaywrightCrawlingContext,
     }
 }
 
+/**
+ * Tries to expand collapsed content by clicking on it, so that its text is included in the extracted
+ * content, e.g. https://www.checkout.com/docs/support/reporting (adapted from: Website Content Crawler).
+ *
+ * A click handler can still navigate the page with JavaScript, and then the content is extracted from
+ * the page it navigated to, the same as in Website Content Crawler.
+ */
+async function expandClickableElements(page: PlaywrightCrawlingContext['page'], cssSelector: string) {
+    const clickedCount = await page.evaluate((selector) => {
+        // only click on items that don't have `href` attribute or they lead to the current page
+        const elements = [...document.querySelectorAll(selector)].filter((el) => {
+            const href = el.getAttribute('href');
+            return (!href || href.startsWith('#')) && typeof (el as HTMLElement).click === 'function';
+        });
+
+        for (const el of elements) {
+            (el as HTMLElement).click();
+        }
+
+        return elements.length;
+    }, cssSelector);
+
+    if (clickedCount > 0) {
+        log.debug(`Clicked ${clickedCount} element(s) matching \`${cssSelector}\``);
+        await sleep(CLICK_RENDER_WAIT_MS);
+    }
+}
+
 type ContentCrawlingContext = PlaywrightCrawlingContext<ContentCrawlerUserData> | CheerioCrawlingContext<ContentCrawlerUserData>;
 
 function isValidContentType(contentType: string | undefined) {
@@ -227,6 +260,11 @@ export async function requestHandlerPlaywright(
         addTimeMeasureEvent(request.userData, 'playwright-remove-cookie');
     }
 
+    if (page) {
+        await expandClickableElements(page, CLICK_ELEMENTS_CSS_SELECTOR);
+        addTimeMeasureEvent(request.userData, 'playwright-expand-clickable-elements');
+    }
+
     // Parsing the page after the dynamic content has been loaded / cookie warnings removed
     const $ = await context.parseWithCheerio();
     addTimeMeasureEvent(request.userData, 'playwright-parse-with-cheerio');

src/types.ts

@@ -69,6 +69,7 @@ export interface TimeMeasure {
         | 'error'
         | 'playwright-request-start'
         | 'playwright-wait-dynamic-content'
+        | 'playwright-expand-clickable-elements'
         | 'playwright-parse-with-cheerio'
         | 'playwright-process-html'
         | 'playwright-remove-cookie'

tests/helpers/html/clickable.html

@@ -0,0 +1,41 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Clickable Test Page</title>
+</head>
+<body>
+    <p>always visible content</p>
+
+    <button id="toggle">Show details</button>
+    <div id="panel"></div>
+
+    <a id="anchor" href="#anchor-panel">Show more</a>
+    <div id="anchor-panel"></div>
+
+    <a id="link" href="/basic#section">Link to another page</a>
+    <div id="link-panel"></div>
+
+    <script>
+        // Collapsed content is added to the page only when clicked, and takes a moment to render
+        const reveal = (panelId, text) => {
+            window.setTimeout(() => {
+                document.getElementById(panelId).textContent = text;
+            }, 150);
+        };
+
+        document.getElementById('toggle').addEventListener('click', () => reveal('panel', 'collapsed panel content'));
+        document.getElementById('anchor').addEventListener('click', () => reveal('anchor-panel', 'anchor panel content'));
+        document.getElementById('link').addEventListener('click', () => reveal('link-panel', 'link panel content'));
+
+        // The elements become collapsed only once the page is interactive, so they cannot be clicked
+        // before the Actor waits for the dynamic content
+        window.setTimeout(() => {
+            for (const id of ['toggle', 'anchor', 'link']) {
+                document.getElementById(id).setAttribute('aria-expanded', 'false');
+            }
+        }, 200);
+    </script>
+</body>
+</html>

tests/helpers/server.ts

@@ -30,6 +30,10 @@ export function createTestServer() {
         sendHtml('basic.html', res);
     });
 
+    app.get('/clickable', (_req, res) => {
+        sendHtml('clickable.html', res);
+    });
+
     app.get('/with-image', (_req, res) => {
         sendHtml('with-image.html', res);
     });

tests/playwright-crawler.content.test.ts

@@ -7,7 +7,7 @@ import { firefox } from 'playwright';
 import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
 
 import { requestHandlerPlaywright } from '../src/request-handler.js';
-import type { ContentCrawlerUserData } from '../src/types.js';
+import type { ContentCrawlerUserData, ContentScraperSettings, Output } from '../src/types.js';
 import { createRequest } from '../src/utils.js';
 import { startTestServer, stopTestServer } from './helpers/server.js';
 
@@ -27,9 +27,12 @@ describe('Playwright Crawler Content Tests', () => {
         await stopTestServer(testServer);
     });
 
-    it('test basic content extraction with playwright', async () => {
+    /**
+     * Scrapes a single URL and returns the results the request handler pushed to the dataset.
+     */
+    async function scrapeWithPlaywright(url: string, settings: Partial<ContentScraperSettings> = {}) {
+        const results: Output[] = [];
         const failedUrls = new Set<string>();
-        const successUrls = new Set<string>();
 
         // Create memory storage and request queue
         const client = new MemoryStorage({ persistStorage: false });
@@ -38,14 +41,10 @@ describe('Playwright Crawler Content Tests', () => {
         const crawler = new PlaywrightCrawler({
             requestQueue,
             requestHandler: async (context) => {
-                const pushDataSpy = vi.spyOn(context, 'pushData').mockResolvedValue(undefined);
+                vi.spyOn(context, 'pushData').mockImplementation(async (data) => {
+                    results.push(data as Output);
+                });
                 await requestHandlerPlaywright(context as unknown as PlaywrightCrawlingContext<ContentCrawlerUserData>);
-
-                expect(pushDataSpy).toHaveBeenCalledTimes(1);
-                expect(pushDataSpy).toHaveBeenCalledWith(expect.objectContaining({
-                    text: expect.stringContaining('hello world'),
-                }));
-                successUrls.add(context.request.url);
             },
             failedRequestHandler: async ({ request }, error) => {
                 log.error(`Request ${request.url} failed with error: ${error.message}`);
@@ -65,7 +64,7 @@ describe('Playwright Crawler Content Tests', () => {
         const r = createRequest(
             'query',
             {
-                url: `${baseUrl}/basic`,
+                url,
                 description: 'Test request',
                 rank: 1,
                 title: 'Test title',
@@ -76,6 +75,7 @@ describe('Playwright Crawler Content Tests', () => {
                 outputFormats: ['text'],
                 maxHtmlCharsToProcess: 100000,
                 dynamicContentWaitSecs: 20,
+                ...settings,
             },
             [],
         );
@@ -85,7 +85,30 @@ describe('Playwright Crawler Content Tests', () => {
 
         await crawler.run();
 
+        return { results, failedUrls };
+    }
+
+    it('test basic content extraction with playwright', async () => {
+        const { results, failedUrls } = await scrapeWithPlaywright(`${baseUrl}/basic`);
+
+        expect(failedUrls.size).toBe(0);
+        expect(results).toHaveLength(1);
+        expect(results[0].text).toContain('hello world');
+    });
+
+    it('expands clickable elements to extract collapsed content', async () => {
+        const { results, failedUrls } = await scrapeWithPlaywright(`${baseUrl}/clickable`, {
+            dynamicContentWaitSecs: 2,
+        });
+
         expect(failedUrls.size).toBe(0);
-        expect(successUrls.size).toBe(1);
+        expect(results).toHaveLength(1);
+        expect(results[0].text).toContain('always visible content');
+        // Content of every collapsed element, which is added to the page only once it's clicked
+        expect(results[0].text).toContain('collapsed panel content');
+        expect(results[0].text).toContain('anchor panel content');
+        // The link leading to another page must not be clicked, not even to its fragment
+        expect(results[0].text).not.toContain('link panel content');
+        expect(results[0].text).not.toContain('hello world');
     });
 });

vitest.config.ts

@@ -2,7 +2,8 @@ import { defineConfig } from 'vitest/config';
 
 export default defineConfig({
     test: {
-        testTimeout: 15000,
+        // Generous timeout because most of the tests launch a browser, which is slow on a cold start
+        testTimeout: 30000,
         globals: true,
         environment: 'node',
         include: ['tests/**/*.test.ts'],