From 05bb64020fd9f0eb504c35e7bdbcf5e5cfb24785 Mon Sep 17 00:00:00 2001
From: nikitachapovskii-dev <nikita.chapovskii@apify.com>
Date: Mon, 3 Aug 2026 12:10:40 +0200
Subject: [PATCH 1/2] feat: expand clickable elements

---
 src/request-handler.ts                        | 56 ++++++++++++++++++
 src/types.ts                                  |  1 +
 tests/cheerio-crawler.content.test.ts         | 41 +++++++++----
 .../helpers/html/clickable-js-navigation.html | 14 +++++
 tests/helpers/html/clickable.html             | 28 +++++++++
 tests/helpers/server.ts                       |  8 +++
 tests/playwright-crawler.content.test.ts      | 59 +++++++++++++++----
 tests/standby.test.ts                         | 10 ++++
 vitest.config.ts                              |  3 +-
 9 files changed, 195 insertions(+), 25 deletions(-)
 create mode 100644 tests/helpers/html/clickable-js-navigation.html
 create mode 100644 tests/helpers/html/clickable.html

diff --git a/src/request-handler.ts b/src/request-handler.ts
index 69044a4..fd29779 100644
--- a/src/request-handler.ts
+++ b/src/request-handler.ts
@@ -11,6 +11,12 @@ import { addTimeMeasureEvent, isActorStandby, transformTimeMeasuresToRelative }
 import { processHtml } from './website-content-crawler/html-processing.js';
 import { htmlToMarkdown } from './website-content-crawler/markdown.js';
 
+/** Collapsed elements that are clicked to expand their content, same default as Website Content Crawler. */
+const CLICK_ELEMENTS_CSS_SELECTOR = '[aria-expanded="false"]';
+
+/** How long to wait for the content expanded by clicking to render. */
+const CLICK_RENDER_WAIT_MS = 500;
+
 let ACTOR_TIMEOUT_AT: number | undefined;
 try {
     ACTOR_TIMEOUT_AT = process.env.ACTOR_TIMEOUT_AT ? new Date(process.env.ACTOR_TIMEOUT_AT).getTime() : undefined;
@@ -70,6 +76,51 @@ export async function waitForDynamicContent(context: PlaywrightCrawlingContext,
     }
 }
 
+/**
+ * Tries to expand collapsed content by clicking on it, so that its text is included in the extracted
+ * content, e.g. https://www.checkout.com/docs/support/reporting (adapted from: Website Content Crawler).
+ *
+ * Failed clicks are only logged, the content is extracted with whatever got expanded. If the clicking
+ * navigates the page away, the page is loaded again, so that we never extract a different page.
+ */
+async function expandClickableElements(page: PlaywrightCrawlingContext['page'], cssSelector: string) {
+    const urlBeforeClicking = page.url();
+
+    const clickedCount = await page.evaluate((selector) => {
+        // Only click on elements that don't have the `href` attribute or that lead to the current page,
+        // so that we don't navigate away from the page
+        const elements = [...document.querySelectorAll(selector)].filter((el) => {
+            const href = el.getAttribute('href');
+            return !href || href.startsWith('#');
+        });
+
+        for (const el of elements) {
+            (el as HTMLElement).click?.();
+        }
+
+        return elements.length;
+    }, cssSelector).catch((err: unknown) => {
+        log.warning(`Failed to expand clickable elements: ${err instanceof Error ? err.message : String(err)}`);
+        // Some of the elements might have been clicked before the failure
+        return null;
+    });
+
+    // Nothing was clicked, so there is nothing to wait for and the page could not have navigated
+    if (clickedCount === 0) {
+        return;
+    }
+
+    log.debug(`Clicked ${clickedCount ?? 'some'} element(s) matching \`${cssSelector}\``);
+    await sleep(CLICK_RENDER_WAIT_MS);
+
+    // A click handler can navigate the page with JavaScript, which no CSS selector can guard against.
+    // Content of a different page would be worse than content without the expanded sections, so undo it.
+    if (page.url().split('#')[0] !== urlBeforeClicking.split('#')[0]) {
+        log.warning(`Clicking navigated the page to ${page.url()}, loading ${urlBeforeClicking} again`);
+        await page.goto(urlBeforeClicking, { waitUntil: 'domcontentloaded' });
+    }
+}
+
 type ContentCrawlingContext = PlaywrightCrawlingContext<ContentCrawlerUserData> | CheerioCrawlingContext<ContentCrawlerUserData>;
 
 function isValidContentType(contentType: string | undefined) {
@@ -227,6 +278,11 @@ export async function requestHandlerPlaywright(
         addTimeMeasureEvent(request.userData, 'playwright-remove-cookie');
     }
 
+    if (page) {
+        await expandClickableElements(page, CLICK_ELEMENTS_CSS_SELECTOR);
+        addTimeMeasureEvent(request.userData, 'playwright-expand-clickable-elements');
+    }
+
     // Parsing the page after the dynamic content has been loaded / cookie warnings removed
     const $ = await context.parseWithCheerio();
     addTimeMeasureEvent(request.userData, 'playwright-parse-with-cheerio');
diff --git a/src/types.ts b/src/types.ts
index 2938e1f..ac6fcae 100644
--- a/src/types.ts
+++ b/src/types.ts
@@ -69,6 +69,7 @@ export interface TimeMeasure {
         | 'error'
         | 'playwright-request-start'
         | 'playwright-wait-dynamic-content'
+        | 'playwright-expand-clickable-elements'
         | 'playwright-parse-with-cheerio'
         | 'playwright-process-html'
         | 'playwright-remove-cookie'
diff --git a/tests/cheerio-crawler.content.test.ts b/tests/cheerio-crawler.content.test.ts
index b47706e..4208323 100644
--- a/tests/cheerio-crawler.content.test.ts
+++ b/tests/cheerio-crawler.content.test.ts
@@ -6,7 +6,7 @@ import { CheerioCrawler, type CheerioCrawlingContext, Configuration, log } from
 import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
 
 import { requestHandlerCheerio } from '../src/request-handler.js';
-import type { ContentCrawlerUserData } from '../src/types.js';
+import type { ContentCrawlerUserData, Output } from '../src/types.js';
 import { createRequest } from '../src/utils.js';
 import { startTestServer, stopTestServer } from './helpers/server.js';
 
@@ -25,9 +25,13 @@ describe('Cheerio Crawler Content Tests', () => {
         await stopTestServer(testServer);
     });
 
-    it('test basic content extraction with cheerio', async () => {
+    /**
+     * Scrapes a single URL with the Cheerio request handler and returns the results it pushed
+     * to the dataset, along with the URLs that failed.
+     */
+    async function scrapeWithCheerio(url: string) {
+        const results: Output[] = [];
         const failedUrls = new Set<string>();
-        const successUrls = new Set<string>();
 
         // Create memory storage and request queue
         const client = new MemoryStorage({ persistStorage: false });
@@ -36,14 +40,10 @@ describe('Cheerio Crawler Content Tests', () => {
         const crawler = new CheerioCrawler({
             requestQueue,
             requestHandler: async (context: CheerioCrawlingContext<ContentCrawlerUserData>) => {
-                const pushDataSpy = vi.spyOn(context, 'pushData').mockResolvedValue(undefined);
+                vi.spyOn(context, 'pushData').mockImplementation(async (data) => {
+                    results.push(data as Output);
+                });
                 await requestHandlerCheerio(context);
-
-                expect(pushDataSpy).toHaveBeenCalledTimes(1);
-                expect(pushDataSpy).toHaveBeenCalledWith(expect.objectContaining({
-                    text: expect.stringContaining('hello world'),
-                }));
-                successUrls.add(context.request.url);
             },
             failedRequestHandler: async ({ request }, error) => {
                 log.error(`Request ${request.url} failed with error: ${error.message}`);
@@ -56,7 +56,7 @@ describe('Cheerio Crawler Content Tests', () => {
         const r = createRequest(
             'query',
             {
-                url: `${baseUrl}/basic`,
+                url,
                 description: 'Test request',
                 rank: 1,
                 title: 'Test title',
@@ -76,7 +76,24 @@ describe('Cheerio Crawler Content Tests', () => {
 
         await crawler.run();
 
+        return { results, failedUrls };
+    }
+
+    it('test basic content extraction with cheerio', async () => {
+        const { results, failedUrls } = await scrapeWithCheerio(`${baseUrl}/basic`);
+
+        expect(failedUrls.size).toBe(0);
+        expect(results).toHaveLength(1);
+        expect(results[0].text).toContain('hello world');
+    });
+
+    it('does not expand clickable elements in the raw HTTP mode', async () => {
+        const { results, failedUrls } = await scrapeWithCheerio(`${baseUrl}/clickable`);
+
         expect(failedUrls.size).toBe(0);
-        expect(successUrls.size).toBe(1);
+        expect(results).toHaveLength(1);
+        expect(results[0].text).toContain('always visible content');
+        // Clicking needs a browser, so the panel content never makes it into the DOM
+        expect(results[0].text).not.toContain('collapsed panel content');
     });
 });
diff --git a/tests/helpers/html/clickable-js-navigation.html b/tests/helpers/html/clickable-js-navigation.html
new file mode 100644
index 0000000..c3c9bd7
--- /dev/null
+++ b/tests/helpers/html/clickable-js-navigation.html
@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Clickable JS Navigation Test Page</title>
+</head>
+<body>
+    <p>page navigating on click</p>
+
+    <!-- There is no `href` to filter on, the click handler navigates the page with JavaScript -->
+    <button aria-expanded="false" onclick="window.location.href = '/basic'">Go elsewhere</button>
+</body>
+</html>
diff --git a/tests/helpers/html/clickable.html b/tests/helpers/html/clickable.html
new file mode 100644
index 0000000..a8c1c41
--- /dev/null
+++ b/tests/helpers/html/clickable.html
@@ -0,0 +1,28 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Clickable Test Page</title>
+</head>
+<body>
+    <p>always visible content</p>
+
+    <!-- The panel content is added to the DOM only after the toggle is clicked -->
+    <button id="toggle" aria-expanded="false" aria-controls="panel">Show details</button>
+    <div id="panel"></div>
+
+    <!-- Links leading to another page must not be clicked, otherwise we navigate away from the page -->
+    <a id="link" href="/basic" aria-expanded="false">Link to another page</a>
+
+    <!-- Same-page anchors are clicked, they only change the URL hash -->
+    <a id="anchor" href="#panel" aria-expanded="false">Anchor to the panel</a>
+
+    <script>
+        document.getElementById('toggle').addEventListener('click', (event) => {
+            event.currentTarget.setAttribute('aria-expanded', 'true');
+            document.getElementById('panel').textContent = 'collapsed panel content';
+        });
+    </script>
+</body>
+</html>
diff --git a/tests/helpers/server.ts b/tests/helpers/server.ts
index f549e21..5111385 100644
--- a/tests/helpers/server.ts
+++ b/tests/helpers/server.ts
@@ -30,6 +30,14 @@ export function createTestServer() {
         sendHtml('basic.html', res);
     });
 
+    app.get('/clickable', (_req, res) => {
+        sendHtml('clickable.html', res);
+    });
+
+    app.get('/clickable-js-navigation', (_req, res) => {
+        sendHtml('clickable-js-navigation.html', res);
+    });
+
     app.get('/with-image', (_req, res) => {
         sendHtml('with-image.html', res);
     });
diff --git a/tests/playwright-crawler.content.test.ts b/tests/playwright-crawler.content.test.ts
index 920edc9..095fbf4 100644
--- a/tests/playwright-crawler.content.test.ts
+++ b/tests/playwright-crawler.content.test.ts
@@ -7,7 +7,7 @@ import { firefox } from 'playwright';
 import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
 
 import { requestHandlerPlaywright } from '../src/request-handler.js';
-import type { ContentCrawlerUserData } from '../src/types.js';
+import type { ContentCrawlerUserData, ContentScraperSettings, Output } from '../src/types.js';
 import { createRequest } from '../src/utils.js';
 import { startTestServer, stopTestServer } from './helpers/server.js';
 
@@ -27,9 +27,13 @@ describe('Playwright Crawler Content Tests', () => {
         await stopTestServer(testServer);
     });
 
-    it('test basic content extraction with playwright', async () => {
+    /**
+     * Scrapes a single URL with the Playwright request handler and returns the results it pushed
+     * to the dataset, along with the URLs that failed.
+     */
+    async function scrapeWithPlaywright(url: string, settings: Partial<ContentScraperSettings> = {}) {
+        const results: Output[] = [];
         const failedUrls = new Set<string>();
-        const successUrls = new Set<string>();
 
         // Create memory storage and request queue
         const client = new MemoryStorage({ persistStorage: false });
@@ -38,14 +42,10 @@ describe('Playwright Crawler Content Tests', () => {
         const crawler = new PlaywrightCrawler({
             requestQueue,
             requestHandler: async (context) => {
-                const pushDataSpy = vi.spyOn(context, 'pushData').mockResolvedValue(undefined);
+                vi.spyOn(context, 'pushData').mockImplementation(async (data) => {
+                    results.push(data as Output);
+                });
                 await requestHandlerPlaywright(context as unknown as PlaywrightCrawlingContext<ContentCrawlerUserData>);
-
-                expect(pushDataSpy).toHaveBeenCalledTimes(1);
-                expect(pushDataSpy).toHaveBeenCalledWith(expect.objectContaining({
-                    text: expect.stringContaining('hello world'),
-                }));
-                successUrls.add(context.request.url);
             },
             failedRequestHandler: async ({ request }, error) => {
                 log.error(`Request ${request.url} failed with error: ${error.message}`);
@@ -65,7 +65,7 @@ describe('Playwright Crawler Content Tests', () => {
         const r = createRequest(
             'query',
             {
-                url: `${baseUrl}/basic`,
+                url,
                 description: 'Test request',
                 rank: 1,
                 title: 'Test title',
@@ -76,6 +76,7 @@ describe('Playwright Crawler Content Tests', () => {
                 outputFormats: ['text'],
                 maxHtmlCharsToProcess: 100000,
                 dynamicContentWaitSecs: 20,
+                ...settings,
             },
             [],
         );
@@ -85,7 +86,41 @@ describe('Playwright Crawler Content Tests', () => {
 
         await crawler.run();
 
+        return { results, failedUrls };
+    }
+
+    it('test basic content extraction with playwright', async () => {
+        const { results, failedUrls } = await scrapeWithPlaywright(`${baseUrl}/basic`);
+
+        expect(failedUrls.size).toBe(0);
+        expect(results).toHaveLength(1);
+        expect(results[0].text).toContain('hello world');
+    });
+
+    it('expands clickable elements to extract collapsed content', async () => {
+        const { results, failedUrls } = await scrapeWithPlaywright(`${baseUrl}/clickable`, {
+            dynamicContentWaitSecs: 2,
+        });
+
+        expect(failedUrls.size).toBe(0);
+        expect(results).toHaveLength(1);
+        expect(results[0].text).toContain('always visible content');
+        // The panel content is added to the DOM only after the toggle is clicked
+        expect(results[0].text).toContain('collapsed panel content');
+        // Links to other pages must not be clicked, i.e. we must stay on the original page
+        expect(results[0].text).not.toContain('hello world');
+    });
+
+    it('returns content of the original page when clicking navigates away', async () => {
+        const { results, failedUrls } = await scrapeWithPlaywright(`${baseUrl}/clickable-js-navigation`, {
+            dynamicContentWaitSecs: 2,
+        });
+
+        // A navigation triggered by JavaScript cannot be prevented by the selector, so the page is
+        // loaded again instead of returning the content of the page it navigated to
         expect(failedUrls.size).toBe(0);
-        expect(successUrls.size).toBe(1);
+        expect(results).toHaveLength(1);
+        expect(results[0].text).toContain('page navigating on click');
+        expect(results[0].text).not.toContain('hello world');
     });
 });
diff --git a/tests/standby.test.ts b/tests/standby.test.ts
index 5abc511..e2f7c8d 100644
--- a/tests/standby.test.ts
+++ b/tests/standby.test.ts
@@ -76,6 +76,16 @@ describe('Standby RAG tests', () => {
         expect(data[0].markdown).toContain('hello world');
     });
 
+    it('standby request playwright expands clickable elements', async () => {
+        const response = await fetch(`http://localhost:${browserServerPort}/search?query=${baseUrl}/clickable&scrapingTool=browser-playwright`);
+        const data = await response.json();
+
+        expect(response.status).toBe(200);
+        expect(data[0].metadata.url).toBe(`${baseUrl}/clickable`);
+        // The panel content is added to the DOM only after the collapsed element is clicked
+        expect(data[0].markdown).toContain('collapsed panel content');
+    });
+
     it('standby request with a media file URL is skipped without downloading it', async () => {
         resetImageRequestCount();
 
diff --git a/vitest.config.ts b/vitest.config.ts
index af08605..3526d8d 100644
--- a/vitest.config.ts
+++ b/vitest.config.ts
@@ -2,7 +2,8 @@ import { defineConfig } from 'vitest/config';
 
 export default defineConfig({
     test: {
-        testTimeout: 15000,
+        // Generous timeout because most of the tests launch a browser, which is slow on a cold start
+        testTimeout: 30000,
         globals: true,
         environment: 'node',
         include: ['tests/**/*.test.ts'],

From 1c14aebab72ceb7dd6d20610cae45528573f2591 Mon Sep 17 00:00:00 2001
From: nikitachapovskii-dev <nikita.chapovskii@apify.com>
Date: Mon, 3 Aug 2026 15:27:48 +0200
Subject: [PATCH 2/2] chore: simplify the clickable elements expansion and
 document it

---
 actors/apify_rag-web-browser/README.md        | 14 +++++++
 actors/apify_url-to-markdown/README.md        |  3 ++
 src/request-handler.ts                        | 38 +++++------------
 tests/cheerio-crawler.content.test.ts         | 41 ++++++-------------
 .../helpers/html/clickable-js-navigation.html | 14 -------
 tests/helpers/html/clickable.html             | 33 ++++++++++-----
 tests/helpers/server.ts                       |  4 --
 tests/playwright-crawler.content.test.ts      | 22 +++-------
 tests/standby.test.ts                         | 10 -----
 9 files changed, 67 insertions(+), 112 deletions(-)
 delete mode 100644 tests/helpers/html/clickable-js-navigation.html

diff --git a/actors/apify_rag-web-browser/README.md b/actors/apify_rag-web-browser/README.md
index 94d7551..fde51e6 100644
--- a/actors/apify_rag-web-browser/README.md
+++ b/actors/apify_rag-web-browser/README.md
@@ -16,6 +16,7 @@ The extracted text can then be injected into prompts and retrieval augmented gen
 - 🕷 Automatically **bypasses anti-scraping protections** using proxies and browser fingerprints
 - 📝 Output formats include **Markdown**, plain text, and HTML
 - 🔗 **Links are converted to absolute URLs**, so they stay valid outside of the page they came from
+- 🪗 **Collapsed sections are expanded** in Browser mode, so their content is not missing from the output
 - 🔌 Supports **OpenAPI and MCP** for easy integration
 - 🪟 It's **open source**, so you can review and modify it
 
@@ -234,6 +235,19 @@ Media files carry no text for the LLM, so the Actor never downloads them:
 - Search results (and a `query` that is a URL) pointing directly to a media file, e.g. `https://example.com/video.mp4`,
   are not crawled at all. Such a result is returned with an empty text and `Skipped media file` as the HTTP status message.
 
+### Collapsed content
+
+Some web pages keep parts of their content hidden until the reader expands them, e.g. accordions or
+FAQ sections, and add it to the page only when it's clicked. To capture such content, the Actor clicks
+the collapsed elements of the page, i.e. those matching the `[aria-expanded="false"]` CSS selector,
+in Browser mode (`scrapingTool=browser-playwright`) before it extracts the content.
+
+Elements linking to another page are not clicked, so that the Actor stays on the page it was asked
+to extract.
+
+Note that clicking costs about half a second on pages that have such elements, and that content of
+expanded navigation menus can end up in the output as well.
+
 ### Reducing response time
 
 For low-latency applications, it's recommended to run the RAG Web Browser in Standby mode
diff --git a/actors/apify_url-to-markdown/README.md b/actors/apify_url-to-markdown/README.md
index 95bd910..fdd5b30 100644
--- a/actors/apify_url-to-markdown/README.md
+++ b/actors/apify_url-to-markdown/README.md
@@ -88,6 +88,9 @@ Relative links such as `/docs` are resolved into absolute URLs against the page
 ### What happens with media files?
 Media files carry no text to convert, so they are never downloaded. Images, audio, video, and fonts of the converted page are blocked in the Browser mode, and a URL pointing directly to a media file, e.g. `https://example.com/video.mp4`, is not fetched at all — the output contains no content and `Skipped media file` as the HTTP status message.
 
+### What happens with collapsed content?
+Content that a page adds only when the reader expands it, e.g. accordions or FAQ sections, would be missing from the markdown. To capture it, the Browser mode clicks the collapsed elements of the page, i.e. those matching the `[aria-expanded="false"]` CSS selector, before converting it. Elements linking to another page are not clicked, so that the Actor stays on the page it was asked to convert.
+
 ### Can I use URL to Markdown with the Apify API?
 The Apify API gives you programmatic access to the Apify platform. The API is organized around RESTful HTTP endpoints that enable you to manage, schedule, and run Apify Actors. The API also lets you access any datasets, monitor Actor performance, fetch results, create and update versions, and more.
 
diff --git a/src/request-handler.ts b/src/request-handler.ts
index fd29779..e505023 100644
--- a/src/request-handler.ts
+++ b/src/request-handler.ts
@@ -11,10 +11,9 @@ import { addTimeMeasureEvent, isActorStandby, transformTimeMeasuresToRelative }
 import { processHtml } from './website-content-crawler/html-processing.js';
 import { htmlToMarkdown } from './website-content-crawler/markdown.js';
 
-/** Collapsed elements that are clicked to expand their content, same default as Website Content Crawler. */
+/** Same as the default of the `clickElementsCssSelector` input of Website Content Crawler. */
 const CLICK_ELEMENTS_CSS_SELECTOR = '[aria-expanded="false"]';
 
-/** How long to wait for the content expanded by clicking to render. */
 const CLICK_RENDER_WAIT_MS = 500;
 
 let ACTOR_TIMEOUT_AT: number | undefined;
@@ -80,44 +79,27 @@ export async function waitForDynamicContent(context: PlaywrightCrawlingContext,
  * Tries to expand collapsed content by clicking on it, so that its text is included in the extracted
  * content, e.g. https://www.checkout.com/docs/support/reporting (adapted from: Website Content Crawler).
  *
- * Failed clicks are only logged, the content is extracted with whatever got expanded. If the clicking
- * navigates the page away, the page is loaded again, so that we never extract a different page.
+ * A click handler can still navigate the page with JavaScript, and then the content is extracted from
+ * the page it navigated to, the same as in Website Content Crawler.
  */
 async function expandClickableElements(page: PlaywrightCrawlingContext['page'], cssSelector: string) {
-    const urlBeforeClicking = page.url();
-
     const clickedCount = await page.evaluate((selector) => {
-        // Only click on elements that don't have the `href` attribute or that lead to the current page,
-        // so that we don't navigate away from the page
+        // only click on items that don't have `href` attribute or they lead to the current page
         const elements = [...document.querySelectorAll(selector)].filter((el) => {
             const href = el.getAttribute('href');
-            return !href || href.startsWith('#');
+            return (!href || href.startsWith('#')) && typeof (el as HTMLElement).click === 'function';
         });
 
         for (const el of elements) {
-            (el as HTMLElement).click?.();
+            (el as HTMLElement).click();
         }
 
         return elements.length;
-    }, cssSelector).catch((err: unknown) => {
-        log.warning(`Failed to expand clickable elements: ${err instanceof Error ? err.message : String(err)}`);
-        // Some of the elements might have been clicked before the failure
-        return null;
-    });
-
-    // Nothing was clicked, so there is nothing to wait for and the page could not have navigated
-    if (clickedCount === 0) {
-        return;
-    }
-
-    log.debug(`Clicked ${clickedCount ?? 'some'} element(s) matching \`${cssSelector}\``);
-    await sleep(CLICK_RENDER_WAIT_MS);
+    }, cssSelector);
 
-    // A click handler can navigate the page with JavaScript, which no CSS selector can guard against.
-    // Content of a different page would be worse than content without the expanded sections, so undo it.
-    if (page.url().split('#')[0] !== urlBeforeClicking.split('#')[0]) {
-        log.warning(`Clicking navigated the page to ${page.url()}, loading ${urlBeforeClicking} again`);
-        await page.goto(urlBeforeClicking, { waitUntil: 'domcontentloaded' });
+    if (clickedCount > 0) {
+        log.debug(`Clicked ${clickedCount} element(s) matching \`${cssSelector}\``);
+        await sleep(CLICK_RENDER_WAIT_MS);
     }
 }
 
diff --git a/tests/cheerio-crawler.content.test.ts b/tests/cheerio-crawler.content.test.ts
index 4208323..b47706e 100644
--- a/tests/cheerio-crawler.content.test.ts
+++ b/tests/cheerio-crawler.content.test.ts
@@ -6,7 +6,7 @@ import { CheerioCrawler, type CheerioCrawlingContext, Configuration, log } from
 import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
 
 import { requestHandlerCheerio } from '../src/request-handler.js';
-import type { ContentCrawlerUserData, Output } from '../src/types.js';
+import type { ContentCrawlerUserData } from '../src/types.js';
 import { createRequest } from '../src/utils.js';
 import { startTestServer, stopTestServer } from './helpers/server.js';
 
@@ -25,13 +25,9 @@ describe('Cheerio Crawler Content Tests', () => {
         await stopTestServer(testServer);
     });
 
-    /**
-     * Scrapes a single URL with the Cheerio request handler and returns the results it pushed
-     * to the dataset, along with the URLs that failed.
-     */
-    async function scrapeWithCheerio(url: string) {
-        const results: Output[] = [];
+    it('test basic content extraction with cheerio', async () => {
         const failedUrls = new Set<string>();
+        const successUrls = new Set<string>();
 
         // Create memory storage and request queue
         const client = new MemoryStorage({ persistStorage: false });
@@ -40,10 +36,14 @@ describe('Cheerio Crawler Content Tests', () => {
         const crawler = new CheerioCrawler({
             requestQueue,
             requestHandler: async (context: CheerioCrawlingContext<ContentCrawlerUserData>) => {
-                vi.spyOn(context, 'pushData').mockImplementation(async (data) => {
-                    results.push(data as Output);
-                });
+                const pushDataSpy = vi.spyOn(context, 'pushData').mockResolvedValue(undefined);
                 await requestHandlerCheerio(context);
+
+                expect(pushDataSpy).toHaveBeenCalledTimes(1);
+                expect(pushDataSpy).toHaveBeenCalledWith(expect.objectContaining({
+                    text: expect.stringContaining('hello world'),
+                }));
+                successUrls.add(context.request.url);
             },
             failedRequestHandler: async ({ request }, error) => {
                 log.error(`Request ${request.url} failed with error: ${error.message}`);
@@ -56,7 +56,7 @@ describe('Cheerio Crawler Content Tests', () => {
         const r = createRequest(
             'query',
             {
-                url,
+                url: `${baseUrl}/basic`,
                 description: 'Test request',
                 rank: 1,
                 title: 'Test title',
@@ -76,24 +76,7 @@ describe('Cheerio Crawler Content Tests', () => {
 
         await crawler.run();
 
-        return { results, failedUrls };
-    }
-
-    it('test basic content extraction with cheerio', async () => {
-        const { results, failedUrls } = await scrapeWithCheerio(`${baseUrl}/basic`);
-
-        expect(failedUrls.size).toBe(0);
-        expect(results).toHaveLength(1);
-        expect(results[0].text).toContain('hello world');
-    });
-
-    it('does not expand clickable elements in the raw HTTP mode', async () => {
-        const { results, failedUrls } = await scrapeWithCheerio(`${baseUrl}/clickable`);
-
         expect(failedUrls.size).toBe(0);
-        expect(results).toHaveLength(1);
-        expect(results[0].text).toContain('always visible content');
-        // Clicking needs a browser, so the panel content never makes it into the DOM
-        expect(results[0].text).not.toContain('collapsed panel content');
+        expect(successUrls.size).toBe(1);
     });
 });
diff --git a/tests/helpers/html/clickable-js-navigation.html b/tests/helpers/html/clickable-js-navigation.html
deleted file mode 100644
index c3c9bd7..0000000
--- a/tests/helpers/html/clickable-js-navigation.html
+++ /dev/null
@@ -1,14 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Clickable JS Navigation Test Page</title>
-</head>
-<body>
-    <p>page navigating on click</p>
-
-    <!-- There is no `href` to filter on, the click handler navigates the page with JavaScript -->
-    <button aria-expanded="false" onclick="window.location.href = '/basic'">Go elsewhere</button>
-</body>
-</html>
diff --git a/tests/helpers/html/clickable.html b/tests/helpers/html/clickable.html
index a8c1c41..797e2b0 100644
--- a/tests/helpers/html/clickable.html
+++ b/tests/helpers/html/clickable.html
@@ -8,21 +8,34 @@
 <body>
     <p>always visible content</p>
 
-    <!-- The panel content is added to the DOM only after the toggle is clicked -->
-    <button id="toggle" aria-expanded="false" aria-controls="panel">Show details</button>
+    <button id="toggle">Show details</button>
     <div id="panel"></div>
 
-    <!-- Links leading to another page must not be clicked, otherwise we navigate away from the page -->
-    <a id="link" href="/basic" aria-expanded="false">Link to another page</a>
+    <a id="anchor" href="#anchor-panel">Show more</a>
+    <div id="anchor-panel"></div>
 
-    <!-- Same-page anchors are clicked, they only change the URL hash -->
-    <a id="anchor" href="#panel" aria-expanded="false">Anchor to the panel</a>
+    <a id="link" href="/basic#section">Link to another page</a>
+    <div id="link-panel"></div>
 
     <script>
-        document.getElementById('toggle').addEventListener('click', (event) => {
-            event.currentTarget.setAttribute('aria-expanded', 'true');
-            document.getElementById('panel').textContent = 'collapsed panel content';
-        });
+        // Collapsed content is added to the page only when clicked, and takes a moment to render
+        const reveal = (panelId, text) => {
+            window.setTimeout(() => {
+                document.getElementById(panelId).textContent = text;
+            }, 150);
+        };
+
+        document.getElementById('toggle').addEventListener('click', () => reveal('panel', 'collapsed panel content'));
+        document.getElementById('anchor').addEventListener('click', () => reveal('anchor-panel', 'anchor panel content'));
+        document.getElementById('link').addEventListener('click', () => reveal('link-panel', 'link panel content'));
+
+        // The elements become collapsed only once the page is interactive, so they cannot be clicked
+        // before the Actor waits for the dynamic content
+        window.setTimeout(() => {
+            for (const id of ['toggle', 'anchor', 'link']) {
+                document.getElementById(id).setAttribute('aria-expanded', 'false');
+            }
+        }, 200);
     </script>
 </body>
 </html>
diff --git a/tests/helpers/server.ts b/tests/helpers/server.ts
index 5111385..2e11cb3 100644
--- a/tests/helpers/server.ts
+++ b/tests/helpers/server.ts
@@ -34,10 +34,6 @@ export function createTestServer() {
         sendHtml('clickable.html', res);
     });
 
-    app.get('/clickable-js-navigation', (_req, res) => {
-        sendHtml('clickable-js-navigation.html', res);
-    });
-
     app.get('/with-image', (_req, res) => {
         sendHtml('with-image.html', res);
     });
diff --git a/tests/playwright-crawler.content.test.ts b/tests/playwright-crawler.content.test.ts
index 095fbf4..309d69e 100644
--- a/tests/playwright-crawler.content.test.ts
+++ b/tests/playwright-crawler.content.test.ts
@@ -28,8 +28,7 @@ describe('Playwright Crawler Content Tests', () => {
     });
 
     /**
-     * Scrapes a single URL with the Playwright request handler and returns the results it pushed
-     * to the dataset, along with the URLs that failed.
+     * Scrapes a single URL and returns the results the request handler pushed to the dataset.
      */
     async function scrapeWithPlaywright(url: string, settings: Partial<ContentScraperSettings> = {}) {
         const results: Output[] = [];
@@ -105,22 +104,11 @@ describe('Playwright Crawler Content Tests', () => {
         expect(failedUrls.size).toBe(0);
         expect(results).toHaveLength(1);
         expect(results[0].text).toContain('always visible content');
-        // The panel content is added to the DOM only after the toggle is clicked
+        // Content of every collapsed element, which is added to the page only once it's clicked
         expect(results[0].text).toContain('collapsed panel content');
-        // Links to other pages must not be clicked, i.e. we must stay on the original page
-        expect(results[0].text).not.toContain('hello world');
-    });
-
-    it('returns content of the original page when clicking navigates away', async () => {
-        const { results, failedUrls } = await scrapeWithPlaywright(`${baseUrl}/clickable-js-navigation`, {
-            dynamicContentWaitSecs: 2,
-        });
-
-        // A navigation triggered by JavaScript cannot be prevented by the selector, so the page is
-        // loaded again instead of returning the content of the page it navigated to
-        expect(failedUrls.size).toBe(0);
-        expect(results).toHaveLength(1);
-        expect(results[0].text).toContain('page navigating on click');
+        expect(results[0].text).toContain('anchor panel content');
+        // The link leading to another page must not be clicked, not even to its fragment
+        expect(results[0].text).not.toContain('link panel content');
         expect(results[0].text).not.toContain('hello world');
     });
 });
diff --git a/tests/standby.test.ts b/tests/standby.test.ts
index e2f7c8d..5abc511 100644
--- a/tests/standby.test.ts
+++ b/tests/standby.test.ts
@@ -76,16 +76,6 @@ describe('Standby RAG tests', () => {
         expect(data[0].markdown).toContain('hello world');
     });
 
-    it('standby request playwright expands clickable elements', async () => {
-        const response = await fetch(`http://localhost:${browserServerPort}/search?query=${baseUrl}/clickable&scrapingTool=browser-playwright`);
-        const data = await response.json();
-
-        expect(response.status).toBe(200);
-        expect(data[0].metadata.url).toBe(`${baseUrl}/clickable`);
-        // The panel content is added to the DOM only after the collapsed element is clicked
-        expect(data[0].markdown).toContain('collapsed panel content');
-    });
-
     it('standby request with a media file URL is skipped without downloading it', async () => {
         resetImageRequestCount();