perf: don't download media files

actors/apify_rag-web-browser/README.md

@@ -224,6 +224,15 @@ Here are specific situations that might occur when the timeout is reached:
   => the Actor extracts content from the currently loaded HTML
 
 
+### Media files
+
+Media files carry no text for the LLM, so the Actor never downloads them:
+
+- Images, audio, video, and fonts of the scraped page are blocked in Browser mode (`scrapingTool=browser-playwright`).
+  This saves bandwidth and often speeds up the page load, and it has no effect on the extracted content.
+- Search results (and a `query` that is a URL) pointing directly to a media file, e.g. `https://example.com/video.mp4`,
+  are not crawled at all. Such a result is returned with an empty text and `Skipped media file` as the HTTP status message.
+
 ### Reducing response time
 
 For low-latency applications, it's recommended to run the RAG Web Browser in Standby mode

actors/apify_url-to-markdown/README.md

@@ -83,6 +83,9 @@ Markdown is the perfect format to feed large language model (LLM). It is a less
 Using markdown instead of html can help you lower the AI token cost.
 
 
+### What happens with media files?
+Media files carry no text to convert, so they are never downloaded. Images, audio, video, and fonts of the converted page are blocked in the Browser mode, and a URL pointing directly to a media file, e.g. `https://example.com/video.mp4`, is not fetched at all — the output contains no content and `Skipped media file` as the HTTP status message.
+
 ### Can I use URL to Markdown with the Apify API?
 The Apify API gives you programmatic access to the Apify platform. The API is organized around RESTful HTTP endpoints that enable you to manage, schedule, and run Apify Actors. The API also lets you access any datasets, monitor Actor performance, fetch results, create and update versions, and more.
 

src/input.ts

@@ -7,6 +7,7 @@ import { firefox } from 'playwright';
 import ragWebBrowserInputSchema from '../actors/apify_rag-web-browser/.actor/input_schema.json' with { type: 'json' };
 import { ContentCrawlerTypes } from './const.js';
 import { UserInputError } from './errors.js';
+import { blockMediaRequests } from './media.js';
 import { getMiniActor } from './mini-actors.js';
 import type {
     ContentCrawlerOptions,
@@ -228,6 +229,9 @@ function createPlaywrightCrawlerOptions(
                 launcher: firefox,
             },
             preNavigationHooks: [
+                async ({ page }) => {
+                    await blockMediaRequests(page);
+                },
                 (_context, gotoOptions) => {
                     // eslint-disable-next-line no-param-reassign
                     gotoOptions.waitUntil = 'domcontentloaded';

src/media.ts

@@ -0,0 +1,69 @@
+import { log } from 'crawlee';
+import type { Page } from 'playwright';
+
+/**
+ * Extensions of media files (images, audio, video and fonts). Such files carry no text
+ * for us to extract, so URLs pointing to them are never downloaded.
+ */
+const MEDIA_FILE_EXTENSIONS = new Set([
+    // Images
+    'apng', 'avif', 'bmp', 'gif', 'heic', 'heif', 'ico', 'jpeg', 'jpg', 'png', 'svg', 'tif', 'tiff', 'webp',
+    // Audio
+    'aac', 'flac', 'm4a', 'mid', 'midi', 'mp3', 'oga', 'ogg', 'opus', 'wav', 'weba', 'wma',
+    // Video
+    '3gp', 'avi', 'flv', 'm4v', 'mkv', 'mov', 'mp4', 'mpeg', 'mpg', 'ogv', 'webm', 'wmv',
+    // Fonts
+    'eot', 'otf', 'ttf', 'woff', 'woff2',
+]);
+
+/**
+ * Playwright resource types that never contribute to the extracted content.
+ * Stylesheets and scripts are intentionally not blocked, as they affect the rendered page.
+ */
+const BLOCKED_RESOURCE_TYPES = new Set(['font', 'image', 'media']);
+
+/** Reported as the HTTP status message of a media file we did not download. */
+export const SKIPPED_MEDIA_FILE_MESSAGE = 'Skipped media file';
+
+/**
+ * Checks whether the URL points to a media file, based on the extension of its last path segment.
+ */
+export function isMediaUrl(url: string): boolean {
+    let pathname: string;
+    try {
+        pathname = new URL(url).pathname;
+    } catch {
+        return false;
+    }
+
+    const filename = pathname.slice(pathname.lastIndexOf('/') + 1);
+    const dotIndex = filename.lastIndexOf('.');
+    if (dotIndex === -1) return false;
+
+    return MEDIA_FILE_EXTENSIONS.has(filename.slice(dotIndex + 1).toLowerCase());
+}
+
+/**
+ * Prevents the page from downloading images, audio, video and fonts to save bandwidth.
+ * The extracted content is not affected, as these resources contain no text.
+ *
+ * Playwright invokes the route handlers in the order opposite to their registration, and only
+ * the most recently registered one is used unless it defers to the others. Therefore this is called
+ * again once the Ghostery blocker registers its own handler, and non-media requests are passed
+ * on with `route.fallback()` so that the other handlers still get a chance to process them.
+ */
+export async function blockMediaRequests(page: Page): Promise<void> {
+    try {
+        await page.route('**/*', async (route) => {
+            const isBlocked = BLOCKED_RESOURCE_TYPES.has(route.request().resourceType());
+            try {
+                await (isBlocked ? route.abort() : route.fallback());
+            } catch {
+                // The page might have been closed or navigated away in the meantime.
+            }
+        });
+        log.debug('Media request blocking enabled');
+    } catch (err) {
+        log.warning(`Failed to enable media request blocking: ${err instanceof Error ? err.message : String(err)}`);
+    }
+}

src/request-handler.ts

@@ -4,6 +4,7 @@ import { load } from 'cheerio';
 import { type CheerioCrawlingContext, htmlToText, log, type PlaywrightCrawlingContext, type Request, sleep } from 'crawlee';
 
 import { ContentCrawlerStatus, ContentCrawlerTypes } from './const.js';
+import { blockMediaRequests, SKIPPED_MEDIA_FILE_MESSAGE } from './media.js';
 import { addResultToResponse, responseData, sendResponseIfFinished } from './responses.js';
 import type { ContentCrawlerUserData, Output } from './types.js';
 import { addTimeMeasureEvent, isActorStandby, transformTimeMeasuresToRelative } from './utils.js';
@@ -69,39 +70,53 @@ export async function waitForDynamicContent(context: PlaywrightCrawlingContext,
     }
 }
 
+type ContentCrawlingContext = PlaywrightCrawlingContext<ContentCrawlerUserData> | CheerioCrawlingContext<ContentCrawlerUserData>;
+
 function isValidContentType(contentType: string | undefined) {
     return ['text', 'html', 'xml'].some((type) => contentType?.includes(type));
 }
 
+/**
+ * Stores an empty result for a page we haven't extracted any content from.
+ */
+async function pushSkippedResult(
+    context: ContentCrawlingContext,
+    httpStatusMessage: string,
+    httpStatusCode?: number,
+) {
+    const { request } = context;
+    const { responseId } = request.userData;
+
+    const resultSkipped: Output = {
+        crawl: {
+            httpStatusCode,
+            httpStatusMessage,
+            loadedAt: new Date(),
+            uniqueKey: request.uniqueKey,
+            requestStatus: ContentCrawlerStatus.FAILED,
+        },
+        metadata: { url: request.url },
+        searchResult: request.userData.searchResult!,
+        query: request.userData.query,
+        text: '',
+    };
+    log.info(`Adding result to the Apify dataset, url: ${request.url}`);
+    await context.pushData(resultSkipped);
+    if (responseId) {
+        addResultToResponse(responseId, request.uniqueKey, resultSkipped);
+        sendResponseIfFinished(responseId);
+    }
+}
+
 async function checkValidResponse(
     $: CheerioCrawlingContext['$'],
     contentType: string | undefined,
-    context: PlaywrightCrawlingContext<ContentCrawlerUserData> | CheerioCrawlingContext<ContentCrawlerUserData>,
+    statusCode: number | undefined,
+    context: ContentCrawlingContext,
 ) {
-    const { request, response } = context;
-    const { responseId } = request.userData;
-
     if (!$ || !isValidContentType(contentType)) {
-        log.info(`Skipping URL ${request.loadedUrl} as it could not be parsed.`, { contentType });
-        const resultSkipped: Output = {
-            crawl: {
-                httpStatusCode: response?.status(),
-                httpStatusMessage: "Couldn't parse the content",
-                loadedAt: new Date(),
-                uniqueKey: request.uniqueKey,
-                requestStatus: ContentCrawlerStatus.FAILED,
-            },
-            metadata: { url: request.url },
-            searchResult: request.userData.searchResult!,
-            query: request.userData.query,
-            text: '',
-        };
-        log.info(`Adding result to the Apify dataset, url: ${request.url}`);
-        await context.pushData(resultSkipped);
-        if (responseId) {
-            addResultToResponse(responseId, request.uniqueKey, resultSkipped);
-            sendResponseIfFinished(responseId);
-        }
+        log.info(`Skipping URL ${context.request.loadedUrl} as it could not be parsed.`, { contentType });
+        await pushSkippedResult(context, "Couldn't parse the content", statusCode);
         return false;
     }
 
@@ -173,6 +188,13 @@ export async function requestHandlerPlaywright(
 
     log.info(`Processing URL: ${request.url}`);
     addTimeMeasureEvent(request.userData, 'playwright-request-start');
+
+    // Media file requests are created with `skipNavigation` (see `createRequest`), so there is no page to process.
+    if (request.skipNavigation) {
+        await pushSkippedResult(context, SKIPPED_MEDIA_FILE_MESSAGE);
+        return;
+    }
+
     if (settings.dynamicContentWaitSecs > 0) {
         await waitForDynamicContent(context, settings.dynamicContentWaitSecs * 1000);
         addTimeMeasureEvent(request.userData, 'playwright-wait-dynamic-content');
@@ -184,6 +206,9 @@ export async function requestHandlerPlaywright(
             try {
                 await blocker.enableBlockingInPage(page);
                 log.debug('Ghostery blocker enabled');
+                // The Ghostery blocker continues all the requests it doesn't block, which would take
+                // precedence over the media blocking set up in the pre-navigation hook.
+                await blockMediaRequests(page);
             } catch (err) {
                 log.debug(`Ghostery blocker failed: ${err instanceof Error ? err.message : String(err)}`);
             }
@@ -205,12 +230,12 @@ export async function requestHandlerPlaywright(
     addTimeMeasureEvent(request.userData, 'playwright-parse-with-cheerio');
 
     const headers = response?.headers instanceof Function ? response.headers() : response?.headers;
+    const statusCode = response?.status();
+
     // @ts-expect-error false-positive?
-    const isValidResponse = await checkValidResponse($, headers?.['content-type'], context);
+    const isValidResponse = await checkValidResponse($, headers?.['content-type'], statusCode, context);
     if (!isValidResponse) return;
 
-    const statusCode = response?.status();
-
     await handleContent($, ContentCrawlerTypes.PLAYWRIGHT, statusCode, context);
 }
 
@@ -225,10 +250,16 @@ export async function requestHandlerCheerio(
     log.info(`Processing URL: ${request.url}`);
     addTimeMeasureEvent(request.userData, 'cheerio-request-start');
 
-    const isValidResponse = await checkValidResponse($, response.headers['content-type'], context);
-    if (!isValidResponse) return;
+    // Media file requests are created with `skipNavigation` (see `createRequest`), so there is no response.
+    if (request.skipNavigation) {
+        await pushSkippedResult(context, SKIPPED_MEDIA_FILE_MESSAGE);
+        return;
+    }
 
-    const statusCode = response?.statusCode;
+    const { statusCode } = response;
+
+    const isValidResponse = await checkValidResponse($, response.headers['content-type'], statusCode, context);
+    if (!isValidResponse) return;
 
     await handleContent($, ContentCrawlerTypes.CHEERIO, statusCode, context);
 }

src/utils.ts

@@ -7,6 +7,7 @@ import { log } from 'crawlee';
 
 import ragWebBrowserInputSchema from '../actors/apify_rag-web-browser/.actor/input_schema.json' with { type: 'json' };
 import urlToMarkdownInputSchema from '../actors/apify_url-to-markdown/.actor/input_schema.json' with { type: 'json' };
+import { isMediaUrl } from './media.js';
 import type {
     ContentCrawlerUserData,
     ContentScraperSettings,
@@ -174,6 +175,8 @@ export function createRequest(
     return {
         url: result.url!,
         uniqueKey: randomId(),
+        // Media files contain no text to extract, so don't spend any bandwidth on downloading them.
+        skipNavigation: isMediaUrl(result.url!),
         userData: {
             query,
             responseId,

tests/helpers/html/with-image.html

@@ -0,0 +1,12 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Test Page With Image</title>
+</head>
+<body>
+    hello world
+    <img src="/image.png" alt="test image">
+</body>
+</html>

tests/helpers/server.ts

@@ -4,16 +4,44 @@ import path from 'node:path';
 
 import express from 'express';
 
+/** Number of times the test image has been requested, used to verify that media files are not downloaded. */
+let imageRequestCount = 0;
+
+export function getImageRequestCount(): number {
+    return imageRequestCount;
+}
+
+export function resetImageRequestCount(): void {
+    imageRequestCount = 0;
+}
+
 /**
  * Creates and returns an Express server with test routes
  */
 export function createTestServer() {
     const app = express();
 
+    const sendHtml = (name: string, res: express.Response) => {
+        const htmlPath = path.join(__dirname, 'html', name);
+        res.send(fs.readFileSync(htmlPath, 'utf-8'));
+    };
+
     app.get('/basic', (_req, res) => {
-        const htmlPath = path.join(__dirname, 'html', 'basic.html');
-        const htmlContent = fs.readFileSync(htmlPath, 'utf-8');
-        res.send(htmlContent);
+        sendHtml('basic.html', res);
+    });
+
+    app.get('/with-image', (_req, res) => {
+        sendHtml('with-image.html', res);
+    });
+
+    app.get('/image.png', (_req, res) => {
+        imageRequestCount++;
+        // A 1x1 transparent PNG
+        const png = Buffer.from(
+            'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==',
+            'base64',
+        );
+        res.type('png').send(png);
     });
 
     return app;

tests/media.test.ts

@@ -0,0 +1,35 @@
+import { describe, expect, it } from 'vitest';
+
+import { isMediaUrl } from '../src/media.js';
+
+describe('isMediaUrl', () => {
+    it('should detect image, audio, video and font files', () => {
+        expect(isMediaUrl('https://example.com/photo.jpg')).toBe(true);
+        expect(isMediaUrl('https://example.com/assets/logo.SVG')).toBe(true);
+        expect(isMediaUrl('https://example.com/podcast/episode-1.mp3')).toBe(true);
+        expect(isMediaUrl('https://example.com/video.mp4')).toBe(true);
+        expect(isMediaUrl('https://example.com/fonts/inter.woff2')).toBe(true);
+    });
+
+    it('should detect media files with a query string or a fragment', () => {
+        expect(isMediaUrl('https://example.com/photo.png?width=100')).toBe(true);
+        expect(isMediaUrl('https://example.com/photo.png#preview')).toBe(true);
+    });
+
+    it('should not detect web pages as media files', () => {
+        expect(isMediaUrl('https://example.com')).toBe(false);
+        expect(isMediaUrl('https://example.com/article')).toBe(false);
+        expect(isMediaUrl('https://example.com/article.html')).toBe(false);
+        expect(isMediaUrl('https://example.com/article.php?image=photo.jpg')).toBe(false);
+    });
+
+    it('should only consider the extension of the last path segment', () => {
+        expect(isMediaUrl('https://example.com/photo.jpg/details')).toBe(false);
+        expect(isMediaUrl('https://example.com/v1.0/article')).toBe(false);
+    });
+
+    it('should return false for values that are not valid URLs', () => {
+        expect(isMediaUrl('')).toBe(false);
+        expect(isMediaUrl('not-a-url.mp4')).toBe(false);
+    });
+});