chore: use impit as the raw-http HTTP client

Dockerfile

@@ -33,11 +33,11 @@ COPY --chown=myuser package*.json ./
 COPY --chown=myuser policies.json ./
 COPY --chown=myuser patches ./patches
 
-# Install NPM packages, skip optional and development dependencies to
-# keep the image small. Avoid logging too much and print the dependency
-# tree for debugging
+# Install NPM packages, skip development dependencies to keep the image small. Avoid logging too
+# much and print the dependency tree for debugging. Optional dependencies are needed - `impit`
+# ships its native bindings as one.
 RUN npm --quiet set progress=false \
-    && npm install --omit=dev --omit=optional \
+    && npm install --omit=dev \
     && echo "Installed NPM packages:" \
     && (npm list --omit=dev --all || true) \
     && echo "Node.js version:" \

package.json

@@ -4,9 +4,10 @@
     "type": "module",
     "description": "RAG Web Browser - run Google Search queries and extract content from the top search results.",
     "engines": {
-        "node": ">=18.0.0"
+        "node": ">=20.0.0"
     },
     "dependencies": {
+        "@crawlee/impit-client": "^3.17.0",
         "@crawlee/memory-storage": "^3.11.1",
         "@ghostery/adblocker-playwright": "^2.5.2",
         "@modelcontextprotocol/sdk": "^1.0.4",

src/crawlers.ts

@@ -1,5 +1,6 @@
 import { readFile } from 'node:fs/promises';
 
+import { ImpitHttpClient } from '@crawlee/impit-client';
 import { MemoryStorage } from '@crawlee/memory-storage';
 import { PlaywrightBlocker } from '@ghostery/adblocker-playwright';
 import { Actor, RequestQueue } from 'apify';
@@ -26,6 +27,12 @@ import { addTimeMeasureEvent, createRequest, createSearchRequest, isActorStandby
 const crawlers = new Map<string, CheerioCrawler | PlaywrightCrawler>();
 const client = new MemoryStorage({ persistStorage: false });
 
+const contentCrawlerHttpClient = new ImpitHttpClient({
+    browser: 'firefox144',
+    vanillaFallback: true,
+    ignoreTlsErrors: true,
+});
+
 let ghosteryBlocker: PlaywrightBlocker | undefined;
 
 async function getGhosteryBlocker(): Promise<PlaywrightBlocker | undefined> {
@@ -241,6 +248,7 @@ async function createCheerioContentCrawler(
     return new CheerioCrawler({
         ...crawlerOptions,
         keepAlive: crawlerOptions.keepAlive,
+        httpClient: contentCrawlerHttpClient,
         requestQueue: await RequestQueue.open(key, { storageClient: client }),
         requestHandler: (async (context) => {
             const typedContext = context as unknown as CheerioCrawlingContext<ContentCrawlerUserData>;

tests/cheerio-crawler.content.test.ts

@@ -1,10 +1,13 @@
 import type { Server } from 'node:http';
 
+import { ImpitHttpClient } from '@crawlee/impit-client';
 import { MemoryStorage } from '@crawlee/memory-storage';
 import { RequestQueue } from 'apify';
 import { CheerioCrawler, type CheerioCrawlingContext, Configuration, log } from 'crawlee';
 import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
 
+import { ContentCrawlerTypes } from '../src/const.js';
+import { createAndStartContentCrawler } from '../src/crawlers.js';
 import { requestHandlerCheerio } from '../src/request-handler.js';
 import type { ContentCrawlerUserData } from '../src/types.js';
 import { createRequest } from '../src/utils.js';
@@ -82,4 +85,13 @@ describe('Cheerio Crawler Content Tests', () => {
         expect(failedUrls.size).toBe(0);
         expect(successUrls.size).toBe(1);
     });
+
+    it('test the crawler is created with the impit HTTP client', async () => {
+        const { crawler } = await createAndStartContentCrawler(
+            { type: ContentCrawlerTypes.CHEERIO, crawlerOptions: {} },
+            false,
+        );
+
+        expect(Reflect.get(crawler!, 'httpClient')).toBeInstanceOf(ImpitHttpClient);
+    });
 });