From cace94c2c0e0240a0d1d982dab6fddafcf1ce77e Mon Sep 17 00:00:00 2001 From: nikitachapovskii-dev Date: Mon, 3 Aug 2026 16:47:06 +0200 Subject: [PATCH] chore: use impit as the raw-http HTTP client --- Dockerfile | 8 +- package-lock.json | 179 +++++++++++++++++++++++++- package.json | 3 +- src/crawlers.ts | 8 ++ tests/cheerio-crawler.content.test.ts | 12 ++ 5 files changed, 204 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 80562b8..dcc2280 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,11 +33,11 @@ COPY --chown=myuser package*.json ./ COPY --chown=myuser policies.json ./ COPY --chown=myuser patches ./patches -# Install NPM packages, skip optional and development dependencies to -# keep the image small. Avoid logging too much and print the dependency -# tree for debugging +# Install NPM packages, skip development dependencies to keep the image small. Avoid logging too +# much and print the dependency tree for debugging. Optional dependencies are needed - `impit` +# ships its native bindings as one. RUN npm --quiet set progress=false \ - && npm install --omit=dev --omit=optional \ + && npm install --omit=dev \ && echo "Installed NPM packages:" \ && (npm list --omit=dev --all || true) \ && echo "Node.js version:" \ diff --git a/package-lock.json b/package-lock.json index 8b558d6..63183d7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,7 @@ "hasInstallScript": true, "license": "ISC", "dependencies": { + "@crawlee/impit-client": "^3.17.0", "@crawlee/memory-storage": "^3.11.1", "@ghostery/adblocker-playwright": "^2.5.2", "@modelcontextprotocol/sdk": "^1.0.4", @@ -43,7 +44,7 @@ "vitest": "^3.0.5" }, "engines": { - "node": ">=18.0.0" + "node": ">=20.0.0" } }, "node_modules/@apify/consts": { @@ -500,6 +501,23 @@ "entities": "^4.4.0" } }, + "node_modules/@crawlee/impit-client": { + "version": "3.17.0", + "resolved": "https://registry.npmjs.org/@crawlee/impit-client/-/impit-client-3.17.0.tgz", + "integrity": "sha512-Is8AvZF/8yveWa/Mwp6+aKvdv8VtPjm8MUP5JbP0mpRbxi4qmciNM/GJYB3FbuvI9taueFoeHZeZm2myfyXb+g==", + "license": "Apache-2.0", + "dependencies": { + "@apify/datastructures": "^2.0.3", + "impit": "^0.14.1", + "tough-cookie": "^6.0.0" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "@crawlee/core": "^3.12.1" + } + }, "node_modules/@crawlee/jsdom": { "version": "3.17.0", "resolved": "https://registry.npmjs.org/@crawlee/jsdom/-/jsdom-3.17.0.tgz", @@ -6571,6 +6589,165 @@ "node": ">= 4" } }, + "node_modules/impit": { + "version": "0.14.3", + "resolved": "https://registry.npmjs.org/impit/-/impit-0.14.3.tgz", + "integrity": "sha512-SbCLoeW0YDRox5kQoy71jtpjZE+BwjWO411OdgfbLMbNmSX79s5Y0Y2vCE7AO6zQSq9F4Fu8SJTCKnp8rXQzpg==", + "license": "Apache-2.0", + "engines": { + "node": ">= 20" + }, + "optionalDependencies": { + "impit-darwin-arm64": "0.14.3", + "impit-darwin-x64": "0.14.3", + "impit-linux-arm64-gnu": "0.14.3", + "impit-linux-arm64-musl": "0.14.3", + "impit-linux-x64-gnu": "0.14.3", + "impit-linux-x64-musl": "0.14.3", + "impit-win32-arm64-msvc": "0.14.3", + "impit-win32-x64-msvc": "0.14.3" + } + }, + "node_modules/impit-darwin-arm64": { + "version": "0.14.3", + "resolved": "https://registry.npmjs.org/impit-darwin-arm64/-/impit-darwin-arm64-0.14.3.tgz", + "integrity": "sha512-kMoQB+CR+a954pnhe4kf1O2RyJCsqGRE95jIXfgqNOiIMS5O1z0cOMzG/sohJk/nodZVSJ5VdWEV4BKhRWPFSA==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/impit-darwin-x64": { + "version": "0.14.3", + "resolved": "https://registry.npmjs.org/impit-darwin-x64/-/impit-darwin-x64-0.14.3.tgz", + "integrity": "sha512-ft9+kjz1pR8H5xbbf5U1EgwaaIea5iXHxBf2Q3NoinPpiV7KgyzYSr83pCSrAY6evWk+smG9shHgzhQtgbj1Rg==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/impit-linux-arm64-gnu": { + "version": "0.14.3", + "resolved": "https://registry.npmjs.org/impit-linux-arm64-gnu/-/impit-linux-arm64-gnu-0.14.3.tgz", + "integrity": "sha512-+mwta93S6Ndfpca3DDblvrqV1g8Bg6gZFOlEiJHfGJZpiUIQE+A/Pjg2e+inV+WrTtlnLsvff8lgwFvkIGnRug==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/impit-linux-arm64-musl": { + "version": "0.14.3", + "resolved": "https://registry.npmjs.org/impit-linux-arm64-musl/-/impit-linux-arm64-musl-0.14.3.tgz", + "integrity": "sha512-wYhfH1J8laWkpSN2SAFCKx9npY4vXrk23ex+tOzTGf2xBBqIhpMUff2wa6dOFb0or6EhZHBssoRf0HK3h/htXQ==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/impit-linux-x64-gnu": { + "version": "0.14.3", + "resolved": "https://registry.npmjs.org/impit-linux-x64-gnu/-/impit-linux-x64-gnu-0.14.3.tgz", + "integrity": "sha512-hAYnutJDGQO5bPvTPKBBFkHgbB7QL7QLDse8c4s21VSzC/EqGliSS3UqR5ZUUwaFdm3t8DJsXOr+gboAUwDR7w==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/impit-linux-x64-musl": { + "version": "0.14.3", + "resolved": "https://registry.npmjs.org/impit-linux-x64-musl/-/impit-linux-x64-musl-0.14.3.tgz", + "integrity": "sha512-wXBeHiqCjuyqxg4u5eY4OBNVzsS4Karz/ms/0mjB8aqVZLNER1NaUTeE4J3eZCdytFaJRaAeQvKTH8HfjpPsQg==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/impit-win32-arm64-msvc": { + "version": "0.14.3", + "resolved": "https://registry.npmjs.org/impit-win32-arm64-msvc/-/impit-win32-arm64-msvc-0.14.3.tgz", + "integrity": "sha512-XiF8MYpm4tF8TMbtsWrcEDDs6NxRNeirfiHM/AyKcr2jiN34hZwbZDM0EseD30PDwb2Pny2JWB+zTxpE1P5CZg==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/impit-win32-x64-msvc": { + "version": "0.14.3", + "resolved": "https://registry.npmjs.org/impit-win32-x64-msvc/-/impit-win32-x64-msvc-0.14.3.tgz", + "integrity": "sha512-HIaLSpU5SGVx3UDH/R3ZaGMURVgyRVjmalqXj65ABxVUqfXiBo1t5ZObDPPjqvLT3klPnrya1+xF1DQ4dfT+og==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, "node_modules/import-fresh": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", diff --git a/package.json b/package.json index 0fb621e..3c7204a 100644 --- a/package.json +++ b/package.json @@ -4,9 +4,10 @@ "type": "module", "description": "RAG Web Browser - run Google Search queries and extract content from the top search results.", "engines": { - "node": ">=18.0.0" + "node": ">=20.0.0" }, "dependencies": { + "@crawlee/impit-client": "^3.17.0", "@crawlee/memory-storage": "^3.11.1", "@ghostery/adblocker-playwright": "^2.5.2", "@modelcontextprotocol/sdk": "^1.0.4", diff --git a/src/crawlers.ts b/src/crawlers.ts index 0efcaf6..9515bd3 100644 --- a/src/crawlers.ts +++ b/src/crawlers.ts @@ -1,5 +1,6 @@ import { readFile } from 'node:fs/promises'; +import { ImpitHttpClient } from '@crawlee/impit-client'; import { MemoryStorage } from '@crawlee/memory-storage'; import { PlaywrightBlocker } from '@ghostery/adblocker-playwright'; import { Actor, RequestQueue } from 'apify'; @@ -26,6 +27,12 @@ import { addTimeMeasureEvent, createRequest, createSearchRequest, isActorStandby const crawlers = new Map(); const client = new MemoryStorage({ persistStorage: false }); +const contentCrawlerHttpClient = new ImpitHttpClient({ + browser: 'firefox144', + vanillaFallback: true, + ignoreTlsErrors: true, +}); + let ghosteryBlocker: PlaywrightBlocker | undefined; async function getGhosteryBlocker(): Promise { @@ -241,6 +248,7 @@ async function createCheerioContentCrawler( return new CheerioCrawler({ ...crawlerOptions, keepAlive: crawlerOptions.keepAlive, + httpClient: contentCrawlerHttpClient, requestQueue: await RequestQueue.open(key, { storageClient: client }), requestHandler: (async (context) => { const typedContext = context as unknown as CheerioCrawlingContext; diff --git a/tests/cheerio-crawler.content.test.ts b/tests/cheerio-crawler.content.test.ts index 484bb60..ec615d6 100644 --- a/tests/cheerio-crawler.content.test.ts +++ b/tests/cheerio-crawler.content.test.ts @@ -1,10 +1,13 @@ import type { Server } from 'node:http'; +import { ImpitHttpClient } from '@crawlee/impit-client'; import { MemoryStorage } from '@crawlee/memory-storage'; import { RequestQueue } from 'apify'; import { CheerioCrawler, type CheerioCrawlingContext, Configuration, log } from 'crawlee'; import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; +import { ContentCrawlerTypes } from '../src/const.js'; +import { createAndStartContentCrawler } from '../src/crawlers.js'; import { requestHandlerCheerio } from '../src/request-handler.js'; import type { ContentCrawlerUserData } from '../src/types.js'; import { createRequest } from '../src/utils.js'; @@ -82,4 +85,13 @@ describe('Cheerio Crawler Content Tests', () => { expect(failedUrls.size).toBe(0); expect(successUrls.size).toBe(1); }); + + it('test the crawler is created with the impit HTTP client', async () => { + const { crawler } = await createAndStartContentCrawler( + { type: ContentCrawlerTypes.CHEERIO, crawlerOptions: {} }, + false, + ); + + expect(Reflect.get(crawler!, 'httpClient')).toBeInstanceOf(ImpitHttpClient); + }); });