diff --git a/docs/04_upgrading/upgrading_v4.md b/docs/04_upgrading/upgrading_v4.md index 4718f3b336..4571940616 100644 --- a/docs/04_upgrading/upgrading_v4.md +++ b/docs/04_upgrading/upgrading_v4.md @@ -28,7 +28,7 @@ Before (v3): ```ts import { Configuration } from 'apify'; -const config = Configuration.getGlobalConfig(); +const config = Configuration.getGlobalConfiguration(); const token = config.get('token'); config.set('token', 'new-token'); ``` @@ -51,6 +51,8 @@ When a setting is exposed under several environment variables, the Apify-specifi `new Actor({ configuration })` accepts a pre-built `Configuration`, but it must be the Apify SDK's `Configuration` (imported from `apify`), not a bare Crawlee one — otherwise the `APIFY_*` / `ACTOR_*` environment variables are never resolved, so the SDK now throws if given a non-Apify instance. +`Actor.config` was renamed to `Actor.configuration` (both the static getter and the instance property), and `Configuration.getGlobalConfig()` to `Configuration.getGlobalConfiguration()`, following the same renames in Crawlee v4. The public `config` properties of `ProxyConfiguration` and `PlatformEventManager` were renamed to `configuration` as well. + ## ProxyConfiguration: `newUrl()` / `newProxyInfo()` no longer take `sessionId` The `sessionId` parameter has been removed from both `ProxyConfiguration.newUrl()` and `ProxyConfiguration.newProxyInfo()`. Each call now returns an independent URL; for Apify Proxy the SDK mints a fresh random session id internally for every URL it hands out, so consecutive calls resolve to different IPs. @@ -87,15 +89,15 @@ The `tieredProxyUrls` and `tieredProxyConfig` options on `ProxyConfigurationOpti ## EventManager -`PlatformEventManager` now extends Crawlee v4's `EventManager` and integrates with the new service locator. Use `Configuration.getGlobalConfig()` (or pass a `Configuration` instance explicitly) when constructing it directly — the constructor no longer accepts a `config` override via the `override` keyword pattern because Crawlee's base class manages the configuration through `serviceLocator` instead of a `config` field. +`PlatformEventManager` now extends Crawlee v4's `EventManager` and integrates with the new service locator. Use `Configuration.getGlobalConfiguration()` (or pass a `Configuration` instance explicitly) when constructing it directly — the constructor no longer accepts a `config` override via the `override` keyword pattern because Crawlee's base class manages the configuration through `serviceLocator` instead of a `config` field. If you only interact with events through `Actor.on()` / `Actor.off()` / `Actor.events`, no code changes are needed. -## StorageClient +## StorageBackend -The SDK's storage layer was adapted to the new Crawlee v4 `StorageClient` interface. The Apify platform client is wrapped via the `ApifyStorageClient` adapter — now exported from `apify` — which implements `createDatasetClient`, `createKeyValueStoreClient`, and `createRequestQueueClient`. +The SDK's storage layer was adapted to the new Crawlee v4 `StorageBackend` interface. The Apify platform client is wrapped via the `ApifyStorageBackend` adapter — now exported from `apify` — which implements `createDatasetBackend`, `createKeyValueStoreBackend`, and `createRequestQueueBackend`. -`Actor` wires this up for you, so most code needs no changes. But if you previously passed a raw `apify-client` `ApifyClient` straight into a Crawlee storage as its `storageClient` — which worked in v3 — it no longer does: Crawlee v4 calls `createKeyValueStoreClient()` / `createDatasetClient()`, which the raw client doesn't implement. Wrap it in `ApifyStorageClient`: +`Actor` wires this up for you, so most code needs no changes. But if you previously passed a raw `apify-client` `ApifyClient` straight into a Crawlee storage as its `storageClient` — which worked in v3 — it no longer does: Crawlee v4 calls `createKeyValueStoreBackend()` / `createDatasetBackend()`, which the raw client doesn't implement. Wrap it in `ApifyStorageBackend`: ```ts // v3 @@ -105,10 +107,21 @@ const client = new ApifyClient({ token }); const store = await KeyValueStore.open(storeId, { storageClient: client }); // v4 -import { ApifyClient, ApifyStorageClient, KeyValueStore } from 'apify'; +import { ApifyClient, ApifyStorageBackend, KeyValueStore } from 'apify'; const client = new ApifyClient({ token }); -const store = await KeyValueStore.open(storeId, { storageClient: new ApifyStorageClient(client) }); +const store = await KeyValueStore.open(storeId, { storageBackend: new ApifyStorageBackend(client) }); +``` + +### Request queue access modes + +On the platform, request queues can now be consumed in two modes, controlled by the `requestQueueAccess` option of `Actor.init()` (or of `ApifyStorageBackend` when constructing it directly): + +- `'single'` (default) assumes the run is the only consumer of its request queues. Requests are not locked server-side and the queue head is estimated locally, which means fewer (paid) API calls and better performance. Multiple producers may still add requests concurrently. +- `'shared'` locks every fetched request server-side, so several concurrent consumers (e.g. multiple Actor runs) can process one queue safely, at the cost of roughly one extra API call per request. + +```ts +await Actor.init({ requestQueueAccess: 'shared' }); ``` `KeyValueStore.getPublicUrl()` is now asynchronous (it signs URLs server-side when running on the Apify platform). Update call sites accordingly: diff --git a/package.json b/package.json index d92aee24b0..bd639b7ce1 100644 --- a/package.json +++ b/package.json @@ -73,13 +73,14 @@ }, "dependencies": { "@apify/consts": "^2.51.0", + "@apify/datastructures": "^2.0.3", "@apify/input_secrets": "^1.2.0", "@apify/log": "^2.4.3", "@apify/timeout": "^0.3.0", "@apify/utilities": "^2.13.0", - "@crawlee/core": "^4.0.0-beta.61", - "@crawlee/types": "^4.0.0-beta.61", - "@crawlee/utils": "^4.0.0-beta.61", + "@crawlee/core": "^4.0.0-beta.105", + "@crawlee/types": "^4.0.0-beta.105", + "@crawlee/utils": "^4.0.0-beta.105", "apify-client": "^2.23.4", "semver": "^7.5.4", "tslib": "^2.6.2", @@ -90,7 +91,6 @@ "@apify/oxlint-config": "^0.2.5", "@apify/tsconfig": "^0.1.2", "@commitlint/config-conventional": "^21.0.0", - "@crawlee/memory-storage": "^4.0.0-beta.61", "@playwright/browser-chromium": "^1.60.0", "@types/content-type": "^1.1.8", "@types/node": "^24.0.0", @@ -98,7 +98,7 @@ "@types/tough-cookie": "^4.0.5", "@types/ws": "^8.5.12", "commitlint": "^21.0.0", - "crawlee": "^4.0.0-beta.61", + "crawlee": "^4.0.0-beta.105", "globby": "^16.0.0", "husky": "^9.1.7", "lint-staged": "^17.0.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a188cbcb5c..98de20d192 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -16,6 +16,9 @@ importers: '@apify/consts': specifier: ^2.51.0 version: 2.52.0 + '@apify/datastructures': + specifier: ^2.0.3 + version: 2.0.3 '@apify/input_secrets': specifier: ^1.2.0 version: 1.2.28 @@ -29,14 +32,14 @@ importers: specifier: ^2.13.0 version: 2.26.0 '@crawlee/core': - specifier: ^4.0.0-beta.61 - version: 4.0.0-beta.61 + specifier: ^4.0.0-beta.105 + version: 4.0.0-beta.105 '@crawlee/types': - specifier: ^4.0.0-beta.61 - version: 4.0.0-beta.61 + specifier: ^4.0.0-beta.105 + version: 4.0.0-beta.105 '@crawlee/utils': - specifier: ^4.0.0-beta.61 - version: 4.0.0-beta.61 + specifier: ^4.0.0-beta.105 + version: 4.0.0-beta.105 apify-client: specifier: ^2.23.4 version: 2.23.4 @@ -62,9 +65,6 @@ importers: '@commitlint/config-conventional': specifier: ^21.0.0 version: 21.0.1 - '@crawlee/memory-storage': - specifier: ^4.0.0-beta.61 - version: 4.0.0-beta.61 '@playwright/browser-chromium': specifier: ^1.60.0 version: 1.60.0 @@ -87,8 +87,8 @@ importers: specifier: ^21.0.0 version: 21.0.1(@types/node@24.12.2)(conventional-commits-parser@6.4.0)(typescript@6.0.3) crawlee: - specifier: ^4.0.0-beta.61 - version: 4.0.0-beta.61(@types/node@24.12.2)(playwright@1.60.0)(puppeteer@25.1.0) + specifier: ^4.0.0-beta.105 + version: 4.0.0-beta.105(@types/node@24.12.2)(playwright@1.60.0)(puppeteer@25.1.0) globby: specifier: ^16.0.0 version: 16.2.0 @@ -390,12 +390,12 @@ packages: engines: {node: '>= 0.10'} hasBin: true - '@apify/pseudo_url@2.0.75': - resolution: {integrity: sha512-dSP/ikQQJhokoOUlNmX9M3HaCgIjpO2deDHhgNkZgs0xyCL0HJVGD9pdRThT0kITY73DCLu8MPc1jaTpRJ+NOg==} - '@apify/timeout@0.3.2': resolution: {integrity: sha512-JnOLIOpqfm366q7opKrA6HrL0iYRpYYDn8Mi77sMR2GZ1fPbwMWCVzN23LJWfJV7izetZbCMrqRUXsR1etZ7dA==} + '@apify/timeout@0.4.8': + resolution: {integrity: sha512-SKBSXUYVYSaKCa5ogEEcAeKctbyRHq3v2yNLmzxZ/Rb+X4AuEFySd04nURGQPY2trtcKwKZP1sxrWvqwRH2WRA==} + '@apify/tsconfig@0.1.2': resolution: {integrity: sha512-9dzEI1ZQ5+iM0k0fmPJrpdSSPUolVdeI1nDGFZMjD9UabTmIvjQrzui+1a25uy913AUEBrKTojEPj87pU9/Ekg==} @@ -1076,12 +1076,12 @@ packages: conventional-commits-parser: optional: true - '@crawlee/basic@4.0.0-beta.61': - resolution: {integrity: sha512-HZCbqAxv6h6AUNTkPMYUuMKMyKKC7vk6Or4p4GCj+eQvqUcWTLwLt6/jdr0LmG+yiepQ4dfK4hzCsexhDicvSQ==} + '@crawlee/basic@4.0.0-beta.105': + resolution: {integrity: sha512-QUQ903uF8tHIZE7kuIYdU7V3Rb82gocVVZPj15FtznbcDgd8SOI6IgaXpy6m1KrxDauUrOMI7p4GFD9GWyRuTw==} engines: {node: '>=22.0.0'} - '@crawlee/browser-pool@4.0.0-beta.61': - resolution: {integrity: sha512-tCV94HnI12TV4+94nqtNDlkoSSVmp4y0l9Z18fkTeVuOOsqdx3U3E9oSc0H1j/mfK8bkWcggDbcaxj4pVmGjxQ==} + '@crawlee/browser-pool@4.0.0-beta.105': + resolution: {integrity: sha512-2cT4ahMRU1PesBH5Ny6Nl/+LpkWbmUdO/875vlXM2qOVyJ34qt2Q9n7YnMMmoq43ezhfujPCHwXACncHgpihNQ==} engines: {node: '>=22.0.0'} peerDependencies: playwright: '*' @@ -1092,8 +1092,8 @@ packages: puppeteer: optional: true - '@crawlee/browser@4.0.0-beta.61': - resolution: {integrity: sha512-JwsgVfnR0uPSlxRXZcpdXYtTUZJUOHSabeIe3xAF64KzOTfoRMVo+38079dVF7vZ8252VD6KhjUAMdtuHRvJGg==} + '@crawlee/browser@4.0.0-beta.105': + resolution: {integrity: sha512-RBXsdvVgBnpMjwUUmPZIHAh2319hFFhVkqfIlp2EimvcG43owh8et9cRP7Q7edHEIlFJq/Q/t9/0RE0324QV0g==} engines: {node: '>=22.0.0'} peerDependencies: playwright: '*' @@ -1104,45 +1104,74 @@ packages: puppeteer: optional: true - '@crawlee/cheerio@4.0.0-beta.61': - resolution: {integrity: sha512-dOq2zhQV40Ck3vLTmGFTNhu3XlwVeoPEd4u9e3b3qiJFqHrMeHPgWkUCL15kcYg5wtvZ9U6oPLjwhN3D99B4XA==} + '@crawlee/cheerio@4.0.0-beta.105': + resolution: {integrity: sha512-wKnZ87Wx6uHBSXEwO6YskCo4xw/b/ngM0zLX2h7w3lonwldrYyJBLVwDAO9DhKAFZavID7M9hXRCX2Y/ayqdVA==} engines: {node: '>=22.0.0'} - '@crawlee/cli@4.0.0-beta.61': - resolution: {integrity: sha512-c9CvwDIj3qMnMFJo41cdgWBcWtZkACkBcLoJ2CA/ysdhOPx4XvGSJGGDVZXT5cX4psJVmKUF2jFiWfmC/Jbfaw==} + '@crawlee/cli@4.0.0-beta.105': + resolution: {integrity: sha512-MWKCNZcuVNg71yR3Ikkhbw1seig5PTWeDT61gwL7y4z8BiyHWOto06yr+vn1IBwOXe+v7q9PTW1JDQDCTb6qDw==} engines: {node: '>=22.0.0'} hasBin: true - '@crawlee/core@4.0.0-beta.61': - resolution: {integrity: sha512-iuJlwScQ5yZW6sf9/hNoRkVeRSED8d0wsuQp3/3tpKLS2Y/er1mhUrtng16RstQWrN31hi5ea1Fv4zp8e0swdQ==} + '@crawlee/core@4.0.0-beta.105': + resolution: {integrity: sha512-TKD8FzkXCvPN2PCkoMEUhgvXjymKxPR7+qsnxKwy/2aY5LyyGaj7HQkhJVVwTegNi8R84v1T9GmktKCk+98VWQ==} engines: {node: '>=22.0.0'} - '@crawlee/got-scraping-client@4.0.0-beta.61': - resolution: {integrity: sha512-xz2PM7urfL+EWOG7kvy/HKTyfJZNjV7GLfm7PL1d9aSwdTHATtc4n3E4pPB4Jt4o9ZG5uuRwFMkiCWKSJnR2CA==} + '@crawlee/fs-storage-native-darwin-arm64@0.1.5-beta.18': + resolution: {integrity: sha512-fYZbU61GoMw+3q1JXRvARa8f/A8qZo0BxBpGogzq8duhiOChs4v+uHjffuZjL4E6goFR5qjX2kwOAJzoEp2kZQ==} + engines: {node: '>= 20'} + cpu: [arm64] + os: [darwin] + + '@crawlee/fs-storage-native-darwin-x64@0.1.5-beta.18': + resolution: {integrity: sha512-Wb/d6PLor34790ixEFsRADvTUXKzR846/1E2LUaDTLA3KQ5QUCwjQhs7VpfrgY9AzzgM/s8G+3TO2UI6L3eYkw==} + engines: {node: '>= 20'} + cpu: [x64] + os: [darwin] + + '@crawlee/fs-storage-native-linux-x64-gnu@0.1.5-beta.18': + resolution: {integrity: sha512-J8qfCkj3i6ic87fiD4f05WSXLXaAFLZlboXeeH9nDhJv/6RLxjiwTqec/Ig9H3yon+v4VQUbSHIh3Mm6B0g3XQ==} + engines: {node: '>= 20'} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@crawlee/fs-storage-native-win32-x64-msvc@0.1.5-beta.18': + resolution: {integrity: sha512-76uxjzcsD2E+mWqZ+HENpcSpJJAZVdWf3osKR3pA3wld+yzR9o0cckRysu4QTDhUeNtzL1KIq2NzOF1GPNLqiw==} + engines: {node: '>= 20'} + cpu: [x64] + os: [win32] + + '@crawlee/fs-storage-native@0.1.5-beta.18': + resolution: {integrity: sha512-G5+Alb5GDAZomtu+0mB8oeYA0BCMqOQIqNoTPtZxh2wLcb17CpMfJJvq6D/nT8vANAaBkrMR49bbrodtROV2ug==} + engines: {node: '>= 20'} + + '@crawlee/fs-storage@4.0.0-beta.105': + resolution: {integrity: sha512-PSag6q6bFsSPNi2Rypx9NXxaFhzQeIeLc+K5tOVx/QhNXQS78Rl6nLyLhLk12+Ti03nk+E/MTJrTXBJd6YJlPg==} engines: {node: '>=22.0.0'} - '@crawlee/http-client@4.0.0-beta.61': - resolution: {integrity: sha512-oShefU7Tzy4f8oqFYCgolQKOad6bWZQ59SGXJjTa+TZb+jj7ETQfF29R+RwmH/AZeDgzXBzkEqmFQi4iOuYFDg==} + '@crawlee/http-client@4.0.0-beta.105': + resolution: {integrity: sha512-gsRXsf3eSuqYLrkh0zzY1PnluSCyTwPDXE7Dsa67J59e6idkY0deXWZDUkd5IEiFdUpe43bIdUBJhyNV4imkuQ==} engines: {node: '>=22.0.0'} - '@crawlee/http@4.0.0-beta.61': - resolution: {integrity: sha512-SuTCpfyCzg2zQLc2WnDQlE794m4EC4Lqx+zIS7am0hw4mNXQqoRbsnwabN3Wo/tvvhBfRCso7sQ1xcN5NGF5AQ==} + '@crawlee/http@4.0.0-beta.105': + resolution: {integrity: sha512-DV9x2OX0xbIWQcUcQSZL9uxIy3F3FiyUJ9V1B7w6DY/5PuEuMY9X/kznHlj9nlT0UU6/TiCwMzmLOJItXrvTmw==} engines: {node: '>=22.0.0'} - '@crawlee/jsdom@4.0.0-beta.61': - resolution: {integrity: sha512-ysuWxuRcriD6+taxu2Y7lOLX1Jwp8u3zcfc/SxSgdjIQQouDPNFs2WTFycPt2+TB9HRSalTBRCZ68QxHfBk2oQ==} + '@crawlee/impit-client@4.0.0-beta.105': + resolution: {integrity: sha512-ihj2pr4zhqDN11TWSuwwaIURDWgSH8DPflTYUXBTSR/Yb5MubphvAVGxQV19DmtG1w698+gB8TZsMfjsA1VUIw==} engines: {node: '>=22.0.0'} - '@crawlee/linkedom@4.0.0-beta.61': - resolution: {integrity: sha512-TN1s73PfN+Ws6GIcpjK9dN+EdUTZ4hy5B7TApoidWZJGHOdR0NHrpV/dHk2ug6KnyKIa5ELxdZaLJM9G2rnU5A==} + '@crawlee/jsdom@4.0.0-beta.105': + resolution: {integrity: sha512-J89wkze6SYJtUWmEOCw291nPE6V56eE96VG0IILiiiF7t5UCRKhLA8Nn7VPr9elrnrQ4wCPFUSKoBHThyyK4aQ==} engines: {node: '>=22.0.0'} - '@crawlee/memory-storage@4.0.0-beta.61': - resolution: {integrity: sha512-4fQI5GHfMIkz52N3b3DMDQg8lFmDmF2xbSoszx/s79eBzmlmKD22njcgDxgIg/G8uEQfIfk69he7oJDtBwY2QA==} + '@crawlee/linkedom@4.0.0-beta.105': + resolution: {integrity: sha512-FkOYBumZILMKjOvcpfKeJ/1cb9XWhdAGKzlr3Ivcc3VqHISzAiZNDGucPEC/u+I7kKil/I6OJl3w/MTMyPMuqw==} engines: {node: '>=22.0.0'} - '@crawlee/playwright@4.0.0-beta.61': - resolution: {integrity: sha512-lf/y19YAqvlrEWI86Fu7E3lMk/d7zlgRkWVbZFEOXvoVCt48Uxu9lu/s94jfpvxwzjNgHeUfTN1DONO8PFJtsA==} + '@crawlee/playwright@4.0.0-beta.105': + resolution: {integrity: sha512-rJfFpaaGxRmYw4iNQvBc05L4Js7BQ9JS/10xUfMOM7OZgb2YwMR6GRpqaID1EytG4Q+mAqTyb1ESVDKBtWsJkQ==} engines: {node: '>=22.0.0'} peerDependencies: idcac-playwright: ^0.2.0 @@ -1153,8 +1182,8 @@ packages: playwright: optional: true - '@crawlee/puppeteer@4.0.0-beta.61': - resolution: {integrity: sha512-fiGkTU+xu2JU0IN7R96cAkDFe5iNHp13VjQPgXL+gvS1oNOSthVue8+j3Be5ZxvmyqnCb3qGMhqf8pn2knym7w==} + '@crawlee/puppeteer@4.0.0-beta.105': + resolution: {integrity: sha512-wb0FbTJg0G4hmC4nfoitiY9shpUYMuErZmbRe8qGoK8HYcJI3c3LnBaKgtpaPoVFT21UaYRr8hJP78LfIzWQZA==} engines: {node: '>=22.0.0'} peerDependencies: idcac-playwright: ^0.2.0 @@ -1165,20 +1194,20 @@ packages: puppeteer: optional: true - '@crawlee/templates@4.0.0-beta.61': - resolution: {integrity: sha512-5XXPefaxHlNHJE8T9V5+BV6oiOj07bKsM+kA+jF/U1uSw+doKumrVrQ5INjHuOriwRHKSpDuIOWVYekgSgs9yw==} + '@crawlee/templates@4.0.0-beta.105': + resolution: {integrity: sha512-5V6PWjymwsNJOBqtXRlI6lHwxazx6U4s+I9EyMVvPpV1G9bxXqY/JxvQYNqEGpToKhu3FFh1GVAYWU+0yzRVzA==} engines: {node: '>=22.0.0'} '@crawlee/types@3.16.0': resolution: {integrity: sha512-CcIM+JDVx4gzQzMPl+9RJiEeqdzTrx2RLPA7y4IMJSyfZm3J/VrEunielKA3NQrk095j9OuvS/rQL2y8mBV1qw==} engines: {node: '>=16.0.0'} - '@crawlee/types@4.0.0-beta.61': - resolution: {integrity: sha512-eCCGO2eoDvjNgZcIHUtY0bP5bXfxxUCNQm0W1eZciC0X3kFp1Ouei0RcZOLOqdUTRx69trJnAwWZdOvDiUL3lA==} + '@crawlee/types@4.0.0-beta.105': + resolution: {integrity: sha512-wEN1xTZ5Mi3e6Ocprhz87PKVMCGGba8Zcy3TJoA5xplNwpTNrliDSgG1jRsBTmb3YeCTL1VzELiZ8avzHYlYrg==} engines: {node: '>=22.0.0'} - '@crawlee/utils@4.0.0-beta.61': - resolution: {integrity: sha512-2IloaTvA1W5reQ0tD94amOpeFwONsMPTaHWMKnx/KZABOTnzqv73dYVuYlfkPNatGc4Kqr1Sjwx1iF+tQB22fw==} + '@crawlee/utils@4.0.0-beta.105': + resolution: {integrity: sha512-jJE8YVqm40HmqxcaR3C7/xMJZBOVqj/5TwqL4nVHpxZ1jkOBJtJUXFI+VQwhZBIIj5WY0OF3SJzAvqo3iDfycQ==} engines: {node: '>=22.0.0'} '@csstools/cascade-layer-name-parser@2.0.5': @@ -2538,9 +2567,6 @@ packages: peerDependencies: tslib: '2' - '@keyv/serialize@1.1.1': - resolution: {integrity: sha512-dXn3FZhPv0US+7dtJsIi2R+c7qWYiReoEh5zUntWCf4oSpMNib8FDhSoed6m3QyZdx5hK7iLFkYk3rNxwt8vTA==} - '@leichtgewicht/ip-codec@2.0.5': resolution: {integrity: sha512-Vo+PSpZG2/fmgmiNzYK9qWRh8h/CHrwD0mo1h1DzL4yzHNSfWYujGTYsWGreD000gcgmZ7K4Ys6Tx9TxtsKdDw==} @@ -3305,9 +3331,6 @@ packages: resolution: {integrity: sha512-d9dUmWVA7MMiKobL3VpLF8P2aeanRTu6ypG2OIaEv/ZHH/SUQ2iHOVyi5wAPjQ+HmnMuL0whK9ez8I/raWbtIg==} engines: {node: '>=v16'} - '@sec-ant/readable-stream@0.4.1': - resolution: {integrity: sha512-831qok9r2t8AlxLko40y2ebgSDhenenCatLVeW/uBtnHPyhHOvG0C7TvfgecV+wHzIm5KUICgzmVpWS+IMEAeg==} - '@shikijs/core@1.29.2': resolution: {integrity: sha512-vju0lY9r27jJfOY4Z7+Rt/nIOjzJpZ3y+nYpqtUZInVoXQ/TJZcfGnNOGnKjFdVZb8qexiCuSlZRKcGfhhTTZQ==} @@ -3367,10 +3390,6 @@ packages: resolution: {integrity: sha512-FX4MfcifwJyFOI2lPoX7PQxCqx8BG1HCho7WdiXwpEQx1Ycij0JxkfYtGK7yqNScrZGSlt6RE6sw8QYoH7eKnQ==} engines: {node: '>=16'} - '@sindresorhus/is@7.2.0': - resolution: {integrity: sha512-P1Cz1dWaFfR4IR+U13mqqiGsLFf1KbayybWwdd2vfctdV6hDpUkgCY0nKOLLTMSoRd/jJNjtbqzf13K8DCCXQw==} - engines: {node: '>=18'} - '@sindresorhus/merge-streams@4.0.0': resolution: {integrity: sha512-tlqY9xq5ukxTUZBmoOp+m61cqwQD5pHJtFY3Mn8CA8ps6yghLH/Hw8UPdqg4OLmFW3IFlcXnQNmo/dh8HzXYIQ==} engines: {node: '>=18'} @@ -4222,10 +4241,6 @@ packages: resolution: {integrity: sha512-tjwM5exMg6BGRI+kNmTntNsvdZS1X8BFYS6tnJ2hdH0kVxM6/eVZ2xy+FqStSWvYmtfFMDLIxurorHwDKfDz5Q==} engines: {node: '>=18'} - byte-counter@0.1.0: - resolution: {integrity: sha512-jheRLVMeUKrDBjVw2O5+k4EvR4t9wtxHL+bo/LxfkxsVeuGMy3a5SEGgXdAFA4FSzTrU8rQXQIrsZ3oBq5a0pQ==} - engines: {node: '>=20'} - bytes@3.0.0: resolution: {integrity: sha512-pMhOfFDPiv9t5jjIXkHosWmkSyQbvsgEVNkz0ERHbuLh2T/7j4Mqqpz523Fe8MVY89KC6Sh/QfS2sM+SjgFDcw==} engines: {node: '>= 0.8'} @@ -4246,10 +4261,6 @@ packages: resolution: {integrity: sha512-zkDT5WAF4hSSoUgyfg5tFIxz8XQK+25W/TLVojJTMKBaxevLBBtLxgqguAuVQB8PVW79FVjHcU+GJ9tVbDZ9mQ==} engines: {node: '>=14.16'} - cacheable-request@13.0.18: - resolution: {integrity: sha512-rFWadDRKJs3s2eYdXlGggnBZKG7MTblkFBB0YllFds+UYnfogDp2wcR6JN97FhRkHTvq59n2vhNoHNZn29dh/Q==} - engines: {node: '>=18'} - call-bind-apply-helpers@1.0.2: resolution: {integrity: sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==} engines: {node: '>= 0.4'} @@ -4571,8 +4582,8 @@ packages: typescript: optional: true - crawlee@4.0.0-beta.61: - resolution: {integrity: sha512-Ravqv5dni2uoTM8GOoV6amKhpY0VZaeZWhf3H57l/OrUVkG8IXezYvfZal2IvkMm+shXjW1njty4UqLkJOgGng==} + crawlee@4.0.0-beta.105: + resolution: {integrity: sha512-j+IL/2viybfl2ZY+z7roJ1kme0sB/MNoTFI00lShtVtPDMKnwwpTcihlrIkW1NZMcxA9pbckyimz4vcu8s7IqA==} engines: {node: '>=22.0.0'} hasBin: true peerDependencies: @@ -4792,10 +4803,6 @@ packages: resolution: {integrity: sha512-+8VxcR21HhTy8nOt6jf20w0c9CADrw1O8d+VZ/YzzCt4bJ3uBjw+D1q2osAB8RnpwwaeYBxy0HyKQxD5JBMuuQ==} engines: {node: '>=14.16'} - decompress-response@10.0.0: - resolution: {integrity: sha512-oj7KWToJuuxlPr7VV0vabvxEIiqNMo+q0NueIiL3XhtwC6FVOX7Hr1c0C4eD0bmf7Zr+S/dSf2xvkH3Ad6sU3Q==} - engines: {node: '>=20'} - decompress-response@6.0.0: resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==} engines: {node: '>=10'} @@ -4933,10 +4940,6 @@ packages: resolution: {integrity: sha512-tE7ztYzXHIeyvc7N+hR3oi7FIbf/NIjVP9hmAt3yMXzrQ072/fpjGLx2GxNxGxUl5V73MEqYzioOMoVhGMJ5cA==} engines: {node: '>=10'} - dot-prop@7.2.0: - resolution: {integrity: sha512-Ol/IPXUARn9CSbkrdV4VJo7uCy1I3VuSiWCaFSg+8BdUOzF9n3jefIpcgAydvUZbTdEBZs2vEiTiS9m61ssiDA==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - dot-prop@8.0.2: resolution: {integrity: sha512-xaBe6ZT4DHPkg0k4Ytbvn5xoxgpG0jOS1dYxSOwAHPuNLjP3/OzN0gH55SrLqpx8cBfSaVt91lXYkApjb+nYdQ==} engines: {node: '>=16'} @@ -5292,10 +5295,6 @@ packages: resolution: {integrity: sha512-yDYSgNMraqvnxiEXO4hi88+YZxaHC6QKzb5N84iRCTDeRO7ZALpir/lVmf/uXUhnwUr2O4HU8s/n6x+yNjQkHw==} engines: {node: '>= 14.17'} - form-data-encoder@4.1.0: - resolution: {integrity: sha512-G6NsmEW15s0Uw9XnCg+33H3ViYRyiM0hMrMhhqQOR8NFc5GhYrI+6I3u7OTw7b91J2g8rtvMBZJDbcGb2YUniw==} - engines: {node: '>= 18'} - form-data@4.0.5: resolution: {integrity: sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==} engines: {node: '>= 6'} @@ -5369,10 +5368,6 @@ packages: resolution: {integrity: sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==} engines: {node: '>=10'} - get-stream@9.0.1: - resolution: {integrity: sha512-kVCxPF3vQM/N0B1PmoqVUqgHP+EeVjmZSQn+1oCRPxd2P21P2F19lIgbR3HBosbB1PUhOAoctJnfEn2GbN2eZA==} - engines: {node: '>=18'} - get-tsconfig@4.13.7: resolution: {integrity: sha512-7tN6rFgBlMgpBML5j8typ92BKFi2sFQvIdpAqLA2beia5avZDrMs0FLZiM5etShWq5irVyGcGMEA1jcDaK7A/Q==} @@ -5439,18 +5434,10 @@ packages: resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} engines: {node: '>= 0.4'} - got-scraping@4.2.1: - resolution: {integrity: sha512-rhOlO1L4H4Cm31smHJqPtAaXOUrhSKsiTrbZSHKFQW1E/mkTDopnHHpRnXJpqzE0faj+zPsVQnyifIqO+K+cLQ==} - engines: {node: '>=16'} - got@12.6.1: resolution: {integrity: sha512-mThBblvlAF1d4O5oqyvN+ZxLAYwIJK7bpMxgYqPD9okW0C3qm5FFn7k811QrcuEBwaogR3ngOFoCfs6mRv7teQ==} engines: {node: '>=14.16'} - got@14.6.6: - resolution: {integrity: sha512-QLV1qeYSo5l13mQzWgP/y0LbMr5Plr5fJilgAIwgnwseproEbtNym8xpLsDzeZ6MWXgNE6kdWGBjdh3zT/Qerg==} - engines: {node: '>=20'} - graceful-fs@4.2.10: resolution: {integrity: sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==} @@ -5731,6 +5718,62 @@ packages: engines: {node: '>=16.x'} hasBin: true + impit-darwin-arm64@0.14.3: + resolution: {integrity: sha512-kMoQB+CR+a954pnhe4kf1O2RyJCsqGRE95jIXfgqNOiIMS5O1z0cOMzG/sohJk/nodZVSJ5VdWEV4BKhRWPFSA==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [darwin] + + impit-darwin-x64@0.14.3: + resolution: {integrity: sha512-ft9+kjz1pR8H5xbbf5U1EgwaaIea5iXHxBf2Q3NoinPpiV7KgyzYSr83pCSrAY6evWk+smG9shHgzhQtgbj1Rg==} + engines: {node: '>= 10'} + cpu: [x64] + os: [darwin] + + impit-linux-arm64-gnu@0.14.3: + resolution: {integrity: sha512-+mwta93S6Ndfpca3DDblvrqV1g8Bg6gZFOlEiJHfGJZpiUIQE+A/Pjg2e+inV+WrTtlnLsvff8lgwFvkIGnRug==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + libc: [glibc] + + impit-linux-arm64-musl@0.14.3: + resolution: {integrity: sha512-wYhfH1J8laWkpSN2SAFCKx9npY4vXrk23ex+tOzTGf2xBBqIhpMUff2wa6dOFb0or6EhZHBssoRf0HK3h/htXQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + libc: [musl] + + impit-linux-x64-gnu@0.14.3: + resolution: {integrity: sha512-hAYnutJDGQO5bPvTPKBBFkHgbB7QL7QLDse8c4s21VSzC/EqGliSS3UqR5ZUUwaFdm3t8DJsXOr+gboAUwDR7w==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + libc: [glibc] + + impit-linux-x64-musl@0.14.3: + resolution: {integrity: sha512-wXBeHiqCjuyqxg4u5eY4OBNVzsS4Karz/ms/0mjB8aqVZLNER1NaUTeE4J3eZCdytFaJRaAeQvKTH8HfjpPsQg==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + libc: [musl] + + impit-win32-arm64-msvc@0.14.3: + resolution: {integrity: sha512-XiF8MYpm4tF8TMbtsWrcEDDs6NxRNeirfiHM/AyKcr2jiN34hZwbZDM0EseD30PDwb2Pny2JWB+zTxpE1P5CZg==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [win32] + + impit-win32-x64-msvc@0.14.3: + resolution: {integrity: sha512-HIaLSpU5SGVx3UDH/R3ZaGMURVgyRVjmalqXj65ABxVUqfXiBo1t5ZObDPPjqvLT3klPnrya1+xF1DQ4dfT+og==} + engines: {node: '>= 10'} + cpu: [x64] + os: [win32] + + impit@0.14.3: + resolution: {integrity: sha512-SbCLoeW0YDRox5kQoy71jtpjZE+BwjWO411OdgfbLMbNmSX79s5Y0Y2vCE7AO6zQSq9F4Fu8SJTCKnp8rXQzpg==} + engines: {node: '>= 20'} + import-fresh@3.3.1: resolution: {integrity: sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==} engines: {node: '>=6'} @@ -5912,10 +5955,6 @@ packages: resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==} engines: {node: '>=8'} - is-stream@4.0.1: - resolution: {integrity: sha512-Dnz92NInDqYckGEUJv689RbRiTSEHCQ7wOVeALbkOz999YpqT46yMRIGtSNl2iCL1waAZSx40+h59NV/EwzV/A==} - engines: {node: '>=18'} - is-typedarray@1.0.0: resolution: {integrity: sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA==} @@ -6018,9 +6057,6 @@ packages: keyv@4.5.4: resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} - keyv@5.6.0: - resolution: {integrity: sha512-CYDD3SOtsHtyXeEORYRx2qBtpDJFjRTGXUtmNEMGyzYOKj1TE3tycdlho7kA1Ufx9OYWZzg52QFBGALTirzDSw==} - kind-of@6.0.3: resolution: {integrity: sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==} engines: {node: '>=0.10.0'} @@ -6708,10 +6744,6 @@ packages: resolution: {integrity: sha512-dD4UpyBh/9m4X2NVjA+73/ZPBRF+uF4zIMFvvQsabMiEK8x41L3rQ8EENOi35kyyoaJwNxEeJcP6Fj1H4U409Q==} engines: {node: '>=12'} - ow@1.1.1: - resolution: {integrity: sha512-sJBRCbS5vh1Jp9EOgwp1Ws3c16lJrUkJYlvWTYC03oyiYVwS/ns7lKRWow4w4XjDyTrA2pplQv4B2naWSR6yDA==} - engines: {node: '>=14.16'} - ow@2.0.0: resolution: {integrity: sha512-ESUigmGrdhUZ2nQSFNkeKSl6ZRPupXzprMs3yF9DYlNVpJ8XAjM/fI9RUZxA7PI1K9HQDCCvBo1jr/GEIo9joQ==} engines: {node: '>=18'} @@ -6750,10 +6782,6 @@ packages: resolution: {integrity: sha512-mlVgR3PGuzlo0MmTdk4cXqXWlwQDLnONTAg6sm62XkMJEiRxN3GL3SffkYvqwonbkJBcrI7Uvv5Zh9yjvn2iUw==} engines: {node: '>=12.20'} - p-cancelable@4.0.1: - resolution: {integrity: sha512-wBowNApzd45EIKdO1LaU+LrMBwAcjfPaYtVzV3lmfM3gf8Z4CHZsiIqlM8TZZ8okYvh5A1cP6gTfCRQtwUpaUg==} - engines: {node: '>=14.16'} - p-event@6.0.1: resolution: {integrity: sha512-Q6Bekk5wpzW5qIyUP4gdMEujObYstZl6DMMOSenwBvV0BlE5LkDwkjs5yHbZmdCEq2o4RJx4tE1vwxFVf2FG1w==} engines: {node: '>=16.17'} @@ -7493,9 +7521,6 @@ packages: prop-types@15.8.1: resolution: {integrity: sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==} - proper-lockfile@4.1.2: - resolution: {integrity: sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==} - property-information@7.1.0: resolution: {integrity: sha512-TwEZ+X+yCJmYfL7TPUOcvBZ4QfoT5YenQiJuX//0th53DE6w0xxLEtfK3iyryQFddXuvkIk51EEgrJQ0WJkOmQ==} @@ -7840,18 +7865,10 @@ packages: resolution: {integrity: sha512-40yHxbNcl2+rzXvZuVkrYohathsSJlMTXKryG5y8uciHv1+xDLHQpgjG64JUO9nrEq2jGLH6IZ8BcZyw3wrweg==} engines: {node: '>=14.16'} - responselike@4.0.2: - resolution: {integrity: sha512-cGk8IbWEAnaCpdAt1BHzJ3Ahz5ewDJa0KseTsE3qIRMJ3C698W8psM7byCeWVpd/Ha7FUYzuRVzXoKoM6nRUbA==} - engines: {node: '>=20'} - restore-cursor@5.1.0: resolution: {integrity: sha512-oMA2dcrw6u0YfxJQXm342bFKX/E4sG9rbTzO9ptUcR/e8A33cHuvStiYOwH7fszkZlZ1z/ta9AAoPk2F4qIOHA==} engines: {node: '>=18'} - retry@0.12.0: - resolution: {integrity: sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==} - engines: {node: '>= 4'} - retry@0.13.1: resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==} engines: {node: '>= 4'} @@ -9321,12 +9338,10 @@ snapshots: dependencies: event-stream: 3.3.4 - '@apify/pseudo_url@2.0.75': - dependencies: - '@apify/log': 2.5.34 - '@apify/timeout@0.3.2': {} + '@apify/timeout@0.4.8': {} + '@apify/tsconfig@0.1.2': {} '@apify/ui-icons@1.34.1(react-dom@19.2.5(react@19.2.5))(react@19.2.5)': @@ -10253,28 +10268,30 @@ snapshots: optionalDependencies: conventional-commits-parser: 6.4.0 - '@crawlee/basic@4.0.0-beta.61': + '@crawlee/basic@4.0.0-beta.105': dependencies: - '@apify/timeout': 0.3.2 + '@apify/datastructures': 2.0.3 + '@apify/timeout': 0.4.8 '@apify/utilities': 2.26.0 - '@crawlee/core': 4.0.0-beta.61 - '@crawlee/got-scraping-client': 4.0.0-beta.61 - '@crawlee/types': 4.0.0-beta.61 - '@crawlee/utils': 4.0.0-beta.61 + '@crawlee/core': 4.0.0-beta.105 + '@crawlee/http-client': 4.0.0-beta.105 + '@crawlee/types': 4.0.0-beta.105 + '@crawlee/utils': 4.0.0-beta.105 csv-stringify: 6.7.0 - fs-extra: 11.3.4 ow: 2.0.0 tldts: 7.0.28 tslib: 2.8.1 type-fest: 4.41.0 + optionalDependencies: + '@crawlee/impit-client': 4.0.0-beta.105 transitivePeerDependencies: - supports-color - '@crawlee/browser-pool@4.0.0-beta.61(playwright@1.60.0)(puppeteer@25.1.0)': + '@crawlee/browser-pool@4.0.0-beta.105(playwright@1.60.0)(puppeteer@25.1.0)': dependencies: - '@apify/timeout': 0.3.2 - '@crawlee/core': 4.0.0-beta.61 - '@crawlee/types': 4.0.0-beta.61 + '@apify/timeout': 0.4.8 + '@crawlee/core': 4.0.0-beta.105 + '@crawlee/types': 4.0.0-beta.105 fingerprint-generator: 2.1.82 fingerprint-injector: 2.1.82(playwright@1.60.0)(puppeteer@25.1.0) lodash.merge: 4.6.2 @@ -10291,13 +10308,13 @@ snapshots: transitivePeerDependencies: - supports-color - '@crawlee/browser@4.0.0-beta.61(playwright@1.60.0)(puppeteer@25.1.0)': + '@crawlee/browser@4.0.0-beta.105(playwright@1.60.0)(puppeteer@25.1.0)': dependencies: - '@apify/timeout': 0.3.2 - '@crawlee/basic': 4.0.0-beta.61 - '@crawlee/browser-pool': 4.0.0-beta.61(playwright@1.60.0)(puppeteer@25.1.0) - '@crawlee/types': 4.0.0-beta.61 - '@crawlee/utils': 4.0.0-beta.61 + '@apify/timeout': 0.4.8 + '@crawlee/basic': 4.0.0-beta.105 + '@crawlee/browser-pool': 4.0.0-beta.105(playwright@1.60.0)(puppeteer@25.1.0) + '@crawlee/types': 4.0.0-beta.105 + '@crawlee/utils': 4.0.0-beta.105 ow: 2.0.0 tslib: 2.8.1 type-fest: 4.41.0 @@ -10307,43 +10324,44 @@ snapshots: transitivePeerDependencies: - supports-color - '@crawlee/cheerio@4.0.0-beta.61': + '@crawlee/cheerio@4.0.0-beta.105': dependencies: - '@crawlee/http': 4.0.0-beta.61 - '@crawlee/types': 4.0.0-beta.61 - '@crawlee/utils': 4.0.0-beta.61 + '@crawlee/http': 4.0.0-beta.105 + '@crawlee/types': 4.0.0-beta.105 + '@crawlee/utils': 4.0.0-beta.105 cheerio: 1.2.0 htmlparser2: 10.1.0 tslib: 2.8.1 transitivePeerDependencies: - supports-color - '@crawlee/cli@4.0.0-beta.61(@types/node@24.12.2)': + '@crawlee/cli@4.0.0-beta.105(@types/node@24.12.2)': dependencies: - '@crawlee/templates': 4.0.0-beta.61 + '@crawlee/templates': 4.0.0-beta.105 '@inquirer/prompts': 7.10.1(@types/node@24.12.2) ansi-colors: 4.1.3 - fs-extra: 11.3.4 tslib: 2.8.1 yargs: 18.0.0 transitivePeerDependencies: - '@types/node' - '@crawlee/core@4.0.0-beta.61': + '@crawlee/core@4.0.0-beta.105': dependencies: '@apify/consts': 2.52.0 '@apify/datastructures': 2.0.3 '@apify/log': 2.5.34 - '@apify/pseudo_url': 2.0.75 - '@apify/timeout': 0.3.2 + '@apify/timeout': 0.4.8 '@apify/utilities': 2.26.0 - '@crawlee/memory-storage': 4.0.0-beta.61 - '@crawlee/types': 4.0.0-beta.61 - '@crawlee/utils': 4.0.0-beta.61 + '@crawlee/fs-storage': 4.0.0-beta.105 + '@crawlee/types': 4.0.0-beta.105 + '@crawlee/utils': 4.0.0-beta.105 '@sapphire/async-queue': 1.5.5 + '@sapphire/shapeshift': 4.0.0 '@vladfrangu/async_event_emitter': 2.4.7 + content-type: 1.0.5 csv-stringify: 6.7.0 json5: 2.2.3 + mime-types: 3.0.2 minimatch: 10.2.5 ow: 2.0.0 stream-json: 1.9.1 @@ -10355,25 +10373,45 @@ snapshots: transitivePeerDependencies: - supports-color - '@crawlee/got-scraping-client@4.0.0-beta.61': + '@crawlee/fs-storage-native-darwin-arm64@0.1.5-beta.18': + optional: true + + '@crawlee/fs-storage-native-darwin-x64@0.1.5-beta.18': + optional: true + + '@crawlee/fs-storage-native-linux-x64-gnu@0.1.5-beta.18': + optional: true + + '@crawlee/fs-storage-native-win32-x64-msvc@0.1.5-beta.18': + optional: true + + '@crawlee/fs-storage-native@0.1.5-beta.18': + optionalDependencies: + '@crawlee/fs-storage-native-darwin-arm64': 0.1.5-beta.18 + '@crawlee/fs-storage-native-darwin-x64': 0.1.5-beta.18 + '@crawlee/fs-storage-native-linux-x64-gnu': 0.1.5-beta.18 + '@crawlee/fs-storage-native-win32-x64-msvc': 0.1.5-beta.18 + + '@crawlee/fs-storage@4.0.0-beta.105': dependencies: - '@crawlee/http-client': 4.0.0-beta.61 - got-scraping: 4.2.1 + '@crawlee/fs-storage-native': 0.1.5-beta.18 + '@crawlee/types': 4.0.0-beta.105 + '@sapphire/shapeshift': 4.0.0 - '@crawlee/http-client@4.0.0-beta.61': + '@crawlee/http-client@4.0.0-beta.105': dependencies: - '@crawlee/types': 4.0.0-beta.61 + '@crawlee/types': 4.0.0-beta.105 tough-cookie: 6.0.1 - '@crawlee/http@4.0.0-beta.61': + '@crawlee/http@4.0.0-beta.105': dependencies: - '@apify/timeout': 0.3.2 + '@apify/timeout': 0.4.8 '@apify/utilities': 2.26.0 - '@crawlee/basic': 4.0.0-beta.61 - '@crawlee/core': 4.0.0-beta.61 - '@crawlee/http-client': 4.0.0-beta.61 - '@crawlee/types': 4.0.0-beta.61 - '@crawlee/utils': 4.0.0-beta.61 + '@crawlee/basic': 4.0.0-beta.105 + '@crawlee/core': 4.0.0-beta.105 + '@crawlee/http-client': 4.0.0-beta.105 + '@crawlee/types': 4.0.0-beta.105 + '@crawlee/utils': 4.0.0-beta.105 '@types/content-type': 1.1.9 cheerio: 1.2.0 content-type: 1.0.5 @@ -10385,13 +10423,21 @@ snapshots: transitivePeerDependencies: - supports-color - '@crawlee/jsdom@4.0.0-beta.61': + '@crawlee/impit-client@4.0.0-beta.105': dependencies: - '@apify/timeout': 0.3.2 + '@apify/datastructures': 2.0.3 + '@crawlee/http-client': 4.0.0-beta.105 + '@crawlee/types': 4.0.0-beta.105 + impit: 0.14.3 + tough-cookie: 6.0.1 + + '@crawlee/jsdom@4.0.0-beta.105': + dependencies: + '@apify/timeout': 0.4.8 '@apify/utilities': 2.26.0 - '@crawlee/http': 4.0.0-beta.61 - '@crawlee/types': 4.0.0-beta.61 - '@crawlee/utils': 4.0.0-beta.61 + '@crawlee/http': 4.0.0-beta.105 + '@crawlee/types': 4.0.0-beta.105 + '@crawlee/utils': 4.0.0-beta.105 '@types/jsdom': 21.1.7 cheerio: 1.2.0 jsdom: 26.1.0 @@ -10403,13 +10449,13 @@ snapshots: - supports-color - utf-8-validate - '@crawlee/linkedom@4.0.0-beta.61': + '@crawlee/linkedom@4.0.0-beta.105': dependencies: - '@apify/timeout': 0.3.2 + '@apify/timeout': 0.4.8 '@apify/utilities': 2.26.0 - '@crawlee/http': 4.0.0-beta.61 - '@crawlee/types': 4.0.0-beta.61 - '@crawlee/utils': 4.0.0-beta.61 + '@crawlee/http': 4.0.0-beta.105 + '@crawlee/types': 4.0.0-beta.105 + '@crawlee/utils': 4.0.0-beta.105 cheerio: 1.2.0 linkedom: 0.18.12 ow: 2.0.0 @@ -10418,30 +10464,17 @@ snapshots: - canvas - supports-color - '@crawlee/memory-storage@4.0.0-beta.61': - dependencies: - '@crawlee/types': 4.0.0-beta.61 - '@sapphire/async-queue': 1.5.5 - '@sapphire/shapeshift': 4.0.0 - content-type: 1.0.5 - fs-extra: 11.3.4 - json5: 2.2.3 - mime-types: 3.0.2 - p-limit: 6.2.0 - proper-lockfile: 4.1.2 - tslib: 2.8.1 - - '@crawlee/playwright@4.0.0-beta.61(playwright@1.60.0)(puppeteer@25.1.0)': + '@crawlee/playwright@4.0.0-beta.105(playwright@1.60.0)(puppeteer@25.1.0)': dependencies: '@apify/datastructures': 2.0.3 - '@apify/timeout': 0.3.2 - '@crawlee/basic': 4.0.0-beta.61 - '@crawlee/browser': 4.0.0-beta.61(playwright@1.60.0)(puppeteer@25.1.0) - '@crawlee/browser-pool': 4.0.0-beta.61(playwright@1.60.0)(puppeteer@25.1.0) - '@crawlee/cheerio': 4.0.0-beta.61 - '@crawlee/core': 4.0.0-beta.61 - '@crawlee/types': 4.0.0-beta.61 - '@crawlee/utils': 4.0.0-beta.61 + '@apify/timeout': 0.4.8 + '@crawlee/basic': 4.0.0-beta.105 + '@crawlee/browser': 4.0.0-beta.105(playwright@1.60.0)(puppeteer@25.1.0) + '@crawlee/browser-pool': 4.0.0-beta.105(playwright@1.60.0)(puppeteer@25.1.0) + '@crawlee/cheerio': 4.0.0-beta.105 + '@crawlee/core': 4.0.0-beta.105 + '@crawlee/types': 4.0.0-beta.105 + '@crawlee/utils': 4.0.0-beta.105 cheerio: 1.2.0 jquery: 3.7.1 ml-logistic-regression: 2.0.0 @@ -10449,20 +10482,21 @@ snapshots: ow: 2.0.0 string-comparison: 1.3.0 tslib: 2.8.1 + type-fest: 4.41.0 optionalDependencies: playwright: 1.60.0 transitivePeerDependencies: - puppeteer - supports-color - '@crawlee/puppeteer@4.0.0-beta.61(playwright@1.60.0)(puppeteer@25.1.0)': + '@crawlee/puppeteer@4.0.0-beta.105(playwright@1.60.0)(puppeteer@25.1.0)': dependencies: '@apify/datastructures': 2.0.3 - '@crawlee/browser': 4.0.0-beta.61(playwright@1.60.0)(puppeteer@25.1.0) - '@crawlee/browser-pool': 4.0.0-beta.61(playwright@1.60.0)(puppeteer@25.1.0) - '@crawlee/core': 4.0.0-beta.61 - '@crawlee/types': 4.0.0-beta.61 - '@crawlee/utils': 4.0.0-beta.61 + '@crawlee/browser': 4.0.0-beta.105(playwright@1.60.0)(puppeteer@25.1.0) + '@crawlee/browser-pool': 4.0.0-beta.105(playwright@1.60.0)(puppeteer@25.1.0) + '@crawlee/core': 4.0.0-beta.105 + '@crawlee/types': 4.0.0-beta.105 + '@crawlee/utils': 4.0.0-beta.105 cheerio: 1.2.0 devtools-protocol: 0.0.1624250 jquery: 3.7.1 @@ -10474,7 +10508,7 @@ snapshots: - playwright - supports-color - '@crawlee/templates@4.0.0-beta.61': + '@crawlee/templates@4.0.0-beta.105': dependencies: tslib: 2.8.1 @@ -10482,16 +10516,16 @@ snapshots: dependencies: tslib: 2.8.1 - '@crawlee/types@4.0.0-beta.61': + '@crawlee/types@4.0.0-beta.105': dependencies: tough-cookie: 6.0.1 tslib: 2.8.1 - '@crawlee/utils@4.0.0-beta.61': + '@crawlee/utils@4.0.0-beta.105': dependencies: '@apify/ps-tree': 1.2.0 - '@crawlee/http-client': 4.0.0-beta.61 - '@crawlee/types': 4.0.0-beta.61 + '@crawlee/http-client': 4.0.0-beta.105 + '@crawlee/types': 4.0.0-beta.105 '@types/sax': 1.2.7 cheerio: 1.2.0 domhandler: 5.0.3 @@ -12446,8 +12480,6 @@ snapshots: '@jsonjoy.com/codegen': 17.67.0(tslib@2.8.1) tslib: 2.8.1 - '@keyv/serialize@1.1.1': {} - '@leichtgewicht/ip-codec@2.0.5': {} '@mdx-js/mdx@3.1.1': @@ -13049,8 +13081,6 @@ snapshots: fast-deep-equal: 3.1.3 lodash: 4.18.1 - '@sec-ant/readable-stream@0.4.1': {} - '@shikijs/core@1.29.2': dependencies: '@shikijs/engine-javascript': 1.29.2 @@ -13126,8 +13156,6 @@ snapshots: '@sindresorhus/is@6.3.1': {} - '@sindresorhus/is@7.2.0': {} - '@sindresorhus/merge-streams@4.0.0': {} '@slorber/react-helmet-async@1.3.0(react-dom@19.2.5(react@19.2.5))(react@19.2.5)': @@ -14070,8 +14098,6 @@ snapshots: dependencies: run-applescript: 7.1.0 - byte-counter@0.1.0: {} - bytes@3.0.0: {} bytes@3.1.2: {} @@ -14090,16 +14116,6 @@ snapshots: normalize-url: 8.1.1 responselike: 3.0.0 - cacheable-request@13.0.18: - dependencies: - '@types/http-cache-semantics': 4.2.0 - get-stream: 9.0.1 - http-cache-semantics: 4.2.0 - keyv: 5.6.0 - mimic-response: 4.0.0 - normalize-url: 8.1.1 - responselike: 4.0.2 - call-bind-apply-helpers@1.0.2: dependencies: es-errors: 1.3.0 @@ -14430,20 +14446,22 @@ snapshots: optionalDependencies: typescript: 6.0.3 - crawlee@4.0.0-beta.61(@types/node@24.12.2)(playwright@1.60.0)(puppeteer@25.1.0): - dependencies: - '@crawlee/basic': 4.0.0-beta.61 - '@crawlee/browser': 4.0.0-beta.61(playwright@1.60.0)(puppeteer@25.1.0) - '@crawlee/browser-pool': 4.0.0-beta.61(playwright@1.60.0)(puppeteer@25.1.0) - '@crawlee/cheerio': 4.0.0-beta.61 - '@crawlee/cli': 4.0.0-beta.61(@types/node@24.12.2) - '@crawlee/core': 4.0.0-beta.61 - '@crawlee/http': 4.0.0-beta.61 - '@crawlee/jsdom': 4.0.0-beta.61 - '@crawlee/linkedom': 4.0.0-beta.61 - '@crawlee/playwright': 4.0.0-beta.61(playwright@1.60.0)(puppeteer@25.1.0) - '@crawlee/puppeteer': 4.0.0-beta.61(playwright@1.60.0)(puppeteer@25.1.0) - '@crawlee/utils': 4.0.0-beta.61 + crawlee@4.0.0-beta.105(@types/node@24.12.2)(playwright@1.60.0)(puppeteer@25.1.0): + dependencies: + '@crawlee/basic': 4.0.0-beta.105 + '@crawlee/browser': 4.0.0-beta.105(playwright@1.60.0)(puppeteer@25.1.0) + '@crawlee/browser-pool': 4.0.0-beta.105(playwright@1.60.0)(puppeteer@25.1.0) + '@crawlee/cheerio': 4.0.0-beta.105 + '@crawlee/cli': 4.0.0-beta.105(@types/node@24.12.2) + '@crawlee/core': 4.0.0-beta.105 + '@crawlee/fs-storage': 4.0.0-beta.105 + '@crawlee/http': 4.0.0-beta.105 + '@crawlee/impit-client': 4.0.0-beta.105 + '@crawlee/jsdom': 4.0.0-beta.105 + '@crawlee/linkedom': 4.0.0-beta.105 + '@crawlee/playwright': 4.0.0-beta.105(playwright@1.60.0)(puppeteer@25.1.0) + '@crawlee/puppeteer': 4.0.0-beta.105(playwright@1.60.0)(puppeteer@25.1.0) + '@crawlee/utils': 4.0.0-beta.105 import-local: 3.2.0 tslib: 2.8.1 optionalDependencies: @@ -14670,10 +14688,6 @@ snapshots: decode-uri-component@0.4.1: {} - decompress-response@10.0.0: - dependencies: - mimic-response: 4.0.0 - decompress-response@6.0.0: dependencies: mimic-response: 3.1.0 @@ -14808,10 +14822,6 @@ snapshots: dependencies: is-obj: 2.0.0 - dot-prop@7.2.0: - dependencies: - type-fest: 2.19.0 - dot-prop@8.0.2: dependencies: type-fest: 3.13.1 @@ -15206,8 +15216,6 @@ snapshots: form-data-encoder@2.1.4: {} - form-data-encoder@4.1.0: {} - form-data@4.0.5: dependencies: asynckit: 0.4.0 @@ -15275,11 +15283,6 @@ snapshots: get-stream@6.0.1: {} - get-stream@9.0.1: - dependencies: - '@sec-ant/readable-stream': 0.4.1 - is-stream: 4.0.1 - get-tsconfig@4.13.7: dependencies: resolve-pkg-maps: 1.0.0 @@ -15362,16 +15365,6 @@ snapshots: gopd@1.2.0: {} - got-scraping@4.2.1: - dependencies: - got: 14.6.6 - header-generator: 2.1.82 - http2-wrapper: 2.2.1 - mimic-response: 4.0.0 - ow: 1.1.1 - quick-lru: 7.3.0 - tslib: 2.8.1 - got@12.6.1: dependencies: '@sindresorhus/is': 5.6.0 @@ -15386,21 +15379,6 @@ snapshots: p-cancelable: 3.0.0 responselike: 3.0.0 - got@14.6.6: - dependencies: - '@sindresorhus/is': 7.2.0 - byte-counter: 0.1.0 - cacheable-lookup: 7.0.0 - cacheable-request: 13.0.18 - decompress-response: 10.0.0 - form-data-encoder: 4.1.0 - http2-wrapper: 2.2.1 - keyv: 5.6.0 - lowercase-keys: 3.0.0 - p-cancelable: 4.0.1 - responselike: 4.0.2 - type-fest: 4.41.0 - graceful-fs@4.2.10: {} graceful-fs@4.2.11: {} @@ -15840,6 +15818,41 @@ snapshots: image-size@2.0.2: {} + impit-darwin-arm64@0.14.3: + optional: true + + impit-darwin-x64@0.14.3: + optional: true + + impit-linux-arm64-gnu@0.14.3: + optional: true + + impit-linux-arm64-musl@0.14.3: + optional: true + + impit-linux-x64-gnu@0.14.3: + optional: true + + impit-linux-x64-musl@0.14.3: + optional: true + + impit-win32-arm64-msvc@0.14.3: + optional: true + + impit-win32-x64-msvc@0.14.3: + optional: true + + impit@0.14.3: + optionalDependencies: + impit-darwin-arm64: 0.14.3 + impit-darwin-x64: 0.14.3 + impit-linux-arm64-gnu: 0.14.3 + impit-linux-arm64-musl: 0.14.3 + impit-linux-x64-gnu: 0.14.3 + impit-linux-x64-musl: 0.14.3 + impit-win32-arm64-msvc: 0.14.3 + impit-win32-x64-msvc: 0.14.3 + import-fresh@3.3.1: dependencies: parent-module: 1.0.1 @@ -15965,8 +15978,6 @@ snapshots: is-stream@2.0.1: {} - is-stream@4.0.1: {} - is-typedarray@1.0.0: {} is-wsl@2.2.0: @@ -16083,10 +16094,6 @@ snapshots: dependencies: json-buffer: 3.0.1 - keyv@5.6.0: - dependencies: - '@keyv/serialize': 1.1.1 - kind-of@6.0.3: {} kleur@3.0.3: {} @@ -17023,14 +17030,6 @@ snapshots: lodash.isequal: 4.5.0 vali-date: 1.0.0 - ow@1.1.1: - dependencies: - '@sindresorhus/is': 5.6.0 - callsites: 4.2.0 - dot-prop: 7.2.0 - lodash.isequal: 4.5.0 - vali-date: 1.0.0 - ow@2.0.0: dependencies: '@sindresorhus/is': 6.3.1 @@ -17098,8 +17097,6 @@ snapshots: p-cancelable@3.0.0: {} - p-cancelable@4.0.1: {} - p-event@6.0.1: dependencies: p-timeout: 6.1.4 @@ -17997,12 +17994,6 @@ snapshots: object-assign: 4.1.1 react-is: 16.13.1 - proper-lockfile@4.1.2: - dependencies: - graceful-fs: 4.2.11 - retry: 0.12.0 - signal-exit: 3.0.7 - property-information@7.1.0: {} proto-list@1.2.4: {} @@ -18478,17 +18469,11 @@ snapshots: dependencies: lowercase-keys: 3.0.0 - responselike@4.0.2: - dependencies: - lowercase-keys: 3.0.0 - restore-cursor@5.1.0: dependencies: onetime: 7.0.0 signal-exit: 4.1.0 - retry@0.12.0: {} - retry@0.13.1: {} reusify@1.1.0: {} diff --git a/src/actor.ts b/src/actor.ts index f9941488ff..1b29d2e72f 100644 --- a/src/actor.ts +++ b/src/actor.ts @@ -9,8 +9,8 @@ import type { UseStateOptions, } from '@crawlee/core'; import { Dataset, EventType, purgeDefaultStorages, RequestQueue, serviceLocator } from '@crawlee/core'; -import type { Awaitable, Constructor, Dictionary, SetStatusMessageOptions, StorageClient } from '@crawlee/types'; -import { sleep, snakeCaseToCamelCase } from '@crawlee/utils'; +import type { Awaitable, Constructor, Dictionary, StorageBackend } from '@crawlee/types'; +import { sleep } from '@crawlee/utils'; import type { ActorCallOptions, ActorStartOptions, @@ -34,12 +34,13 @@ import { decryptInputSecrets } from '@apify/input_secrets'; import log from '@apify/log'; import { addTimeoutToPromise } from '@apify/timeout'; +import type { RequestQueueAccessMode } from './apify_request_queue_backend.js'; import { - ApifyStorageClient, + ApifyStorageBackend, type PpeAwarePushDataContext, pushDataChargingContext, USES_PUSH_DATA_INTERCEPTION, -} from './apify_storage_client.js'; +} from './apify_storage_backend.js'; import type { ChargeOptions, ChargeResult } from './charging.js'; import { ChargingManager, pushDataAndCharge } from './charging.js'; import type { ConfigurationOptions } from './configuration.js'; @@ -51,10 +52,31 @@ import type { ProxyConfigurationOptions } from './proxy_configuration.js'; import { ProxyConfiguration } from './proxy_configuration.js'; import type { OpenStorageOptions, StorageIdentifier, StorageIdentifierWithoutAlias } from './storage.js'; import { openStorage } from './storage.js'; -import { checkCrawleeVersion, getSystemInfo, isNonEmptyObject, printOutdatedSdkWarning, validate } from './utils.js'; +import { + checkCrawleeVersion, + getSystemInfo, + isNonEmptyObject, + printOutdatedSdkWarning, + snakeCaseToCamelCase, + validate, +} from './utils.js'; export interface InitOptions { - storage?: StorageClient; + storage?: StorageBackend; + /** + * Determines how request queues opened on the Apify platform are consumed. + * + * - `'single'` (default) assumes this run is the only consumer of its request queues. Requests + * are not locked server-side and the queue head is estimated locally, which means fewer + * (paid) API calls and better performance. + * - `'shared'` locks every fetched request server-side, so any number of concurrent consumers + * (e.g. several Actor runs) can safely process the same queue, at the cost of roughly one + * extra API call per processed request. + * + * Only applies on the Apify platform (or with `forceCloud`); local storage ignores it. + * @default 'single' + */ + requestQueueAccess?: RequestQueueAccessMode; /** * Whether to automatically handle platform shutdown signals. * When enabled, `Actor.exit()` is called on `aborting` events and `Actor.reboot()` on `migrating` events. @@ -100,6 +122,13 @@ export interface ExitOptions { export interface MainOptions extends ExitOptions, InitOptions {} +export interface SetStatusMessageOptions { + /** If `true`, the status message is treated as final and won't be overwritten by the platform. */ + isStatusMessageTerminal?: boolean; + /** Log level used when the status message is also logged locally. Defaults to `INFO`. */ + level?: 'DEBUG' | 'INFO' | 'WARNING' | 'ERROR'; +} + /** * Parsed representation of the Apify environment variables. * This object is returned by the {@apilink Actor.getEnv} function. @@ -392,7 +421,7 @@ export class Actor { * Configuration of this SDK instance (provided to its constructor). See {@apilink Configuration} for details. * @internal */ - readonly config: Configuration; + readonly configuration: Configuration; /** * Default {@apilink ApifyClient} instance. @@ -444,6 +473,9 @@ export class Actor { */ purgedStorageAliases = new Set(); + /** How Apify platform request queues are consumed; set from {@link InitOptions.requestQueueAccess}. */ + private requestQueueAccess: RequestQueueAccessMode = 'single'; + constructor(options: ActorOptions = {}) { const { configuration, ...configOptions } = options; if (configuration) { @@ -459,16 +491,16 @@ export class Actor { 'not a crawlee Configuration, otherwise APIFY_*/ACTOR_* environment variables are not resolved.', ); } - this.config = configuration; + this.configuration = configuration; } else if (Object.keys(configOptions).length === 0) { // use default configuration object if nothing overridden (it fallbacks to env vars) - this.config = Configuration.getGlobalConfig(); + this.configuration = Configuration.getGlobalConfiguration(); } else { - this.config = new Configuration(configOptions); + this.configuration = new Configuration(configOptions); } this.apifyClient = this.newClient(); - this.eventManager = new PlatformEventManager(this.config); - this.chargingManager = new ChargingManager(this.config, this.apifyClient); + this.eventManager = new PlatformEventManager(this.configuration); + this.chargingManager = new ChargingManager(this.configuration, this.apifyClient); } /** @@ -584,15 +616,15 @@ export class Actor { // Register this Actor's config as the global one so crawlee storages and // the event manager resolve the same instance (`availableMemoryRatio` / // `disableBrowserSandbox` at-home defaults now live in `Configuration`). - serviceLocator.setConfiguration(this.config); + serviceLocator.setConfiguration(this.configuration); + + this.requestQueueAccess = options.requestQueueAccess ?? 'single'; if (this.isAtHome()) { - serviceLocator.setStorageClient( - new ApifyStorageClient(this.apifyClient, this.config, () => this.chargingManager), - ); + serviceLocator.setStorageBackend(this.createApifyStorageBackend()); serviceLocator.setEventManager(this.eventManager); } else if (options.storage) { - serviceLocator.setStorageClient(options.storage); + serviceLocator.setStorageBackend(options.storage); } // Init the event manager the config uses @@ -626,7 +658,7 @@ export class Actor { } await purgeDefaultStorages({ - config: this.config, + configuration: this.configuration, onlyPurgeOnce: true, }); log.debug(`Default storages purged`); @@ -656,7 +688,7 @@ export class Actor { this._ensureActorInit('exit'); - const client = serviceLocator.getStorageClient(); + const client = serviceLocator.getStorageBackend(); const events = serviceLocator.getEventManager(); // Remove graceful shutdown handlers to prevent them from interfering with exit @@ -896,8 +928,8 @@ export class Actor { return; } - const { customAfterSleepMillis = this.config.metamorphAfterSleepMillis, ...metamorphOpts } = options; - const runId = this.config.actorRunId!; + const { customAfterSleepMillis = this.configuration.metamorphAfterSleepMillis, ...metamorphOpts } = options; + const runId = this.configuration.actorRunId!; await this.apifyClient.run(runId).metamorph(targetActorId, input, metamorphOpts); // Wait some time for container to be stopped. @@ -940,11 +972,11 @@ export class Actor { .map(async (x: (...args: unknown[]) => unknown) => x({})), ]); - const runId = this.config.actorRunId!; + const runId = this.configuration.actorRunId!; await this.apifyClient.run(runId).reboot(); // Wait some time for container to be stopped. - const { customAfterSleepMillis = this.config.metamorphAfterSleepMillis } = options; + const { customAfterSleepMillis = this.configuration.metamorphAfterSleepMillis } = options; await sleep(customAfterSleepMillis); } @@ -986,7 +1018,7 @@ export class Actor { return undefined; } - const runId = this.config.actorRunId!; + const runId = this.configuration.actorRunId!; if (!runId) { throw new Error(`Environment variable ${ACTOR_ENV_VARS.RUN_ID} is not set!`); } @@ -1031,27 +1063,14 @@ export class Actor { break; } - const client = serviceLocator.getStorageClient(); - - // just to be sure, this should be fast - await addTimeoutToPromise( - async () => - client.setStatusMessage!(statusMessage, { - isStatusMessageTerminal, - level, - }), - 1000, - 'Setting status message timed out after 1s', - ).catch((e) => log.warning(e.message)); - - const runId = this.config.actorRunId!; + const runId = this.configuration.actorRunId!; if (runId) { // just to be sure, this should be fast const run = await addTimeoutToPromise( - async () => this.apifyClient.run(runId).get(), + async () => this.apifyClient.run(runId).update({ statusMessage, isStatusMessageTerminal }), 1000, - 'Getting the current run timed out after 1s', + 'Setting status message timed out after 1s', ).catch((e) => log.warning(e.message)); if (run) { @@ -1242,8 +1261,8 @@ export class Actor { async getInput(): Promise { this._ensureActorInit('getInput'); - const { inputSecretsPrivateKeyFile, inputSecretsPrivateKeyPassphrase } = this.config; - const rawInput = await this.getValue(this.config.inputKey); + const { inputSecretsPrivateKeyFile, inputSecretsPrivateKeyPassphrase } = this.configuration; + const rawInput = await this.getValue(this.configuration.inputKey); let input = rawInput as T; @@ -1332,9 +1351,6 @@ export class Actor { const queue = await this._openStorage(RequestQueue, queueIdOrName, options); - // eslint-disable-next-line dot-notation - queue['initialCount'] = (await queue.client.getMetadata())?.totalRequestCount ?? 0; - return queue; } @@ -1393,7 +1409,7 @@ export class Actor { return undefined; } - const proxyConfiguration = new ProxyConfiguration(options, this.config); + const proxyConfiguration = new ProxyConfiguration(options, this.configuration); if (await proxyConfiguration.initialize({ checkAccess })) { return proxyConfiguration; @@ -1494,13 +1510,13 @@ export class Actor { * @ignore */ newClient(options: ApifyClientOptions = {}): ApifyClient { - const { storageDir, ...storageClientOptions } = (this.config.storageClientOptions ?? {}) as Dictionary; + const { storageDir, ...storageClientOptions } = (this.configuration.storageClientOptions ?? {}) as Dictionary; const { apifyVersion, crawleeVersion } = getSystemInfo(); return new ApifyClient({ - baseUrl: this.config.apiBaseUrl, - publicBaseUrl: this.config.apiPublicBaseUrl, - token: this.config.token, + baseUrl: this.configuration.apiBaseUrl, + publicBaseUrl: this.configuration.apiPublicBaseUrl, + token: this.configuration.token, userAgentSuffix: [`SDK/${apifyVersion}`, `Crawlee/${crawleeVersion}`], ...storageClientOptions, ...options, // allow overriding the instance configuration @@ -1532,7 +1548,7 @@ export class Actor { this._ensureActorInit('useState'); const kvStore = await KeyValueStore.open(options?.keyValueStoreName, { - config: options?.config || Configuration.getGlobalConfig(), + configuration: options?.configuration || Configuration.getGlobalConfiguration(), }); return kvStore.getAutoSavedValue(name || 'APIFY_GLOBAL_STATE', defaultValue); } @@ -2198,8 +2214,8 @@ export class Actor { } /** Default {@apilink Configuration} instance. */ - static get config(): Configuration { - return Actor.getDefaultInstance().config; + static get configuration(): Configuration { + return Actor.getDefaultInstance().configuration; } /** @internal */ @@ -2209,7 +2225,7 @@ export class Actor { } private usesPushDataInterception(dataset: Dataset): boolean { - return Boolean((dataset.client as any)[USES_PUSH_DATA_INTERCEPTION]); + return Boolean((dataset.backend as any)[USES_PUSH_DATA_INTERCEPTION]); } private async pushDataViaInterceptedClient( @@ -2252,7 +2268,7 @@ export class Actor { }; } - const isDefaultDataset = dataset.id === this.config.defaultDatasetId; + const isDefaultDataset = dataset.id === this.configuration.defaultDatasetId; return pushDataAndCharge({ chargingManager: this.chargingManager, @@ -2271,14 +2287,20 @@ export class Actor { options: OpenStorageOptions = {}, ) { return openStorage(storageClass, identifier, { - config: this.config, - client: options.forceCloud - ? new ApifyStorageClient(this.apifyClient, this.config, () => this.chargingManager) - : undefined, + config: this.configuration, + backend: options.forceCloud ? this.createApifyStorageBackend() : undefined, purgedStorageAliases: this.purgedStorageAliases, }); } + private createApifyStorageBackend(): ApifyStorageBackend { + return new ApifyStorageBackend(this.apifyClient, { + configuration: this.configuration, + requestQueueAccess: this.requestQueueAccess, + getChargingManager: () => this.chargingManager, + }); + } + private _ensureActorInit(methodCalled: string) { // If we already warned the user once, don't do it again to prevent spam if (this.warnedAboutMissingInitCall) { diff --git a/src/apify_dataset_backend.ts b/src/apify_dataset_backend.ts new file mode 100644 index 0000000000..6010c21651 --- /dev/null +++ b/src/apify_dataset_backend.ts @@ -0,0 +1,40 @@ +import type { DatasetBackend, DatasetBackendListOptions, DatasetInfo, Dictionary, PaginatedList } from '@crawlee/types'; +import type { DatasetClient } from 'apify-client'; + +/** + * Implements crawlee v4's {@link DatasetBackend} interface on top of `apify-client`'s + * dataset API. A thin method-mapping wrapper — the interfaces differ only in naming + * (`getMetadata`/`get`, `drop`/`delete`, `pushData`/`pushItems`, `getData`/`listItems`). + * + * @internal + */ +export class ApifyDatasetBackend implements DatasetBackend { + constructor(private readonly client: DatasetClient) {} + + async getMetadata(): Promise { + const metadata = await this.client.get(); + if (!metadata) { + throw new Error('Dataset not found or has been deleted.'); + } + return metadata; + } + + async drop(): Promise { + await this.client.delete(); + } + + async purge(): Promise { + throw new Error( + 'Purging a dataset is not supported on the Apify platform. ' + + 'Use `drop()` to delete the dataset entirely, or open a new dataset instead.', + ); + } + + async pushData(items: Dictionary[]): Promise { + await this.client.pushItems(items); + } + + async getData(options?: DatasetBackendListOptions): Promise> { + return await this.client.listItems(options); + } +} diff --git a/src/apify_key_value_store_backend.ts b/src/apify_key_value_store_backend.ts new file mode 100644 index 0000000000..aacc846920 --- /dev/null +++ b/src/apify_key_value_store_backend.ts @@ -0,0 +1,73 @@ +import type { + KeyValueStoreBackend, + KeyValueStoreInfo, + KeyValueStoreInputRecord, + KeyValueStoreItemData, + KeyValueStoreListKeysOptions, + KeyValueStoreListKeysResult, + KeyValueStoreRecord, +} from '@crawlee/types'; +import type { KeyValueStoreClient } from 'apify-client'; + +/** + * Implements crawlee v4's {@link KeyValueStoreBackend} interface on top of `apify-client`'s + * key-value store API. Mostly a method-mapping wrapper (`getValue`/`getRecord`, + * `setValue`/`setRecord`, `drop`/`delete`, ...); the one semantic difference is that storage + * backends are byte transports, so records are read unparsed (see {@link getValue}). + * + * @internal + */ +export class ApifyKeyValueStoreBackend implements KeyValueStoreBackend { + constructor(private readonly client: KeyValueStoreClient) {} + + async getMetadata(): Promise { + const metadata = await this.client.get(); + if (!metadata) { + throw new Error('Key-value store not found or has been deleted.'); + } + return metadata; + } + + async drop(): Promise { + await this.client.delete(); + } + + async purge(): Promise { + throw new Error( + 'Purging a key-value store is not supported on the Apify platform. ' + + 'Use `drop()` to delete the store entirely, or open a new store instead.', + ); + } + + async getValue(key: string): Promise { + // Storage backends are byte transports — the KeyValueStore frontend parses values + // according to their content type, so the record must be returned unparsed. + return this.client.getRecord(key, { buffer: true }); + } + + async setValue(record: KeyValueStoreInputRecord): Promise { + await this.client.setRecord(record as Parameters[0]); + } + + async deleteValue(key: string): Promise { + await this.client.deleteRecord(key); + } + + async listKeys(options?: KeyValueStoreListKeysOptions): Promise { + const result = await this.client.listKeys(options); + // The API does not report a content type for listed keys; crawlee's item shape + // requires the field, so it is left undefined via the cast. + return { + ...result, + items: result.items.map(({ key, size }) => ({ key, size }) as KeyValueStoreItemData), + }; + } + + async getPublicUrl(key: string): Promise { + return this.client.getRecordPublicUrl(key); + } + + async recordExists(key: string): Promise { + return this.client.recordExists(key); + } +} diff --git a/src/apify_request_queue_backend.ts b/src/apify_request_queue_backend.ts new file mode 100644 index 0000000000..66acd30345 --- /dev/null +++ b/src/apify_request_queue_backend.ts @@ -0,0 +1,183 @@ +import { createHash } from 'node:crypto'; + +import type { + BatchAddRequestsResult, + QueueOperationInfo, + RequestQueueBackend, + RequestQueueInfo, + RequestQueueOperationOptions, + RequestSchema, + UpdateRequestSchema, +} from '@crawlee/types'; +import type { RequestQueueClient as ApifyRequestQueueApiClient } from 'apify-client'; + +/** + * Determines how an Apify platform request queue is consumed. + * + * - `'single'` — optimized for a single consumer. The client keeps a local estimate of the queue + * head and never locks requests, which means fewer API calls, better performance and lower cost. + * Multiple producers may still add requests concurrently, but only one client may *consume* + * (fetch and process) them. + * - `'shared'` — safe for multiple concurrent consumers (e.g. several Actor runs processing one + * queue). Requests are locked server-side while they are being processed, at the cost of more + * API calls. + */ +export type RequestQueueAccessMode = 'single' | 'shared'; + +/** Apify request IDs are the first 15 chars of a base64 SHA-256 of the unique key. */ +const REQUEST_ID_LENGTH = 15; + +/** + * Derives a request id from its unique key, exactly as the Apify platform does + * (`sha256(uniqueKey)` → base64 → strip `+`/`/`/`=` → first 15 chars). Lets us + * address a request by unique key without an extra round-trip. + */ +export function uniqueKeyToRequestId(uniqueKey: string): string { + const hash = createHash('sha256').update(uniqueKey).digest('base64').replace(/[+/=]/g, ''); + return hash.slice(0, REQUEST_ID_LENGTH); +} + +/** + * Common base of the Apify platform implementations of Crawlee v4's stateful, pull-based + * {@link RequestQueueBackend} interface, built on top of `apify-client`'s REST request-queue API. + * + * The mode-specific consumption logic lives in the subclasses: + * {@link ApifyRequestQueueSingleBackend} (single consumer, no locking) and + * {@link ApifyRequestQueueSharedBackend} (multiple consumers, server-side locking). + * Modeled on the Apify Python SDK's request-queue clients. + * + * @internal + */ +export abstract class ApifyRequestQueueBackend implements RequestQueueBackend { + /** + * Local estimates of the queue counters, updated as this client adds/handles requests. The API + * counters can lag behind by a few seconds, so {@link getMetadata} reports whichever is higher. + */ + protected estimatedTotalRequestCount = 0; + protected estimatedHandledRequestCount = 0; + + constructor(protected readonly client: ApifyRequestQueueApiClient) {} + + abstract addBatchOfRequests( + requests: RequestSchema[], + options?: RequestQueueOperationOptions, + ): Promise; + + abstract getRequest(uniqueKey: string): Promise; + + abstract fetchNextRequest(): Promise; + + abstract markRequestAsHandled(request: UpdateRequestSchema): Promise; + + abstract reclaimRequest( + request: UpdateRequestSchema, + options?: RequestQueueOperationOptions, + ): Promise; + + abstract isEmpty(): Promise; + + abstract isFinished(): Promise; + + async setExpectedRequestProcessingTimeSecs(_secs: number): Promise { + // Only relevant for backends that reserve requests via locking; see the shared backend. + } + + async getMetadata(): Promise { + const metadata = await this.client.get(); + if (!metadata) { + throw new Error('Request queue not found or has been deleted.'); + } + return { + id: metadata.id, + name: metadata.name, + createdAt: metadata.createdAt, + modifiedAt: metadata.modifiedAt, + accessedAt: metadata.accessedAt, + totalRequestCount: Math.max(metadata.totalRequestCount, this.estimatedTotalRequestCount), + handledRequestCount: Math.max(metadata.handledRequestCount, this.estimatedHandledRequestCount), + pendingRequestCount: metadata.pendingRequestCount, + }; + } + + async drop(): Promise { + await this.client.delete(); + } + + async purge(): Promise { + throw new Error( + 'Purging a request queue is not supported on the Apify platform. ' + + 'Use `drop()` to delete the queue entirely, or open a new queue instead.', + ); + } + + protected requestIdFromUniqueKey(uniqueKey: string): string { + return uniqueKeyToRequestId(uniqueKey); + } + + /** + * Fetches the full request record by id. + * + * The apify-client return type understates the payload (the API returns the complete request + * record including `userData`, `payload`, `handledAt`, ...), hence the cast. + */ + protected async getRequestById(id: string): Promise { + const request = await this.client.getRequest(id); + return (request as unknown as UpdateRequestSchema | undefined) ?? undefined; + } + + /** + * Adds new requests to the platform queue. The API assigns ids itself, so any incoming id is + * stripped to pass its strict input validation. `apify-client` internally chunks the batch and + * retries transient failures. + */ + protected async sendBatch(requests: RequestSchema[], forefront?: boolean): Promise { + const apiRequests = requests.map((request) => { + const { id: _id, ...rest } = request; + return rest; + }); + const result = await this.client.batchAddRequests( + apiRequests as Parameters[0], + { forefront }, + ); + return result as unknown as BatchAddRequestsResult; + } + + /** Updates a request record on the platform and maps the result to crawlee's shape. */ + protected async updateRequestOnPlatform( + request: UpdateRequestSchema, + forefront?: boolean, + ): Promise { + const result = await this.client.updateRequest( + request as Parameters[0], + { forefront }, + ); + return { + requestId: result.requestId, + wasAlreadyPresent: result.wasAlreadyPresent, + wasAlreadyHandled: result.wasAlreadyHandled, + }; + } + + /** Counts freshly added requests from an add-batch result into the local metadata estimates. */ + protected recordAddedRequests(result: BatchAddRequestsResult): void { + const newRequestCount = result.processedRequests.filter( + (request) => !request.wasAlreadyPresent && !request.wasAlreadyHandled, + ).length; + this.estimatedTotalRequestCount += newRequestCount; + } +} + +/** + * A minimal FIFO mutex — serializes the async critical sections passed to {@link runExclusive}. + * @internal + */ +export class AsyncLock { + private tail: Promise = Promise.resolve(); + + async runExclusive(fn: () => Promise): Promise { + const run = this.tail.then(fn); + // Keep the chain alive even when the critical section throws. + this.tail = run.catch(() => {}); + return run; + } +} diff --git a/src/apify_request_queue_shared_backend.ts b/src/apify_request_queue_shared_backend.ts new file mode 100644 index 0000000000..b0749f879c --- /dev/null +++ b/src/apify_request_queue_shared_backend.ts @@ -0,0 +1,257 @@ +import type { + BatchAddRequestsResult, + ProcessedRequest, + QueueOperationInfo, + RequestQueueOperationOptions, + RequestSchema, + UpdateRequestSchema, +} from '@crawlee/types'; +import { LruCache } from '@apify/datastructures'; +import log from '@apify/log'; + +import { ApifyRequestQueueBackend, AsyncLock } from './apify_request_queue_backend.js'; + +/** Maximum number of request dedup records cached locally. */ +const MAX_CACHED_REQUESTS = 1_000_000; + +/** Default lock duration for requests fetched via `fetchNextRequest`. */ +const DEFAULT_REQUEST_LOCK_SECS = 3 * 60; + +/** How many head requests to lock per `listAndLockHead` round-trip. */ +const HEAD_LOCK_LIMIT = 25; + +/** Locally cached dedup information about a request that reached the platform. */ +interface CachedRequestInfo { + wasAlreadyHandled: boolean; +} + +/** + * Request queue backend safe for multi-consumer scenarios on the Apify platform. + * + * Requests fetched via {@link fetchNextRequest} are locked server-side (`listAndLockHead`), so any + * number of clients — including other Actor runs — can process the same queue concurrently without + * handing out a request twice. The lock is held until the request is marked as handled or + * reclaimed, and its duration follows the consumer's expected processing time + * (see {@link setExpectedRequestProcessingTimeSecs}). This consistency costs roughly one extra API + * call per processed request compared to the single-consumer backend. + * + * @internal + */ +export class ApifyRequestQueueSharedBackend extends ApifyRequestQueueBackend { + /** Ids of requests locked by this client and waiting to be handed out by `fetchNextRequest`. */ + private readonly headIds: string[] = []; + + /** Dedup records for requests known to exist on the platform, keyed by id. */ + private readonly cachedRequestInfo = new LruCache({ maxLength: MAX_CACHED_REQUESTS }); + + /** Ids of requests currently being processed by this client. */ + private readonly inProgressIds = new Set(); + + /** Whether the last head read reported any locked requests left in the queue (any client's). */ + private queueHasLockedRequests?: boolean; + + /** Set after a forefront insert — the next head read starts fresh so the insert is honored. */ + private shouldCheckForefrontRequests = false; + + /** Lock duration applied to fetched requests; raised via `setExpectedRequestProcessingTimeSecs`. */ + private lockSecs = DEFAULT_REQUEST_LOCK_SECS; + + /** Serializes head reads and reclaims — both reorder the shared head state. */ + private readonly headLock = new AsyncLock(); + + override async setExpectedRequestProcessingTimeSecs(secs: number): Promise { + // Only ever raise the lock duration — several consumers may share this client, and a + // short-lived one must not cut the reservation of a long-running one short. + this.lockSecs = Math.max(this.lockSecs, secs); + } + + async addBatchOfRequests( + requests: RequestSchema[], + options: RequestQueueOperationOptions = {}, + ): Promise { + const { forefront = false } = options; + + // Skip requests this client already knows reached the platform — a platform write costs an + // API call and a paid write operation. Whether such a request has been handled in the + // meantime by another client is unknowable locally, so report its last known state. + const alreadyPresent: ProcessedRequest[] = []; + const newRequests: RequestSchema[] = []; + for (const request of requests) { + const id = this.requestIdFromUniqueKey(request.uniqueKey); + const cached = this.cachedRequestInfo.get(id); + if (cached) { + alreadyPresent.push({ + requestId: id, + uniqueKey: request.uniqueKey, + wasAlreadyPresent: true, + wasAlreadyHandled: cached.wasAlreadyHandled || request.handledAt != null, + }); + } else { + newRequests.push(request); + } + } + + let result: BatchAddRequestsResult = { processedRequests: [], unprocessedRequests: [] }; + if (newRequests.length > 0) { + result = await this.sendBatch(newRequests, forefront); + for (const processed of result.processedRequests) { + this.cacheRequestInfo(processed.requestId, { wasAlreadyHandled: processed.wasAlreadyHandled }); + } + // A forefront insert changes the head order — have the next head read re-fetch the + // front of the queue instead of draining the local buffer first. + if (forefront) { + this.shouldCheckForefrontRequests = true; + } + } + + result.processedRequests.push(...alreadyPresent); + this.recordAddedRequests(result); + return result; + } + + async getRequest(uniqueKey: string): Promise { + // The queue is shared — another client may modify a request at any time, so always read + // through to the platform. + return this.getRequestById(this.requestIdFromUniqueKey(uniqueKey)); + } + + async fetchNextRequest(): Promise { + const id = await this.headLock.runExclusive(async () => { + await this.ensureHeadIsNonEmpty(); + return this.headIds.shift(); + }); + if (!id) return undefined; + + // Head items carry only partial request data (no userData, payload or headers), so the + // full record has to be hydrated with a round-trip. + const request = await this.getRequestById(id); + if (!request) { + // The head read can briefly report a request the main table does not serve yet — leave + // it out of the local head; it will reappear in a later head read. + log.debug(`Request fetched from the queue head was not found (id: ${id}), will be retried later`); + return undefined; + } + if (request.handledAt) { + // Handled by another client in the meantime. + this.cacheRequestInfo(id, { wasAlreadyHandled: true }); + return undefined; + } + this.inProgressIds.add(id); + return request; + } + + async markRequestAsHandled(request: UpdateRequestSchema): Promise { + const id = this.requestIdFromUniqueKey(request.uniqueKey); + // Contract: marking a request that does not exist in the queue is a no-op — it must not be + // added as a side effect (the platform update endpoint would upsert it). + if (!(await this.isKnownOrExists(id))) { + this.inProgressIds.delete(id); + return undefined; + } + + const handledAt = request.handledAt ?? new Date().toISOString(); + const info = await this.updateRequestOnPlatform({ ...request, id, handledAt }); + + this.inProgressIds.delete(id); + this.cacheRequestInfo(id, { wasAlreadyHandled: true }); + if (!info.wasAlreadyHandled) { + this.estimatedHandledRequestCount += 1; + } + return info; + } + + async reclaimRequest( + request: UpdateRequestSchema, + options: RequestQueueOperationOptions = {}, + ): Promise { + const { forefront = false } = options; + const id = this.requestIdFromUniqueKey(request.uniqueKey); + // Same contract as `markRequestAsHandled` — never insert as a side effect. + if (!(await this.isKnownOrExists(id))) { + this.inProgressIds.delete(id); + return undefined; + } + + return this.headLock.runExclusive(async () => { + const info = await this.updateRequestOnPlatform({ ...request, id, handledAt: undefined }, forefront); + + // Release the server-side lock so the request becomes fetchable again immediately — + // by any consumer — rather than only after the lock expires. + try { + await this.client.deleteRequestLock(id, { forefront }); + } catch (err) { + log.debug(`Failed to delete the lock of a reclaimed request (id: ${id}): ${(err as Error).message}`); + } + + this.inProgressIds.delete(id); + this.cacheRequestInfo(id, { wasAlreadyHandled: false }); + if (forefront) { + this.shouldCheckForefrontRequests = true; + } + if (info.wasAlreadyHandled) { + this.estimatedHandledRequestCount -= 1; + } + return info; + }); + } + + async isEmpty(): Promise { + return this.headLock.runExclusive(async () => { + if (this.headIds.length > 0) return false; + await this.listAndLockHead(1); + return this.headIds.length === 0; + }); + } + + async isFinished(): Promise { + return this.headLock.runExclusive(async () => { + if (this.headIds.length > 0) return false; + // The head read also refreshes `queueHasLockedRequests`, so the order matters here. + await this.listAndLockHead(1); + return this.headIds.length === 0 && !this.queueHasLockedRequests; + }); + } + + /** Must be called with the head lock held. */ + private async ensureHeadIsNonEmpty(): Promise { + if (this.headIds.length > 1 && !this.shouldCheckForefrontRequests) { + return; + } + await this.listAndLockHead(HEAD_LOCK_LIMIT); + } + + /** Must be called with the head lock held. */ + private async listAndLockHead(limit: number): Promise { + // After a forefront insert the local buffer no longer starts at the true front of the + // queue — re-fetch the front and keep the already-locked leftovers for afterwards. + let leftoverIds: string[] = []; + if (this.shouldCheckForefrontRequests) { + leftoverIds = this.headIds.splice(0); + this.shouldCheckForefrontRequests = false; + } + + const head = await this.client.listAndLockHead({ limit, lockSecs: this.lockSecs }); + this.queueHasLockedRequests = head.queueHasLockedRequests; + + for (const item of head.items) { + if (this.inProgressIds.has(item.id)) continue; + if (this.headIds.includes(item.id) || leftoverIds.includes(item.id)) continue; + this.cacheRequestInfo(item.id, { wasAlreadyHandled: false }); + this.headIds.push(item.id); + } + this.headIds.push(...leftoverIds); + } + + private async isKnownOrExists(id: string): Promise { + if (this.inProgressIds.has(id) || this.cachedRequestInfo.get(id)) { + return true; + } + return (await this.getRequestById(id)) !== undefined; + } + + private cacheRequestInfo(id: string, info: CachedRequestInfo): void { + // `LruCache.add` does not overwrite existing entries, so remove first. + this.cachedRequestInfo.remove(id); + this.cachedRequestInfo.add(id, info); + } +} diff --git a/src/apify_request_queue_single_backend.ts b/src/apify_request_queue_single_backend.ts new file mode 100644 index 0000000000..2096ad4d07 --- /dev/null +++ b/src/apify_request_queue_single_backend.ts @@ -0,0 +1,297 @@ +import type { + BatchAddRequestsResult, + ProcessedRequest, + QueueOperationInfo, + RequestQueueOperationOptions, + RequestSchema, + UpdateRequestSchema, +} from '@crawlee/types'; +import { LruCache } from '@apify/datastructures'; +import log from '@apify/log'; + +import { ApifyRequestQueueBackend } from './apify_request_queue_backend.js'; + +/** Maximum number of full request objects cached locally. */ +const MAX_CACHED_REQUESTS = 1_000_000; + +/** The maximum head items read count, limited by the API. */ +const MAX_HEAD_ITEMS = 1000; + +/** How many new head items to aim for per `listHead` round-trip. */ +const DESIRED_NEW_HEAD_ITEMS = 200; + +/** How many existing requests to prefetch into the local caches on the first add. */ +const INIT_CACHES_REQUEST_LIMIT = 10_000; + +/** + * Request queue backend optimized for single-consumer scenarios on the Apify platform. + * + * Minimizes API calls by keeping a local estimate of the queue head and a local cache of the + * requests this client added — a request fetched from the head is usually served straight from the + * cache, with no per-request round-trip and no server-side locking. + * + * ### Usage constraints + * + * - **Single consumer** — only one client may fetch and process requests from the queue at a time. + * - **Multiple producers allowed** — other clients may add requests concurrently, but their + * forefront requests may not be prioritized immediately, as this client relies on a local head + * estimate instead of frequent head fetching. + * - **Append-only queue** — other clients must not delete or modify existing requests, as such + * changes are not reflected in the local cache. Marking requests as handled elsewhere is + * tolerated, but may occasionally lead to a request being processed twice. + * + * If these constraints do not hold, use the shared backend (`requestQueueAccess: 'shared'`) instead. + * + * @internal + */ +export class ApifyRequestQueueSingleBackend extends ApifyRequestQueueBackend { + /** Local estimate of the queue head — request ids in the order they should be fetched. */ + private readonly headIds: string[] = []; + + /** Unhandled full request objects added by (or fetched through) this client, keyed by id. */ + private readonly cachedRequests = new LruCache({ maxLength: MAX_CACHED_REQUESTS }); + + /** Ids of requests known to be already handled — cheap dedup without caching full objects. */ + private readonly handledIds = new Set(); + + /** Ids of requests currently being processed by this client. */ + private readonly inProgressIds = new Set(); + + /** Memoized one-time prefetch of existing queue contents into the local caches. */ + private initCachesPromise?: Promise; + + async addBatchOfRequests( + requests: RequestSchema[], + options: RequestQueueOperationOptions = {}, + ): Promise { + const { forefront = false } = options; + await (this.initCachesPromise ??= this.initCaches()); + + // Split the batch into requests we already know about (dedup them locally — a platform + // write costs an API call and a paid write operation) and genuinely new ones. + const alreadyPresent: ProcessedRequest[] = []; + const newRequests: RequestSchema[] = []; + for (const request of requests) { + const id = this.requestIdFromUniqueKey(request.uniqueKey); + if (this.handledIds.has(id)) { + alreadyPresent.push({ + requestId: id, + uniqueKey: request.uniqueKey, + wasAlreadyPresent: true, + wasAlreadyHandled: true, + }); + } else if (this.cachedRequests.get(id)) { + alreadyPresent.push({ + requestId: id, + uniqueKey: request.uniqueKey, + wasAlreadyPresent: true, + wasAlreadyHandled: request.handledAt != null, + }); + } else { + newRequests.push(request); + } + } + + let result: BatchAddRequestsResult = { processedRequests: [], unprocessedRequests: [] }; + if (newRequests.length > 0) { + result = await this.sendBatch(newRequests, forefront); + + // Commit the accepted requests to the local caches and the head estimate. The platform + // response is authoritative — a request it reports as already handled (e.g. handled by + // a previous run of a resurrected Actor beyond the prefetch limit) must not re-enter + // the head. + const processedByKey = new Map( + result.processedRequests.map((processed) => [processed.uniqueKey, processed]), + ); + for (const request of newRequests) { + const processed = processedByKey.get(request.uniqueKey); + if (!processed) continue; // rejected by the platform, reported in `unprocessedRequests` + if (processed.wasAlreadyHandled) { + this.handledIds.add(processed.requestId); + continue; + } + this.cacheRequest({ ...request, id: processed.requestId }); + if (forefront) { + this.headIds.unshift(processed.requestId); + } else { + this.headIds.push(processed.requestId); + } + } + } + + result.processedRequests.push(...alreadyPresent); + this.recordAddedRequests(result); + return result; + } + + async getRequest(uniqueKey: string): Promise { + const id = this.requestIdFromUniqueKey(uniqueKey); + const cached = this.cachedRequests.get(id); + if (cached) return cached; + + const request = await this.getRequestById(id); + if (!request) return undefined; + + // Requests already in progress are ones the client knows about — no caching needed. + if (!this.inProgressIds.has(id)) { + if (request.handledAt) { + this.handledIds.add(id); + } else { + this.cacheRequest(request); + } + } + return request; + } + + async fetchNextRequest(): Promise { + await this.ensureHeadIsNonEmpty(); + + while (this.headIds.length > 0) { + const id = this.headIds.shift()!; + if (this.inProgressIds.has(id) || this.handledIds.has(id)) { + continue; + } + this.inProgressIds.add(id); + // Requests added by this client are served straight from the cache; only requests + // discovered via `listHead` (added by another producer) need a round-trip. + const request = this.cachedRequests.get(id) ?? (await this.getRequestById(id)); + if (!request) { + this.inProgressIds.delete(id); + continue; + } + if (request.handledAt) { + // Handled elsewhere in the meantime — skip it and remember the outcome. + this.inProgressIds.delete(id); + this.handledIds.add(id); + this.cachedRequests.remove(id); + continue; + } + return request; + } + return undefined; + } + + async markRequestAsHandled(request: UpdateRequestSchema): Promise { + const id = this.requestIdFromUniqueKey(request.uniqueKey); + // Contract: marking a request that does not exist in the queue is a no-op — it must not be + // added as a side effect (the platform update endpoint would upsert it). + if (!(await this.isKnownOrExists(id))) { + this.inProgressIds.delete(id); + return undefined; + } + + const handledAt = request.handledAt ?? new Date().toISOString(); + const info = await this.updateRequestOnPlatform({ ...request, id, handledAt }); + + this.inProgressIds.delete(id); + this.handledIds.add(id); + this.cachedRequests.remove(id); + if (!info.wasAlreadyHandled) { + this.estimatedHandledRequestCount += 1; + } + return info; + } + + async reclaimRequest( + request: UpdateRequestSchema, + options: RequestQueueOperationOptions = {}, + ): Promise { + const { forefront = false } = options; + const id = this.requestIdFromUniqueKey(request.uniqueKey); + // Same contract as `markRequestAsHandled` — never insert as a side effect. + if (!(await this.isKnownOrExists(id))) { + this.inProgressIds.delete(id); + return undefined; + } + + // Reclaiming returns the request to the queue for reprocessing. + const reclaimed: UpdateRequestSchema = { ...request, id, handledAt: undefined }; + const info = await this.updateRequestOnPlatform(reclaimed, forefront); + + this.inProgressIds.delete(id); + this.handledIds.delete(id); + this.cacheRequest(reclaimed); + // Return the id to the local head estimate right away — the platform head read can lag a + // few seconds behind the update, and `isFinished` must never report `true` while a + // reclaimed request is still waiting to be reprocessed. + if (!this.headIds.includes(id)) { + if (forefront) { + this.headIds.unshift(id); + } else { + this.headIds.push(id); + } + } + if (info.wasAlreadyHandled) { + this.estimatedHandledRequestCount -= 1; + } + return info; + } + + async isEmpty(): Promise { + await this.ensureHeadIsNonEmpty(); + return this.headIds.length === 0; + } + + async isFinished(): Promise { + return (await this.isEmpty()) && this.inProgressIds.size === 0; + } + + private async ensureHeadIsNonEmpty(): Promise { + if (this.headIds.length <= 1) { + await this.listHead(); + } + } + + private async listHead(): Promise { + // The head read returns in-progress requests too, so fetch enough to find new ones. + const limit = Math.min(MAX_HEAD_ITEMS, DESIRED_NEW_HEAD_ITEMS + this.inProgressIds.size); + const head = await this.client.listHead({ limit }); + for (const item of head.items) { + if (this.inProgressIds.has(item.id) || this.handledIds.has(item.id)) { + continue; + } + // `headIds` is nearly drained whenever this runs (see `ensureHeadIsNonEmpty`), so the + // linear dedup scan stays cheap. + if (!this.headIds.includes(item.id)) { + this.headIds.push(item.id); + } + } + } + + /** + * One-time prefetch of the existing queue contents into the local caches, so that re-added + * requests of a resurrected run are deduplicated locally (one read API call for the whole + * cache) instead of on the platform (one write operation per request). + */ + private async initCaches(): Promise { + try { + const response = await this.client.listRequests({ limit: INIT_CACHES_REQUEST_LIMIT }); + for (const request of response.items) { + if (request.handledAt) { + this.handledIds.add(request.id); + } else { + this.cacheRequest(request as unknown as UpdateRequestSchema); + } + } + } catch (err) { + // The prefetch is a cost optimization, not a correctness requirement — deduplication + // falls back to the platform. + log.warning( + `Failed to prefetch the request queue contents into the local cache: ${(err as Error).message}`, + ); + } + } + + private async isKnownOrExists(id: string): Promise { + if (this.inProgressIds.has(id) || this.handledIds.has(id) || this.cachedRequests.get(id)) { + return true; + } + return (await this.getRequestById(id)) !== undefined; + } + + private cacheRequest(request: UpdateRequestSchema): void { + // `LruCache.add` does not overwrite existing entries, so remove first. + this.cachedRequests.remove(request.id); + this.cachedRequests.add(request.id, request); + } +} diff --git a/src/apify_storage_backend.ts b/src/apify_storage_backend.ts new file mode 100644 index 0000000000..cfb48e6e47 --- /dev/null +++ b/src/apify_storage_backend.ts @@ -0,0 +1,340 @@ +/* eslint-disable max-classes-per-file */ +import { AsyncLocalStorage } from 'node:async_hooks'; +import { createHash } from 'node:crypto'; + +import type { + DatasetBackend, + KeyValueStoreBackend, + RequestQueueBackend, + StorageBackend, + StorageIdentifier, +} from '@crawlee/types'; +import type { ApifyClient } from 'apify-client'; +import { DatasetClient as ApifyDatasetClient } from 'apify-client'; +import { cryptoRandomObjectId } from '@apify/utilities'; + +import { ApifyDatasetBackend } from './apify_dataset_backend.js'; +import { ApifyKeyValueStoreBackend } from './apify_key_value_store_backend.js'; +import type { RequestQueueAccessMode } from './apify_request_queue_backend.js'; +import { ApifyRequestQueueSharedBackend } from './apify_request_queue_shared_backend.js'; +import { ApifyRequestQueueSingleBackend } from './apify_request_queue_single_backend.js'; +import { + type ChargeResult, + type ChargingManager, + DEFAULT_DATASET_ITEM_EVENT, + mergeChargeResults, + pushDataAndCharge, +} from './charging.js'; +import type { Configuration } from './configuration.js'; + +type StorageType = 'Dataset' | 'KeyValueStore' | 'RequestQueue'; + +/** The reserved alias crawlee uses for the default (unnamed) storage. */ +const DEFAULT_STORAGE_ALIAS = '__default__'; + +/** The maximum clientKey length accepted by the request queue API. */ +const MAX_CLIENT_KEY_LENGTH = 32; + +const DEFAULT_ID_CONFIG_KEY = { + Dataset: 'defaultDatasetId', + KeyValueStore: 'defaultKeyValueStoreId', + RequestQueue: 'defaultRequestQueueId', +} as const; + +const ACTOR_STORAGES_TYPE_KEY = { + Dataset: 'datasets', + KeyValueStore: 'keyValueStores', + RequestQueue: 'requestQueues', +} as const; + +/** The parsed shape of the `ACTOR_STORAGES_JSON` environment variable. */ +interface ActorStorages { + datasets?: Record; + keyValueStores?: Record; + requestQueues?: Record; +} + +/** Marks a dataset backend whose underlying client charges for pushed items (pay-per-event). @internal */ +export const USES_PUSH_DATA_INTERCEPTION = Symbol('apify:uses-push-data-interception'); + +/** + * Context of a single `Actor.pushData()` call, shared with the intercepted + * `pushItems()` calls so they can (1) know which event to charge and + * (2) aggregate the {@link ChargeResult} across the multiple `pushItems()` + * calls a single `pushData()` may trigger (Crawlee batches large pushes). + */ +export interface PpeAwarePushDataContext { + eventName: string | undefined; + chargeResult?: ChargeResult; +} + +export const pushDataChargingContext = new AsyncLocalStorage(); + +/** + * Default `DatasetClient` that charges for pushed items (pay-per-event). Used + * only for the run's default dataset when a `apify-default-dataset-item` price + * is configured; for everything else the plain `apify-client` dataset client is + * used. + */ +class PpeAwareDatasetClient< + Data extends Record = Record, +> extends ApifyDatasetClient { + constructor( + options: ConstructorParameters>[0], + private readonly getChargingManager: () => ChargingManager, + ) { + super(options); + } + + private normalizeItems(items: string | Data | string[] | Data[]): Data[] { + if (typeof items === 'string') { + const parsed = JSON.parse(items); + return Array.isArray(parsed) ? parsed : [parsed]; + } + if (Array.isArray(items)) { + return items.flatMap((item) => + typeof item === 'string' ? (JSON.parse(item) as Data | Data[]) : item, + ) as Data[]; + } + return [items]; + } + + override async pushItems(items: string | Data | string[] | Data[]): Promise { + const context = pushDataChargingContext.getStore(); + + // A single JSON string may encode multiple items (e.g. '[{...},{...}]'), + // which the charging logic would miscount — parse strings into arrays so + // each logical item is counted individually. + const normalizedItems = this.normalizeItems(items); + + const result = await pushDataAndCharge({ + chargingManager: this.getChargingManager(), + items: normalizedItems, + eventName: context?.eventName, + isDefaultDataset: true, + // stringify for faster validation in the Apify client + pushFn: async (limitedItems) => super.pushItems(JSON.stringify(limitedItems)), + }); + + if (!context) return; + + // One `Actor.pushData()` may map to several `pushItems()` calls — aggregate. + context.chargeResult = + context.chargeResult === undefined ? result : mergeChargeResults(context.chargeResult, result); + } +} + +export interface ApifyStorageBackendOptions { + /** + * SDK configuration providing the run's default storage ids and related environment values. + * Without it, opening storages requires an explicit id or name. + */ + configuration?: Configuration; + + /** + * Determines how request queues opened through this backend are consumed — + * `'single'` (default) assumes this is the queue's only consumer and skips request locking for + * fewer (paid) API calls; `'shared'` locks requests server-side so any number of concurrent + * consumers can process the same queue safely. + */ + requestQueueAccess?: RequestQueueAccessMode; + + /** + * Supplies the charging manager for pay-per-event runs, enabling the charging-aware default + * dataset client. + * @internal + */ + getChargingManager?: () => ChargingManager; +} + +/** + * Bridges `apify-client`'s synchronous resource accessors (`dataset(id)`, + * `keyValueStore(id)`, `requestQueue(id, options?)`) to crawlee v4's + * `StorageBackend` interface (async factory methods accepting an `id`, + * a `name`, or an `alias`). + * + * For the run's default dataset it transparently swaps in a charging-aware + * dataset client (pay-per-event on `Actor.pushData()`), provided a charging + * manager is supplied and a default-dataset-item price is configured. + * + * `Actor` wires this up automatically; construct it directly only to use Apify + * platform storage with crawlee's storage classes outside of `Actor` — e.g. to + * read another run's output with an explicit token: + * + * ```ts + * import { ApifyClient, ApifyStorageBackend, Dataset } from 'apify'; + * + * const client = new ApifyClient({ token }); + * const dataset = await Dataset.open(datasetId, { storageBackend: new ApifyStorageBackend(client) }); + * const { items } = await dataset.getData(); + * ``` + */ +export class ApifyStorageBackend implements StorageBackend { + private readonly config?: Configuration; + private readonly requestQueueAccess: RequestQueueAccessMode; + private readonly getChargingManager?: () => ChargingManager; + + /** Unnamed storages created for aliases in this process, so an alias maps to one storage. */ + private readonly aliasIdCache = new Map(); + + /** Fallback request queue client key when the run id is unavailable — one per backend. */ + private fallbackClientKey?: string; + + constructor( + private readonly client: ApifyClient, + options: ApifyStorageBackendOptions = {}, + ) { + this.config = options.configuration; + this.requestQueueAccess = options.requestQueueAccess ?? 'single'; + this.getChargingManager = options.getChargingManager; + } + + /** + * Partitions crawlee's storage-instance cache by API base URL and token, so the same storage + * opened through two differently-authenticated backends is cached separately. The request + * queue access mode is deliberately not part of the key — opening the same queue in `single` + * and `shared` mode at once is not supported, and whichever backend opens it first wins. + */ + getStorageBackendCacheKey(): string { + const hash = createHash('sha256') + .update(`${this.client.publicBaseUrl}${this.client.token ?? ''}`) + .digest('hex') + .slice(0, 8); + return `ApifyStorageBackend:${hash}`; + } + + async storageExists(id: string, type: StorageType): Promise { + // Lets `Dataset.open(idOrName)` and friends resolve a string to an id first (when one + // exists on the platform) and fall back to a name otherwise; without this, crawlee would + // treat every string as a name and silently create a new storage named like the passed id. + // Apify's `GET /v2/{kind}/{idOrName}` matches by either id or name; + // confirm it was an *id* match so crawlee can fall through to `{ name }`. + const info = await this.resourceClient(id, type).get(); + return info?.id === id; + } + + async createDatasetBackend(options?: StorageIdentifier): Promise { + const id = await this.resolveId(options, 'Dataset'); + const chargingClient = this.chargingDatasetClient(id); + const backend = new ApifyDatasetBackend(chargingClient ?? this.client.dataset(id)); + if (chargingClient) { + // `Actor.pushData()` looks for this marker on the dataset's backend to know the + // pay-per-event charging happens inside the intercepted `pushItems()` calls. + Object.assign(backend, { [USES_PUSH_DATA_INTERCEPTION]: true }); + } + return backend; + } + + async createKeyValueStoreBackend(options?: StorageIdentifier): Promise { + const id = await this.resolveId(options, 'KeyValueStore'); + return new ApifyKeyValueStoreBackend(this.client.keyValueStore(id)); + } + + async createRequestQueueBackend(options?: StorageIdentifier): Promise { + const id = await this.resolveId(options, 'RequestQueue'); + const client = this.client.requestQueue(id, { clientKey: this.requestQueueClientKey() }); + return this.requestQueueAccess === 'shared' + ? new ApifyRequestQueueSharedBackend(client) + : new ApifyRequestQueueSingleBackend(client); + } + + /** + * A stable per-run client key makes the API's `hadMultipleClients` flag meaningful and lets a + * migrated or resurrected run re-acquire the request locks of its previous incarnation. + */ + private requestQueueClientKey(): string { + const key = this.config?.actorRunId ?? (this.fallbackClientKey ??= cryptoRandomObjectId(MAX_CLIENT_KEY_LENGTH)); + return key.slice(0, MAX_CLIENT_KEY_LENGTH); + } + + /** + * Returns a charging-aware dataset client when `id` is the run's default + * dataset and a default-dataset-item price is configured; otherwise + * `undefined` (caller uses the plain client). + */ + private chargingDatasetClient(id: string): ApifyDatasetClient | undefined { + const { getChargingManager } = this; + if (!getChargingManager) return undefined; + if (id !== this.config?.defaultDatasetId) return undefined; + + const hasDefaultDatasetItemEvent = + DEFAULT_DATASET_ITEM_EVENT in getChargingManager().getPricingInfo().perEventPrices; + if (!hasDefaultDatasetItemEvent) return undefined; + + return new PpeAwareDatasetClient( + { + id, + baseUrl: this.client.baseUrl, + publicBaseUrl: this.client.publicBaseUrl, + apifyClient: this.client, + httpClient: this.client.httpClient, + }, + getChargingManager, + ); + } + + /** + * Resolves a crawlee {@link StorageIdentifier} to a platform storage id. + * + * Aliases resolve to unnamed storages: the reserved `__default__` alias maps to the run's + * default storage, and other aliases to the storages declared in the Actor's schema (via the + * `ACTOR_STORAGES_JSON` environment variable, maintained by the platform). Outside the + * platform, an unnamed storage is created per alias instead (remembered for this process only). + */ + private async resolveId(options: StorageIdentifier | undefined, type: StorageType): Promise { + if (options?.id) return options.id; + if (options?.name) { + return (await this.collectionClient(type).getOrCreate(options.name)).id; + } + + const alias = (options && 'alias' in options && options.alias) || DEFAULT_STORAGE_ALIAS; + + if (alias === DEFAULT_STORAGE_ALIAS) { + const defaultId = this.config?.[DEFAULT_ID_CONFIG_KEY[type]]; + if (defaultId) return defaultId; + } else { + const declaredId = this.aliasFromActorStorages(alias, type); + if (declaredId) return declaredId; + if (this.config?.isAtHome) { + throw new Error( + `Storage alias "${alias}" cannot be resolved because it is not declared in the Actor's schema storages. ` + + `Declare it in the Actor schema, or open the storage by name instead.`, + ); + } + } + + // No platform-provided id for this alias (e.g. cloud storage used locally via an API + // token) — create an unnamed storage for it, one per alias per process. + const cacheKey = `${type}:${alias}`; + const cachedId = this.aliasIdCache.get(cacheKey); + if (cachedId) return cachedId; + const created = await this.collectionClient(type).getOrCreate(); + this.aliasIdCache.set(cacheKey, created.id); + return created.id; + } + + /** Looks an alias up in the Actor's schema storages (the `ACTOR_STORAGES_JSON` env var). */ + private aliasFromActorStorages(alias: string, type: StorageType): string | undefined { + const storagesJson = this.config?.actorStoragesJson; + if (!storagesJson) return undefined; + let storages: ActorStorages; + try { + storages = JSON.parse(storagesJson); + } catch { + throw new Error(`Failed to parse ACTOR_STORAGES_JSON environment variable: ${storagesJson}`); + } + return storages[ACTOR_STORAGES_TYPE_KEY[type]]?.[alias]; + } + + private resourceClient(id: string, type: StorageType) { + if (type === 'Dataset') return this.client.dataset(id); + if (type === 'KeyValueStore') return this.client.keyValueStore(id); + return this.client.requestQueue(id); + } + + private collectionClient(type: StorageType) { + if (type === 'Dataset') return this.client.datasets(); + if (type === 'KeyValueStore') return this.client.keyValueStores(); + return this.client.requestQueues(); + } +} diff --git a/src/apify_storage_client.ts b/src/apify_storage_client.ts deleted file mode 100644 index 749fb70e0f..0000000000 --- a/src/apify_storage_client.ts +++ /dev/null @@ -1,263 +0,0 @@ -/* eslint-disable max-classes-per-file */ -import { AsyncLocalStorage } from 'node:async_hooks'; - -import type { - CreateDatasetClientOptions, - CreateKeyValueStoreClientOptions, - CreateRequestQueueClientOptions, - DatasetClient, - KeyValueStoreClient, - RequestQueueClient, - StorageClient, -} from '@crawlee/types'; -import type { ApifyClient } from 'apify-client'; -import { DatasetClient as ApifyDatasetClient } from 'apify-client'; - -import { - type ChargeResult, - type ChargingManager, - DEFAULT_DATASET_ITEM_EVENT, - mergeChargeResults, - pushDataAndCharge, -} from './charging.js'; -import type { Configuration } from './configuration.js'; - -type StorageType = 'Dataset' | 'KeyValueStore' | 'RequestQueue'; - -const DEFAULT_ID_CONFIG_KEY = { - Dataset: 'defaultDatasetId', - KeyValueStore: 'defaultKeyValueStoreId', - RequestQueue: 'defaultRequestQueueId', -} as const; - -/** Marks a dataset client whose `pushItems` charges for pay-per-event. @internal */ -export const USES_PUSH_DATA_INTERCEPTION = Symbol('apify:uses-push-data-interception'); - -/** - * Context of a single `Actor.pushData()` call, shared with the intercepted - * `pushItems()` calls so they can (1) know which event to charge and - * (2) aggregate the {@link ChargeResult} across the multiple `pushItems()` - * calls a single `pushData()` may trigger (Crawlee batches large pushes). - */ -export interface PpeAwarePushDataContext { - eventName: string | undefined; - chargeResult?: ChargeResult; -} - -export const pushDataChargingContext = new AsyncLocalStorage(); - -/** - * Default `DatasetClient` that charges for pushed items (pay-per-event). Used - * only for the run's default dataset when a `apify-default-dataset-item` price - * is configured; for everything else the plain `apify-client` dataset client is - * used. - */ -class PpeAwareDatasetClient< - Data extends Record = Record, -> extends ApifyDatasetClient { - constructor( - options: ConstructorParameters>[0], - private readonly getChargingManager: () => ChargingManager, - ) { - super(options); - } - - private normalizeItems(items: string | Data | string[] | Data[]): Data[] { - if (typeof items === 'string') { - const parsed = JSON.parse(items); - return Array.isArray(parsed) ? parsed : [parsed]; - } - if (Array.isArray(items)) { - return items.flatMap((item) => - typeof item === 'string' ? (JSON.parse(item) as Data | Data[]) : item, - ) as Data[]; - } - return [items]; - } - - override async pushItems(items: string | Data | string[] | Data[]): Promise { - const context = pushDataChargingContext.getStore(); - - // A single JSON string may encode multiple items (e.g. '[{...},{...}]'), - // which the charging logic would miscount — parse strings into arrays so - // each logical item is counted individually. - const normalizedItems = this.normalizeItems(items); - - const result = await pushDataAndCharge({ - chargingManager: this.getChargingManager(), - items: normalizedItems, - eventName: context?.eventName, - isDefaultDataset: true, - // stringify for faster validation in the Apify client - pushFn: async (limitedItems) => super.pushItems(JSON.stringify(limitedItems)), - }); - - if (!context) return; - - // One `Actor.pushData()` may map to several `pushItems()` calls — aggregate. - context.chargeResult = - context.chargeResult === undefined ? result : mergeChargeResults(context.chargeResult, result); - } -} - -// crawlee v4's `StorageClient` sub-client interfaces use different method names -// than `apify-client`'s resource clients (`getValue`/`getRecord`, -// `pushData`/`pushItems`, `getData`/`listItems`, `getMetadata`/`get`, -// `drop`/`delete`). `adapt` wraps a client in a name-remapping proxy: `renames` -// aliases the differing methods and `overrides` replaces the few whose return -// shape differs; everything else — identically-named methods and the -// pay-per-event marker symbol — passes straight through. -// -// `purge()` has no apify-client equivalent and isn't needed on the platform -// (a run's storages are already fresh), so it's a no-op. -const noPurge = { purge: async () => {} }; - -function adapt( - client: T, - renames: Record, - overrides: Record unknown> = {}, -): T { - return new Proxy(client, { - get(target, prop) { - if (typeof prop === 'string' && prop in overrides) return overrides[prop]; - const value = Reflect.get(target, (typeof prop === 'string' && renames[prop]) || prop, target); - return typeof value === 'function' ? (value as (...args: unknown[]) => unknown).bind(target) : value; - }, - }); -} - -/** - * Bridges `apify-client`'s synchronous resource accessors (`dataset(id)`, - * `keyValueStore(id)`, `requestQueue(id, options?)`) to crawlee v4's - * `StorageClient` interface (async factory methods accepting either an `id` - * or a `name`). - * - * For the run's default dataset it transparently swaps in a charging-aware - * dataset client (pay-per-event on `Actor.pushData()`), provided a charging - * manager is supplied and a default-dataset-item price is configured. - * - * `storageExists()` lets `Dataset.open(idOrName)` resolve a string to an id - * first (when one exists on the platform) and fall back to a name otherwise — - * otherwise crawlee's `resolveStorageIdentifier` treats every string as a name - * and the SDK would silently create a new storage named like the passed id. - * - * `Actor` wires this up automatically; construct it directly only to use Apify - * platform storage with crawlee's storage classes outside of `Actor` — e.g. to - * read another run's output with an explicit token: - * - * ```ts - * import { ApifyClient, ApifyStorageClient, Dataset } from 'apify'; - * - * const client = new ApifyClient({ token }); - * const dataset = await Dataset.open(datasetId, { storageClient: new ApifyStorageClient(client) }); - * const { items } = await dataset.getData(); - * ``` - */ -export class ApifyStorageClient implements StorageClient { - constructor( - private readonly client: ApifyClient, - private readonly config?: Configuration, - private readonly getChargingManager?: () => ChargingManager, - ) {} - - async storageExists(id: string, type: StorageType): Promise { - // Apify's `GET /v2/{kind}/{idOrName}` matches by either id or name; - // confirm it was an *id* match so crawlee can fall through to `{ name }`. - const info = await this.resourceClient(id, type).get(); - return info?.id === id; - } - - async createDatasetClient(options?: CreateDatasetClientOptions): Promise { - const id = await this.resolveId(options, 'Dataset'); - const client = this.chargingDatasetClient(id) ?? this.client.dataset(id); - return adapt( - client, - { - getMetadata: 'get', - drop: 'delete', - pushData: 'pushItems', - getData: 'listItems', - }, - noPurge, - ) as unknown as DatasetClient; - } - - async createKeyValueStoreClient(options?: CreateKeyValueStoreClientOptions): Promise { - const id = await this.resolveId(options, 'KeyValueStore'); - const client = this.client.keyValueStore(id); - return adapt( - client, - { - getMetadata: 'get', - getValue: 'getRecord', - setValue: 'setRecord', - deleteValue: 'deleteRecord', - drop: 'delete', - getPublicUrl: 'getRecordPublicUrl', - }, - { - ...noPurge, - // crawlee expects an array; apify-client returns `{ items }`. - listKeys: async (opts?: Parameters[0]) => (await client.listKeys(opts)).items, - }, - ) as unknown as KeyValueStoreClient; - } - - async createRequestQueueClient(options?: CreateRequestQueueClientOptions): Promise { - const id = await this.resolveId(options, 'RequestQueue'); - const client = this.client.requestQueue(id, options?.clientKey ? { clientKey: options.clientKey } : undefined); - return adapt(client, { getMetadata: 'get', drop: 'delete' }, noPurge) as unknown as RequestQueueClient; - } - - /** - * Returns a charging-aware dataset client when `id` is the run's default - * dataset and a default-dataset-item price is configured; otherwise - * `undefined` (caller uses the plain client). - */ - private chargingDatasetClient(id: string): ApifyDatasetClient | undefined { - const { getChargingManager } = this; - if (!getChargingManager) return undefined; - if (id !== this.config?.defaultDatasetId) return undefined; - - const hasDefaultDatasetItemEvent = - DEFAULT_DATASET_ITEM_EVENT in getChargingManager().getPricingInfo().perEventPrices; - if (!hasDefaultDatasetItemEvent) return undefined; - - const datasetClient = new PpeAwareDatasetClient( - { - id, - baseUrl: this.client.baseUrl, - publicBaseUrl: this.client.publicBaseUrl, - apifyClient: this.client, - httpClient: this.client.httpClient, - }, - getChargingManager, - ); - Object.assign(datasetClient as object, { - [USES_PUSH_DATA_INTERCEPTION]: true, - }); - return datasetClient; - } - - private async resolveId(options: { id?: string; name?: string } | undefined, type: StorageType): Promise { - if (options?.id) return options.id; - if (options?.name) { - return (await this.collectionClient(type).getOrCreate(options.name)).id; - } - // No id/name (crawlee's `__default__` alias): use the default storage - // id from the run's environment. apify-client rejects an empty id. - return this.config?.[DEFAULT_ID_CONFIG_KEY[type]] ?? ''; - } - - private resourceClient(id: string, type: StorageType) { - if (type === 'Dataset') return this.client.dataset(id); - if (type === 'KeyValueStore') return this.client.keyValueStore(id); - return this.client.requestQueue(id); - } - - private collectionClient(type: StorageType) { - if (type === 'Dataset') return this.client.datasets(); - if (type === 'KeyValueStore') return this.client.keyValueStores(); - return this.client.requestQueues(); - } -} diff --git a/src/configuration.ts b/src/configuration.ts index e0e51a0dd3..b90589379e 100644 --- a/src/configuration.ts +++ b/src/configuration.ts @@ -168,13 +168,13 @@ export interface Configuration extends ApifyResolvedConfigValues {} /** * `Configuration` is a value object holding the SDK configuration. We can use it in two ways: * - * 1. When using `Actor` class, we can get the instance configuration via `sdk.config` + * 1. When using `Actor` class, we can get the instance configuration via `sdk.configuration` * * ```javascript * import { Actor } from 'apify'; * * const sdk = new Actor({ token: '123' }); - * console.log(sdk.config.token); // '123' + * console.log(sdk.configuration.token); // '123' * ``` * * 2. To get the global configuration (singleton instance). It will respect the environment variables. @@ -182,7 +182,7 @@ export interface Configuration extends ApifyResolvedConfigValues {} * ```javascript * import { Configuration } from 'apify'; * - * const config = Configuration.getGlobalConfig(); + * const config = Configuration.getGlobalConfiguration(); * console.log(config.headless); * console.log(config.persistStateIntervalMillis); * ``` @@ -255,7 +255,7 @@ export class Configuration extends CoreConfiguration { * what crawlee internals resolve against; this singleton is only the * fallback for code reaching for a configuration without an explicit one. */ - static override getGlobalConfig(): Configuration { + static override getGlobalConfiguration(): Configuration { Configuration.globalConfig ??= new Configuration(); return Configuration.globalConfig; } diff --git a/src/index.ts b/src/index.ts index 23b60f5c79..552f94417b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,5 +1,6 @@ export * from './actor.js'; -export { ApifyStorageClient } from './apify_storage_client.js'; +export { ApifyStorageBackend, type ApifyStorageBackendOptions } from './apify_storage_backend.js'; +export type { RequestQueueAccessMode } from './apify_request_queue_backend.js'; export { ArgumentValidationError } from './utils.js'; export type { OpenStorageOptions, @@ -24,7 +25,6 @@ export { DatasetOptions, DatasetContent, RequestQueue, - QueueOperationInfo, RequestQueueOperationOptions, RequestQueueOptions, KeyConsumer, @@ -39,4 +39,5 @@ export { LoggerJson, LoggerText, } from '@crawlee/core'; +export type { QueueOperationInfo } from '@crawlee/types'; export { ApifyClient, ApifyClientOptions } from 'apify-client'; diff --git a/src/input-schemas.ts b/src/input-schemas.ts index 251f02a2f9..5b7578f465 100644 --- a/src/input-schemas.ts +++ b/src/input-schemas.ts @@ -4,7 +4,7 @@ import { existsSync, readFileSync } from 'node:fs'; import { join } from 'node:path'; import process from 'node:process'; -import type { Dictionary } from '@crawlee/utils'; +import type { Dictionary } from '@crawlee/types'; // These paths are used *if* there is no `input` field in the actor.json configuration file! const DEFAULT_INPUT_SCHEMA_PATHS = [ diff --git a/src/key_value_store.ts b/src/key_value_store.ts index eadfd69c7f..a7a7969e45 100644 --- a/src/key_value_store.ts +++ b/src/key_value_store.ts @@ -1,10 +1,10 @@ import type { StorageOpenOptions } from '@crawlee/core'; -import { KeyValueStore as CoreKeyValueStore } from '@crawlee/core'; +import { KeyValueStore as CoreKeyValueStore, serviceLocator } from '@crawlee/core'; import type { KeyValueStoreInfo } from '@crawlee/types'; -import { KeyValueStoreClient as RemoteKeyValueStoreClient } from 'apify-client'; import { createHmacSignature } from '@apify/utilities'; +import { ApifyKeyValueStoreBackend } from './apify_key_value_store_backend.js'; import type { Configuration } from './configuration.js'; // crawlee v4 dropped the `storageObject` cache from `KeyValueStore`, so the @@ -29,24 +29,21 @@ export class KeyValueStore extends CoreKeyValueStore { * implementation (which produces a `file://` URL or returns `undefined`). */ override async getPublicUrl(key: string): Promise { - const config = this.config as Configuration; + const config = serviceLocator.getConfiguration() as Configuration; - // Detect a remote (Apify) store by its client type rather than by + // Detect a remote (Apify) store by its backend type rather than by // `isAtHome`, so that a `forceCloud` store opened locally still gets a - // signed Apify URL (matching the platform behaviour). `client` is + // signed Apify URL (matching the platform behaviour). `backend` is // `private` on `CoreKeyValueStore`, so bypass the visibility check. - const { client } = this as unknown as { client: unknown }; - const isLocalStore = !(client instanceof RemoteKeyValueStoreClient); + const { backend } = this as unknown as { backend: unknown }; - if (isLocalStore) { + if (!(backend instanceof ApifyKeyValueStoreBackend)) { return super.getPublicUrl(key); } const publicUrl = new URL(`${config.apiPublicBaseUrl}/v2/key-value-stores/${this.id}/records/${key}`); - const metadata = (await ( - client as unknown as { getMetadata(): Promise } - ).getMetadata()) as ApifyKeyValueStoreInfo; + const metadata = (await backend.getMetadata()) as ApifyKeyValueStoreInfo; if (metadata?.urlSigningSecretKey) { publicUrl.searchParams.append('signature', createHmacSignature(metadata.urlSigningSecretKey, key)); diff --git a/src/platform_event_manager.ts b/src/platform_event_manager.ts index 0a5f078838..0945bf298d 100644 --- a/src/platform_event_manager.ts +++ b/src/platform_event_manager.ts @@ -48,9 +48,9 @@ export class PlatformEventManager extends EventManager { /** Websocket connection to Actor events. */ private eventsWs?: WebSocket; - constructor(readonly config = Configuration.getGlobalConfig()) { + constructor(readonly configuration = Configuration.getGlobalConfiguration()) { super({ - persistStateIntervalMillis: config.persistStateIntervalMillis, + persistStateIntervalMillis: configuration.persistStateIntervalMillis, }); } @@ -64,7 +64,7 @@ export class PlatformEventManager extends EventManager { } await super.init(); - const eventsWsUrl = this.config.actorEventsWsUrl; + const eventsWsUrl = this.configuration.actorEventsWsUrl; // Locally there is no web socket to connect, so just print a log message. if (!eventsWsUrl) { diff --git a/src/proxy_configuration.ts b/src/proxy_configuration.ts index 66715c552e..3286886667 100644 --- a/src/proxy_configuration.ts +++ b/src/proxy_configuration.ts @@ -8,6 +8,7 @@ import type { ProxyInfo as CoreProxyInfo } from '@crawlee/types'; import { z } from 'zod'; import { APIFY_ENV_VARS, APIFY_PROXY_VALUE_REGEX } from '@apify/consts'; +import defaultLog from '@apify/log'; import { cryptoRandomObjectId } from '@apify/utilities'; import { Actor } from './actor.js'; @@ -196,12 +197,14 @@ export class ProxyConfiguration extends CoreProxyConfiguration { private port?: number; private usesApifyProxy?: boolean; + protected readonly log = defaultLog.child({ prefix: 'ProxyConfiguration' }); + /** * @internal */ constructor( options: ProxyConfigurationOptions = {}, - readonly config = Configuration.getGlobalConfig(), + readonly configuration = Configuration.getGlobalConfiguration(), ) { const { proxyUrls, newUrlFunction, ...rest } = options; super({ @@ -231,14 +234,14 @@ export class ProxyConfiguration extends CoreProxyConfiguration { apifyProxyCountry, subdivisionCode, apifyProxySubdivision, - password = config.proxyPassword, + password = configuration.proxyPassword, } = options; const groupsToUse = groups.length ? groups : apifyProxyGroups; const countryCodeToUse = countryCode || apifyProxyCountry; const subdivisionCodeToUse = subdivisionCode || apifyProxySubdivision; - const hostname = config.proxyHostname; - const port = config.proxyPort; + const hostname = configuration.proxyHostname; + const port = configuration.proxyPort; // The Apify Proxy subdivision is expressed as part of the country // username parameter (`country-US_CA`), so a country is required. @@ -258,7 +261,7 @@ export class ProxyConfiguration extends CoreProxyConfiguration { this.password = password; this.hostname = hostname!; this.port = port; - this.usesApifyProxy = !this.proxyUrls && !this.newUrlFunction; + this.usesApifyProxy = !proxyUrls && !newUrlFunction; if (proxyUrls && proxyUrls.some((url) => url?.includes('apify.com'))) { this.log.warning( @@ -338,7 +341,7 @@ export class ProxyConfiguration extends CoreProxyConfiguration { * `proxyUrls`, the URLs are rotated round-robin. */ override async newUrl(options?: NewUrlOptions): Promise { - if (this.newUrlFunction || this.proxyUrls) { + if (!this.usesApifyProxy) { return super.newUrl(options); } return this.composeDefaultUrl(cryptoRandomObjectId(SESSION_ID_LENGTH)); @@ -379,7 +382,7 @@ export class ProxyConfiguration extends CoreProxyConfiguration { */ // TODO: Make this private protected async _setPasswordIfToken(): Promise { - const { token } = this.config; + const { token } = this.configuration; if (!token) return; try { @@ -413,7 +416,9 @@ export class ProxyConfiguration extends CoreProxyConfiguration { } const { connected, connectionError, isManInTheMiddle } = status; - this.isManInTheMiddle = isManInTheMiddle; + // Declared `readonly false` on the base class; the status check is the one place that + // learns the actual value, so bypass the readonly marker. + (this as { isManInTheMiddle: boolean }).isManInTheMiddle = isManInTheMiddle; if (connected) { return true; @@ -434,7 +439,7 @@ export class ProxyConfiguration extends CoreProxyConfiguration { * Apify Proxy can be down for a second or a minute, but this should not crash processes. */ protected async _fetchStatus(): Promise { - const { proxyStatusUrl } = this.config; + const { proxyStatusUrl } = this.configuration; const statusUrl = `${proxyStatusUrl}/?format=json`; const proxyUrl = await this.newUrl(); @@ -505,4 +510,14 @@ export class ProxyConfiguration extends CoreProxyConfiguration { '"options.subdivisionCode" or "options.apifyProxySubdivision".', ); } + + /** + * Throws cannot combine custom proxies with custom generating function + * @internal + */ + protected _throwCannotCombineCustomMethods() { + throw new Error( + 'Cannot combine custom proxies "options.proxyUrls" with custom generating function "options.newUrlFunction".', + ); + } } diff --git a/src/storage.ts b/src/storage.ts index 1858b618b5..fdab4f071d 100644 --- a/src/storage.ts +++ b/src/storage.ts @@ -1,7 +1,7 @@ -import type { Constructor, IStorage, StorageOpenOptions } from '@crawlee/core'; -import type { StorageClient } from '@crawlee/types'; +import type { IStorage, StorageOpenOptions } from '@crawlee/core'; +import type { Constructor, StorageBackend } from '@crawlee/types'; -import { ApifyStorageClient } from './apify_storage_client.js'; +import { ApifyStorageBackend } from './apify_storage_backend.js'; import type { Configuration } from './configuration.js'; export interface OpenStorageOptions { @@ -137,7 +137,7 @@ function resolveStorageIdentifier( export interface OpenStorageContext { config: Configuration; - client?: StorageClient; + backend?: StorageBackend; purgedStorageAliases: Set; } @@ -154,7 +154,7 @@ export async function openStorage( const isAlias = identifier !== null && identifier !== undefined && typeof identifier === 'object' && 'alias' in identifier; - if (isAlias && !context.config.isAtHome && context.client instanceof ApifyStorageClient) { + if (isAlias && !context.config.isAtHome && context.backend instanceof ApifyStorageBackend) { throw new Error('The `alias` option is not allowed for Apify-based storages running outside of Apify'); } @@ -174,12 +174,12 @@ export async function openStorage( ) { context.purgedStorageAliases.add(identifier.alias); const existingStorage = await storageClass.open(resolvedIdOrName ?? null, { - storageClient: context.client, + storageBackend: context.backend, }); await (existingStorage as T & { drop(): Promise }).drop(); } return storageClass.open(resolvedIdOrName ?? null, { - storageClient: context.client, + storageBackend: context.backend, }); } diff --git a/src/utils.ts b/src/utils.ts index c7759dba47..c17c1412f6 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -28,6 +28,18 @@ export function isNonEmptyObject(value: unknown): value is Record 0; } +/** + * Converts a `SNAKE_CASE` string to `camelCase` (previously provided by `@crawlee/utils`). + * @internal + */ +export function snakeCaseToCamelCase(snakeCaseStr: string): string { + return snakeCaseStr + .toLowerCase() + .split('_') + .map((part, index) => (index > 0 ? part.charAt(0).toUpperCase() + part.slice(1) : part)) + .join(''); +} + /** Formats a zod issue path like `groups[0]` or `countryCode`. */ function formatIssuePath(path: readonly PropertyKey[]): string { let out = ''; diff --git a/test/apify/actor.test.ts b/test/apify/actor.test.ts index 45691440a9..4e18ad6232 100644 --- a/test/apify/actor.test.ts +++ b/test/apify/actor.test.ts @@ -1,7 +1,6 @@ import { createPublicKey } from 'node:crypto'; -import { EventType, serviceLocator } from '@crawlee/core'; -import { MemoryStorage } from '@crawlee/memory-storage'; +import { EventType, MemoryStorageBackend, serviceLocator } from '@crawlee/core'; import { sleep } from '@crawlee/utils'; import type { ApifyEnv } from 'apify'; import { Actor, Configuration, Dataset, KeyValueStore, ProxyConfiguration, RequestQueue } from 'apify'; @@ -614,7 +613,7 @@ describe('Actor', () => { beforeEach(() => { sdk = createIsolatedActor({ - storageClient: new MemoryStorage({ persistStorage: false }), + storageClient: new MemoryStorageBackend(), }).actor as Actor<{ foo: string }>; }); @@ -674,20 +673,14 @@ describe('Actor', () => { const options = { forceCloud: true }; // crawlee v4 opens storages via `.open(id, { storageClient })`; // `forceCloud` is expressed by passing an explicit (Apify) client. - const mockRQ = { - client: { - getMetadata: async () => ({ totalRequestCount: 10 }), - }, - } as unknown as RequestQueue; + const mockRQ = {} as unknown as RequestQueue; const openSpy = vitest.spyOn(RequestQueue, 'open').mockResolvedValueOnce(mockRQ); const queue = await sdk.openRequestQueue(queueId, options); expect(openSpy).toBeCalledTimes(1); expect(openSpy.mock.calls[0][0]).toBe(queueId); - expect(openSpy.mock.calls[0][1]?.storageClient).toBeDefined(); - - // @ts-expect-error private prop - expect(queue.initialCount).toBe(10); + expect(openSpy.mock.calls[0][1]?.storageBackend).toBeDefined(); + expect(queue).toBe(mockRQ); }); test('openDataset should open storage', async () => { @@ -697,7 +690,7 @@ describe('Actor', () => { await sdk.openDataset(datasetName, options); expect(openSpy).toBeCalledTimes(1); expect(openSpy.mock.calls[0][0]).toBe(datasetName); - expect(openSpy.mock.calls[0][1]?.storageClient).toBeDefined(); + expect(openSpy.mock.calls[0][1]?.storageBackend).toBeDefined(); }); describe('StorageIdentifier support', () => { @@ -1408,12 +1401,12 @@ describe('Actor', () => { }); }); - describe('Actor.config and PPE', () => { + describe('Actor.configuration and PPE', () => { test('should work', async () => { delete process.env.ACTOR_MAX_TOTAL_CHARGE_USD; await Actor.init(); // No explicit limit (`0`/empty/unset) is treated as unlimited. - expect(Actor.config.maxTotalChargeUsd).toBe(Infinity); + expect(Actor.configuration.maxTotalChargeUsd).toBe(Infinity); expect(Actor.getChargingManager().getMaxTotalChargeUsd()).toBe(Infinity); await Actor.exit({ exit: false }); diff --git a/test/apify/apify_storage_backend.test.ts b/test/apify/apify_storage_backend.test.ts new file mode 100644 index 0000000000..a1863078cd --- /dev/null +++ b/test/apify/apify_storage_backend.test.ts @@ -0,0 +1,242 @@ +import type { ApifyClient, DatasetClient, KeyValueStoreClient } from 'apify-client'; +import { describe, expect, test, vi } from 'vitest'; + +import { ApifyDatasetBackend } from '../../src/apify_dataset_backend.js'; +import { ApifyKeyValueStoreBackend } from '../../src/apify_key_value_store_backend.js'; +import { ApifyRequestQueueSharedBackend } from '../../src/apify_request_queue_shared_backend.js'; +import { ApifyRequestQueueSingleBackend } from '../../src/apify_request_queue_single_backend.js'; +import { ApifyStorageBackend, USES_PUSH_DATA_INTERCEPTION } from '../../src/apify_storage_backend.js'; +import { DEFAULT_DATASET_ITEM_EVENT } from '../../src/charging.js'; +import { Configuration } from '../../src/configuration.js'; + +function createMockApifyClient() { + let unnamedCounter = 0; + const getOrCreate = vi.fn(async (name?: string) => ({ + id: name ? `id-of-${name}` : `unnamed-${++unnamedCounter}`, + })); + return { + baseUrl: 'https://api.apify.com/v2', + publicBaseUrl: 'https://api.apify.com', + token: 'test-token', + httpClient: {}, + dataset: vi.fn(() => ({ get: vi.fn(async () => undefined) })), + keyValueStore: vi.fn(() => ({ get: vi.fn(async () => undefined) })), + requestQueue: vi.fn(() => ({ get: vi.fn(async () => undefined) })), + datasets: vi.fn(() => ({ getOrCreate })), + keyValueStores: vi.fn(() => ({ getOrCreate })), + requestQueues: vi.fn(() => ({ getOrCreate })), + getOrCreate, + }; +} + +type MockApifyClient = ReturnType; + +function asApifyClient(mock: MockApifyClient): ApifyClient { + return mock as unknown as ApifyClient; +} + +describe('ApifyStorageBackend', () => { + test('picks the request queue backend implementation by access mode', async () => { + const client = createMockApifyClient(); + const configuration = new Configuration({ defaultRequestQueueId: 'default-rq' }); + const single = new ApifyStorageBackend(asApifyClient(client), { configuration }); + const shared = new ApifyStorageBackend(asApifyClient(client), { configuration, requestQueueAccess: 'shared' }); + + await expect(single.createRequestQueueBackend()).resolves.toBeInstanceOf(ApifyRequestQueueSingleBackend); + await expect(shared.createRequestQueueBackend()).resolves.toBeInstanceOf(ApifyRequestQueueSharedBackend); + }); + + test('passes a run-scoped clientKey to the request queue client', async () => { + const client = createMockApifyClient(); + const backend = new ApifyStorageBackend(asApifyClient(client), { + configuration: new Configuration({ defaultRequestQueueId: 'default-rq', actorRunId: 'test-run-id' }), + }); + + await backend.createRequestQueueBackend(); + + expect(client.requestQueue).toHaveBeenCalledWith('default-rq', { clientKey: 'test-run-id' }); + }); + + test('resolves the reserved __default__ alias to the default storage id', async () => { + const client = createMockApifyClient(); + const backend = new ApifyStorageBackend(asApifyClient(client), { + configuration: new Configuration({ defaultDatasetId: 'default-dataset' }), + }); + + await backend.createDatasetBackend({ alias: '__default__' }); + + expect(client.dataset).toHaveBeenCalledWith('default-dataset'); + }); + + test('resolves aliases declared in the Actor schema storages', async () => { + const client = createMockApifyClient(); + const backend = new ApifyStorageBackend(asApifyClient(client), { + configuration: new Configuration({ + isAtHome: true, + actorStoragesJson: JSON.stringify({ datasets: { results: 'declared-dataset-id' } }), + }), + }); + + await backend.createDatasetBackend({ alias: 'results' }); + + expect(client.dataset).toHaveBeenCalledWith('declared-dataset-id'); + }); + + test('rejects undeclared aliases on the platform', async () => { + const client = createMockApifyClient(); + const backend = new ApifyStorageBackend(asApifyClient(client), { + configuration: new Configuration({ isAtHome: true }), + }); + + await expect(backend.createDatasetBackend({ alias: 'unknown' })).rejects.toThrow( + /alias "unknown" cannot be resolved/, + ); + }); + + test('creates one unnamed storage per alias outside the platform', async () => { + const client = createMockApifyClient(); + const backend = new ApifyStorageBackend(asApifyClient(client), { + configuration: new Configuration({ isAtHome: false }), + }); + + await backend.createDatasetBackend({ alias: 'scratch' }); + await backend.createDatasetBackend({ alias: 'scratch' }); + + expect(client.getOrCreate).toHaveBeenCalledTimes(1); + expect(client.dataset).toHaveBeenNthCalledWith(1, 'unnamed-1'); + expect(client.dataset).toHaveBeenNthCalledWith(2, 'unnamed-1'); + }); + + test('opens named storages via getOrCreate', async () => { + const client = createMockApifyClient(); + const backend = new ApifyStorageBackend(asApifyClient(client)); + + await backend.createKeyValueStoreBackend({ name: 'my-store' }); + + expect(client.getOrCreate).toHaveBeenCalledWith('my-store'); + expect(client.keyValueStore).toHaveBeenCalledWith('id-of-my-store'); + }); + + test('partitions the storage cache by API credentials, not by access mode', () => { + const client = createMockApifyClient(); + const single = new ApifyStorageBackend(asApifyClient(client)); + const shared = new ApifyStorageBackend(asApifyClient(client), { requestQueueAccess: 'shared' }); + const otherToken = new ApifyStorageBackend(asApifyClient({ ...client, token: 'other-token' })); + + expect(single.getStorageBackendCacheKey()).toBe(shared.getStorageBackendCacheKey()); + expect(single.getStorageBackendCacheKey()).not.toBe(otherToken.getStorageBackendCacheKey()); + }); + + test('marks the default dataset backend for push-data interception on pay-per-event runs', async () => { + const client = createMockApifyClient(); + const backend = new ApifyStorageBackend(asApifyClient(client), { + configuration: new Configuration({ defaultDatasetId: 'default-dataset' }), + getChargingManager: () => + ({ + getPricingInfo: () => ({ perEventPrices: { [DEFAULT_DATASET_ITEM_EVENT]: {} } }), + }) as never, + }); + + const defaultDataset = await backend.createDatasetBackend({ id: 'default-dataset' }); + const otherDataset = await backend.createDatasetBackend({ id: 'other-dataset' }); + + expect((defaultDataset as never)[USES_PUSH_DATA_INTERCEPTION]).toBe(true); + expect((otherDataset as never)[USES_PUSH_DATA_INTERCEPTION]).toBeUndefined(); + }); +}); + +describe('ApifyDatasetBackend', () => { + function createMockDatasetClient() { + return { + get: vi.fn(async () => ({ id: 'dataset-id', itemCount: 0 })), + delete: vi.fn(async () => {}), + pushItems: vi.fn(async () => {}), + listItems: vi.fn(async () => ({ items: [{ foo: 'bar' }], total: 1, count: 1, offset: 0, limit: 10 })), + }; + } + + test('maps the backend interface onto the apify-client dataset client', async () => { + const client = createMockDatasetClient(); + const backend = new ApifyDatasetBackend(client as unknown as DatasetClient); + + await expect(backend.getMetadata()).resolves.toEqual({ id: 'dataset-id', itemCount: 0 }); + + await backend.pushData([{ foo: 'bar' }]); + expect(client.pushItems).toHaveBeenCalledWith([{ foo: 'bar' }]); + + await expect(backend.getData({ limit: 10 })).resolves.toEqual( + expect.objectContaining({ items: [{ foo: 'bar' }], total: 1 }), + ); + expect(client.listItems).toHaveBeenCalledWith({ limit: 10 }); + + await backend.drop(); + expect(client.delete).toHaveBeenCalled(); + }); + + test('getMetadata throws when the dataset no longer exists, and purge is unsupported', async () => { + const client = createMockDatasetClient(); + client.get.mockResolvedValue(undefined as never); + const backend = new ApifyDatasetBackend(client as unknown as DatasetClient); + + await expect(backend.getMetadata()).rejects.toThrow(/not found/); + await expect(backend.purge()).rejects.toThrow(/not supported on the Apify platform/); + }); +}); + +describe('ApifyKeyValueStoreBackend', () => { + function createMockKvsClient() { + return { + get: vi.fn(async () => ({ id: 'store-id' })), + delete: vi.fn(async () => {}), + getRecord: vi.fn(async () => ({ key: 'INPUT', value: Buffer.from('{}'), contentType: 'application/json' })), + setRecord: vi.fn(async () => {}), + deleteRecord: vi.fn(async () => {}), + listKeys: vi.fn(async () => ({ + items: [{ key: 'INPUT', size: 2, recordPublicUrl: 'https://example.com' }], + count: 1, + limit: 1000, + exclusiveStartKey: undefined, + isTruncated: false, + nextExclusiveStartKey: undefined, + })), + getRecordPublicUrl: vi.fn(async () => 'https://example.com/INPUT'), + recordExists: vi.fn(async () => true), + }; + } + + test('reads records as raw bytes so the frontend can do the parsing', async () => { + const client = createMockKvsClient(); + const backend = new ApifyKeyValueStoreBackend(client as unknown as KeyValueStoreClient); + + const record = await backend.getValue('INPUT'); + + expect(client.getRecord).toHaveBeenCalledWith('INPUT', { buffer: true }); + expect(record?.value).toBeInstanceOf(Buffer); + }); + + test('maps the backend interface onto the apify-client store client', async () => { + const client = createMockKvsClient(); + const backend = new ApifyKeyValueStoreBackend(client as unknown as KeyValueStoreClient); + + await backend.setValue({ key: 'OUTPUT', value: '{}', contentType: 'application/json' }); + expect(client.setRecord).toHaveBeenCalledWith({ key: 'OUTPUT', value: '{}', contentType: 'application/json' }); + + await backend.deleteValue('OUTPUT'); + expect(client.deleteRecord).toHaveBeenCalledWith('OUTPUT'); + + await expect(backend.listKeys({ limit: 1000 })).resolves.toEqual( + expect.objectContaining({ items: [{ key: 'INPUT', size: 2 }], isTruncated: false }), + ); + await expect(backend.recordExists('INPUT')).resolves.toBe(true); + await expect(backend.getPublicUrl('INPUT')).resolves.toBe('https://example.com/INPUT'); + }); + + test('getMetadata throws when the store no longer exists, and purge is unsupported', async () => { + const client = createMockKvsClient(); + client.get.mockResolvedValue(undefined as never); + const backend = new ApifyKeyValueStoreBackend(client as unknown as KeyValueStoreClient); + + await expect(backend.getMetadata()).rejects.toThrow(/not found/); + await expect(backend.purge()).rejects.toThrow(/not supported on the Apify platform/); + }); +}); diff --git a/test/apify/events.test.ts b/test/apify/events.test.ts index d0ab9cb51a..8944fdad2a 100644 --- a/test/apify/events.test.ts +++ b/test/apify/events.test.ts @@ -22,7 +22,7 @@ describe('events', () => { // crawlee v4's serviceLocator throws when a service is re-set, so start // each test from a clean locator and re-resolve the global config. serviceLocator.reset(); - config = Configuration.getGlobalConfig(); + config = Configuration.getGlobalConfiguration(); wss = new WebSocketServer({ port: 9099 }); events = new PlatformEventManager(config); serviceLocator.setEventManager(events); diff --git a/test/apify/request_queue_backend.test.ts b/test/apify/request_queue_backend.test.ts new file mode 100644 index 0000000000..ac27500792 --- /dev/null +++ b/test/apify/request_queue_backend.test.ts @@ -0,0 +1,339 @@ +import type { RequestSchema } from '@crawlee/types'; +import type { RequestQueueClient } from 'apify-client'; +import { describe, expect, test, vi } from 'vitest'; + +import { uniqueKeyToRequestId } from '../../src/apify_request_queue_backend.js'; +import { ApifyRequestQueueSharedBackend } from '../../src/apify_request_queue_shared_backend.js'; +import { ApifyRequestQueueSingleBackend } from '../../src/apify_request_queue_single_backend.js'; + +function request(uniqueKey: string, extra: Partial = {}): RequestSchema { + return { url: `https://example.com/${uniqueKey}`, uniqueKey, ...extra }; +} + +function id(uniqueKey: string): string { + return uniqueKeyToRequestId(uniqueKey); +} + +/** + * A mocked `apify-client` request queue client. Every method is a `vi.fn()`; the defaults emulate + * an empty queue and a platform that accepts everything. + */ +function createMockApiClient() { + return { + get: vi.fn(async () => ({ + id: 'queue-id', + name: undefined, + createdAt: new Date('2025-01-01'), + modifiedAt: new Date('2025-01-01'), + accessedAt: new Date('2025-01-01'), + totalRequestCount: 0, + handledRequestCount: 0, + pendingRequestCount: 0, + })), + delete: vi.fn(async () => {}), + listHead: vi.fn(async () => ({ + items: [], + limit: 100, + queueModifiedAt: new Date(), + hadMultipleClients: false, + })), + listAndLockHead: vi.fn(async () => ({ + items: [], + limit: 25, + lockSecs: 180, + queueModifiedAt: new Date(), + hadMultipleClients: false, + queueHasLockedRequests: false, + clientKey: 'client-key', + })), + listRequests: vi.fn(async () => ({ items: [], limit: 10_000 })), + batchAddRequests: vi.fn(async (requests: { uniqueKey: string }[]) => ({ + processedRequests: requests.map((req) => ({ + requestId: id(req.uniqueKey), + uniqueKey: req.uniqueKey, + wasAlreadyPresent: false, + wasAlreadyHandled: false, + })), + unprocessedRequests: [], + })), + updateRequest: vi.fn(async (req: { id: string }) => ({ + requestId: req.id, + wasAlreadyPresent: true, + wasAlreadyHandled: false, + })), + getRequest: vi.fn(async () => undefined), + deleteRequestLock: vi.fn(async () => {}), + }; +} + +type MockApiClient = ReturnType; + +function asApiClient(mock: MockApiClient): RequestQueueClient { + return mock as unknown as RequestQueueClient; +} + +describe('ApifyRequestQueueSingleBackend', () => { + test('adds requests, strips ids, and never locks', async () => { + const api = createMockApiClient(); + const backend = new ApifyRequestQueueSingleBackend(asApiClient(api)); + + const result = await backend.addBatchOfRequests([request('a', { id: 'bogus' } as RequestSchema), request('b')]); + + expect(result.processedRequests).toHaveLength(2); + expect(api.batchAddRequests).toHaveBeenCalledTimes(1); + expect(api.batchAddRequests.mock.calls[0][0]).toEqual([ + { url: 'https://example.com/a', uniqueKey: 'a' }, + { url: 'https://example.com/b', uniqueKey: 'b' }, + ]); + expect(api.listAndLockHead).not.toHaveBeenCalled(); + }); + + test('deduplicates re-added requests locally without an API call', async () => { + const api = createMockApiClient(); + const backend = new ApifyRequestQueueSingleBackend(asApiClient(api)); + + await backend.addBatchOfRequests([request('a')]); + const second = await backend.addBatchOfRequests([request('a')]); + + expect(api.batchAddRequests).toHaveBeenCalledTimes(1); + expect(second.processedRequests).toEqual([ + expect.objectContaining({ requestId: id('a'), wasAlreadyPresent: true, wasAlreadyHandled: false }), + ]); + }); + + test('prefetches existing queue contents once and dedups against them', async () => { + const api = createMockApiClient(); + api.listRequests.mockResolvedValue({ + items: [ + { ...request('handled'), id: id('handled'), handledAt: '2025-01-01T00:00:00.000Z' }, + { ...request('pending'), id: id('pending') }, + ], + } as never); + const backend = new ApifyRequestQueueSingleBackend(asApiClient(api)); + + const result = await backend.addBatchOfRequests([request('handled'), request('pending'), request('new')]); + + expect(api.listRequests).toHaveBeenCalledTimes(1); + // Only the genuinely new request reaches the platform. + expect(api.batchAddRequests.mock.calls[0][0]).toEqual([request('new')]); + expect(result.processedRequests).toEqual( + expect.arrayContaining([ + expect.objectContaining({ requestId: id('handled'), wasAlreadyPresent: true, wasAlreadyHandled: true }), + expect.objectContaining({ + requestId: id('pending'), + wasAlreadyPresent: true, + wasAlreadyHandled: false, + }), + expect.objectContaining({ requestId: id('new'), wasAlreadyPresent: false }), + ]), + ); + }); + + test('serves fetched requests from the local cache without a read API call', async () => { + const api = createMockApiClient(); + const backend = new ApifyRequestQueueSingleBackend(asApiClient(api)); + + await backend.addBatchOfRequests([request('a', { userData: { foo: 'bar' } })]); + const fetched = await backend.fetchNextRequest(); + + expect(fetched).toEqual(expect.objectContaining({ uniqueKey: 'a', userData: { foo: 'bar' } })); + expect(api.getRequest).not.toHaveBeenCalled(); + }); + + test('hydrates requests discovered via listHead (added by another producer)', async () => { + const api = createMockApiClient(); + api.listHead.mockResolvedValue({ + items: [{ id: id('foreign'), uniqueKey: 'foreign', url: 'https://example.com/foreign' }], + } as never); + api.getRequest.mockResolvedValue({ ...request('foreign'), id: id('foreign') } as never); + const backend = new ApifyRequestQueueSingleBackend(asApiClient(api)); + + const fetched = await backend.fetchNextRequest(); + + expect(fetched).toEqual(expect.objectContaining({ uniqueKey: 'foreign' })); + expect(api.getRequest).toHaveBeenCalledWith(id('foreign')); + }); + + test('tracks in-progress requests for isFinished', async () => { + const api = createMockApiClient(); + const backend = new ApifyRequestQueueSingleBackend(asApiClient(api)); + + await backend.addBatchOfRequests([request('a')]); + const fetched = (await backend.fetchNextRequest())!; + + await expect(backend.isEmpty()).resolves.toBe(true); + await expect(backend.isFinished()).resolves.toBe(false); + + await backend.markRequestAsHandled(fetched); + await expect(backend.isFinished()).resolves.toBe(true); + expect(api.updateRequest).toHaveBeenCalledWith( + expect.objectContaining({ id: id('a'), handledAt: expect.any(String) }), + { forefront: undefined }, + ); + }); + + test('forefront adds are fetched first', async () => { + const api = createMockApiClient(); + const backend = new ApifyRequestQueueSingleBackend(asApiClient(api)); + + await backend.addBatchOfRequests([request('a')]); + await backend.addBatchOfRequests([request('b')], { forefront: true }); + + expect((await backend.fetchNextRequest())?.uniqueKey).toBe('b'); + expect((await backend.fetchNextRequest())?.uniqueKey).toBe('a'); + }); + + test('reclaimed requests return to the local head and are re-fetched', async () => { + const api = createMockApiClient(); + const backend = new ApifyRequestQueueSingleBackend(asApiClient(api)); + + await backend.addBatchOfRequests([request('a')]); + const fetched = (await backend.fetchNextRequest())!; + + await backend.reclaimRequest(fetched, { forefront: true }); + expect(api.updateRequest).toHaveBeenCalledWith(expect.objectContaining({ id: id('a') }), { forefront: true }); + expect(api.deleteRequestLock).not.toHaveBeenCalled(); + + await expect(backend.isFinished()).resolves.toBe(false); + expect((await backend.fetchNextRequest())?.uniqueKey).toBe('a'); + }); + + test('marking an unknown, nonexistent request is a no-op and never upserts', async () => { + const api = createMockApiClient(); + const backend = new ApifyRequestQueueSingleBackend(asApiClient(api)); + + const result = await backend.markRequestAsHandled({ ...request('ghost'), id: id('ghost') }); + + expect(result).toBeUndefined(); + expect(api.updateRequest).not.toHaveBeenCalled(); + + const reclaimed = await backend.reclaimRequest({ ...request('ghost'), id: id('ghost') }); + expect(reclaimed).toBeUndefined(); + expect(api.updateRequest).not.toHaveBeenCalled(); + }); + + test('purge is not supported on the platform', async () => { + const backend = new ApifyRequestQueueSingleBackend(asApiClient(createMockApiClient())); + await expect(backend.purge()).rejects.toThrow(/not supported on the Apify platform/); + }); +}); + +describe('ApifyRequestQueueSharedBackend', () => { + test('fetches requests via listAndLockHead and hydrates them by id', async () => { + const api = createMockApiClient(); + api.listAndLockHead.mockResolvedValue({ + items: [{ id: id('a'), uniqueKey: 'a', url: 'https://example.com/a' }], + queueHasLockedRequests: true, + clientKey: 'client-key', + } as never); + api.getRequest.mockResolvedValue({ ...request('a', { userData: { foo: 'bar' } }), id: id('a') } as never); + const backend = new ApifyRequestQueueSharedBackend(asApiClient(api)); + + const fetched = await backend.fetchNextRequest(); + + expect(api.listAndLockHead).toHaveBeenCalledWith({ limit: 25, lockSecs: 180 }); + expect(fetched).toEqual(expect.objectContaining({ uniqueKey: 'a', userData: { foo: 'bar' } })); + }); + + test('the lock duration follows the expected processing time and only ever rises', async () => { + const api = createMockApiClient(); + const backend = new ApifyRequestQueueSharedBackend(asApiClient(api)); + + await backend.setExpectedRequestProcessingTimeSecs(600); + await backend.setExpectedRequestProcessingTimeSecs(60); + await backend.fetchNextRequest(); + + expect(api.listAndLockHead).toHaveBeenCalledWith({ limit: 25, lockSecs: 600 }); + }); + + test('isFinished stays false while any client holds locked requests', async () => { + const api = createMockApiClient(); + api.listAndLockHead.mockResolvedValue({ items: [], queueHasLockedRequests: true } as never); + const backend = new ApifyRequestQueueSharedBackend(asApiClient(api)); + + await expect(backend.isEmpty()).resolves.toBe(true); + await expect(backend.isFinished()).resolves.toBe(false); + + api.listAndLockHead.mockResolvedValue({ items: [], queueHasLockedRequests: false } as never); + await expect(backend.isFinished()).resolves.toBe(true); + }); + + test('reclaiming updates the request, releases its lock, and honors forefront', async () => { + const api = createMockApiClient(); + api.listAndLockHead.mockResolvedValueOnce({ + items: [ + { id: id('a'), uniqueKey: 'a', url: 'https://example.com/a' }, + { id: id('b'), uniqueKey: 'b', url: 'https://example.com/b' }, + { id: id('c'), uniqueKey: 'c', url: 'https://example.com/c' }, + ], + queueHasLockedRequests: true, + } as never); + api.getRequest.mockImplementation( + async (requestId: string) => + ({ url: 'https://example.com/x', uniqueKey: `hydrated-${requestId}`, id: requestId }) as never, + ); + const backend = new ApifyRequestQueueSharedBackend(asApiClient(api)); + + const fetched = (await backend.fetchNextRequest())!; + await backend.reclaimRequest({ ...fetched, uniqueKey: 'a' }, { forefront: true }); + + expect(api.updateRequest).toHaveBeenCalledWith(expect.objectContaining({ id: id('a') }), { forefront: true }); + expect(api.deleteRequestLock).toHaveBeenCalledWith(id('a'), { forefront: true }); + + // The forefront reclaim invalidates the local head order — the next fetch re-reads the + // head even though buffered ids remain. + expect(api.listAndLockHead).toHaveBeenCalledTimes(1); + await backend.fetchNextRequest(); + expect(api.listAndLockHead).toHaveBeenCalledTimes(2); + }); + + test('skips requests handled by another client in the meantime', async () => { + const api = createMockApiClient(); + api.listAndLockHead.mockResolvedValueOnce({ + items: [{ id: id('a'), uniqueKey: 'a', url: 'https://example.com/a' }], + queueHasLockedRequests: false, + } as never); + api.getRequest.mockResolvedValue({ + ...request('a', { handledAt: '2025-01-01T00:00:00.000Z' }), + id: id('a'), + } as never); + const backend = new ApifyRequestQueueSharedBackend(asApiClient(api)); + + await expect(backend.fetchNextRequest()).resolves.toBeUndefined(); + }); + + test('deduplicates re-added requests locally without an API call', async () => { + const api = createMockApiClient(); + const backend = new ApifyRequestQueueSharedBackend(asApiClient(api)); + + await backend.addBatchOfRequests([request('a')]); + const second = await backend.addBatchOfRequests([request('a')]); + + expect(api.batchAddRequests).toHaveBeenCalledTimes(1); + expect(second.processedRequests).toEqual([ + expect.objectContaining({ requestId: id('a'), wasAlreadyPresent: true }), + ]); + }); + + test('marking an unknown, nonexistent request is a no-op and never upserts', async () => { + const api = createMockApiClient(); + const backend = new ApifyRequestQueueSharedBackend(asApiClient(api)); + + const result = await backend.markRequestAsHandled({ ...request('ghost'), id: id('ghost') }); + + expect(result).toBeUndefined(); + expect(api.updateRequest).not.toHaveBeenCalled(); + }); + + test('getMetadata merges local estimates with the API counters', async () => { + const api = createMockApiClient(); + const backend = new ApifyRequestQueueSharedBackend(asApiClient(api)); + + // The API counters lag behind (still report zero) right after adding. + await backend.addBatchOfRequests([request('a'), request('b')]); + const metadata = await backend.getMetadata(); + + expect(metadata.totalRequestCount).toBe(2); + expect(metadata.handledRequestCount).toBe(0); + }); +}); diff --git a/test/apify/utils.test.ts b/test/apify/utils.test.ts index 9ee6da174e..6fadc06874 100644 --- a/test/apify/utils.test.ts +++ b/test/apify/utils.test.ts @@ -1,7 +1,6 @@ import type { IncomingMessage } from 'node:http'; import type { Request } from '@crawlee/core'; -import { createRequestDebugInfo } from '@crawlee/utils'; import { Actor, Configuration } from 'apify'; import semver from 'semver'; @@ -90,71 +89,3 @@ describe('printOutdatedSdkWarning()', () => { expect(spy).not.toHaveBeenCalled(); }); }); - -describe('createRequestDebugInfo()', () => { - test('handles Puppeteer response', () => { - const request = { - id: 'some-id', - url: 'https://example.com', - loadedUrl: 'https://example.com', - method: 'POST', - retryCount: 2, - errorMessages: ['xxx'], - someThingElse: 'xxx', - someOther: 'yyy', - } as unknown as Request; - - const response = { - status: () => 201, - another: 'yyy', - }; - - const additionalFields = { - foo: 'bar', - }; - - expect(createRequestDebugInfo(request, response, additionalFields)).toEqual({ - requestId: 'some-id', - url: 'https://example.com', - loadedUrl: 'https://example.com', - method: 'POST', - retryCount: 2, - errorMessages: ['xxx'], - statusCode: 201, - foo: 'bar', - }); - }); - - test('handles NodeJS response', () => { - const request = { - id: 'some-id', - url: 'https://example.com', - loadedUrl: 'https://example.com', - method: 'POST', - retryCount: 2, - errorMessages: ['xxx'], - someThingElse: 'xxx', - someOther: 'yyy', - } as unknown as Request; - - const response = { - statusCode: 201, - another: 'yyy', - } as unknown as IncomingMessage; - - const additionalFields = { - foo: 'bar', - }; - - expect(createRequestDebugInfo(request, response, additionalFields)).toEqual({ - requestId: 'some-id', - url: 'https://example.com', - loadedUrl: 'https://example.com', - method: 'POST', - retryCount: 2, - errorMessages: ['xxx'], - statusCode: 201, - foo: 'bar', - }); - }); -}); diff --git a/test/createIsolatedActor.ts b/test/createIsolatedActor.ts index fed6cdea05..b0071aecff 100644 --- a/test/createIsolatedActor.ts +++ b/test/createIsolatedActor.ts @@ -1,6 +1,5 @@ -import { bindMethodsToServiceLocator, ServiceLocator } from '@crawlee/core'; -import { MemoryStorage } from '@crawlee/memory-storage'; -import type { StorageClient } from '@crawlee/types'; +import { bindMethodsToServiceLocator, MemoryStorageBackend, ServiceLocator } from '@crawlee/core'; +import type { StorageBackend } from '@crawlee/types'; import { Actor, Configuration, type PlatformEventManager } from 'apify'; import { onTestFinished } from 'vitest'; @@ -30,7 +29,7 @@ export interface IsolatedActor { * {@link initIsolatedDefaultActor}. */ export function createIsolatedActor( - options: { config?: Configuration; storageClient?: StorageClient } = {}, + options: { config?: Configuration; storageClient?: StorageBackend } = {}, ): IsolatedActor { const config = options.config ?? new Configuration(); const actor = new Actor({ configuration: config }); @@ -76,15 +75,15 @@ export async function initIsolatedDefaultActor(options: { config?: Configuration /** * Replace a locator's storage client with a fresh in-memory one. * - * On the platform, `Actor.init()` registers an `ApifyStorageClient`; tests that + * On the platform, `Actor.init()` registers an `ApifyStorageBackend`; tests that * assert on `pushData()` / `openDataset()` want local in-memory storage instead. * crawlee services are set-once, so swapping one means resetting the locator and * re-setting it — harmless here because the locator is the test's own isolated * instance, not the shared global one. */ -export function useInMemoryStorage(serviceLocator: ServiceLocator): MemoryStorage { - const storage = new MemoryStorage({ persistStorage: false }); +export function useInMemoryStorage(serviceLocator: ServiceLocator): MemoryStorageBackend { + const storage = new MemoryStorageBackend(); serviceLocator.reset(); - serviceLocator.setStorageClient(storage); + serviceLocator.setStorageBackend(storage); return storage; } diff --git a/test/resetGlobalState.ts b/test/resetGlobalState.ts index c504a97e0b..491739153f 100644 --- a/test/resetGlobalState.ts +++ b/test/resetGlobalState.ts @@ -12,7 +12,7 @@ import { Actor, Configuration } from 'apify'; * * - `Actor._instance` — lazy default `Actor` created by `Actor.getDefaultInstance()`. * - `Configuration.globalConfig` — the SDK's own static singleton (its - * Apify-typed default fallback for `getGlobalConfig()`). + * Apify-typed default fallback for `getGlobalConfiguration()`). * - `serviceLocator` — crawlee's cache for `Configuration` / `EventManager` / * `StorageClient` / `Logger`. Dropped via `serviceLocator.reset()`. *