diff --git a/package-lock.json b/package-lock.json index b844bcaa..e2901f26 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,12 +17,14 @@ "@eslint/eslintrc": "^3.3.1", "@eslint/js": "^9.23.0", "@types/adm-zip": "^0.5.0", + "@types/chrome-remote-interface": "^0.31.14", "@types/jest": "^29.0.0", "@types/node": "^22.0.0", "@types/node-fetch": "^2.6.1", "@types/puppeteer": "^7.0.0", "@types/useragent": "^2.3.1", "browserslist": "^4.21.1", + "chrome-remote-interface": "^0.33.3", "eslint": "^9.23.0", "eslint-config-prettier": "^10.1.1", "fast-csv": "^5.0.0", @@ -1744,6 +1746,21 @@ "@babel/types": "^7.20.7" } }, + "node_modules/@types/chrome-remote-interface": { + "version": "0.31.14", + "resolved": "https://registry.npmjs.org/@types/chrome-remote-interface/-/chrome-remote-interface-0.31.14.tgz", + "integrity": "sha512-H9hTcLu1y+Ms6GDPXXeGhgxaOSD69yEo674vjJw5EeW1tTwYo8fEkf7A9nWlnO6ArJsS7c41iZeX6mRDQ1LhEw==", + "dev": true, + "dependencies": { + "devtools-protocol": "0.0.927104" + } + }, + "node_modules/@types/chrome-remote-interface/node_modules/devtools-protocol": { + "version": "0.0.927104", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.927104.tgz", + "integrity": "sha512-5jfffjSuTOv0Lz53wTNNTcCUV8rv7d82AhYcapj28bC2B5tDxEZzVb7k51cNxZP2KHw24QE+sW7ZuSeD9NfMpA==", + "dev": true + }, "node_modules/@types/debug": { "version": "4.1.12", "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.12.tgz", @@ -2888,6 +2905,40 @@ "node": ">=10" } }, + "node_modules/chrome-remote-interface": { + "version": "0.33.3", + "resolved": "https://registry.npmjs.org/chrome-remote-interface/-/chrome-remote-interface-0.33.3.tgz", + "integrity": "sha512-zNnn0prUL86Teru6UCAZ1yU1XeXljHl3gj7OrfPcarEfU62OUU4IujDPdTDW3dAWwRqN3ZMG/Chhkh2gPL/wiw==", + "dev": true, + "dependencies": { + "commander": "2.11.x", + "ws": "^7.2.0" + }, + "bin": { + "chrome-remote-interface": "bin/client.js" + } + }, + "node_modules/chrome-remote-interface/node_modules/ws": { + "version": "7.5.10", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.10.tgz", + "integrity": "sha512-+dbF1tHwZpXcbOJdVOkzLDxZP1ailvSxM6ZweXTegylPny803bFhA+vqBYw4s31NSAk4S2Qz+AKXK9a4wkdjcQ==", + "dev": true, + "engines": { + "node": ">=8.3.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": "^5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, "node_modules/chromium-bidi": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-5.1.0.tgz", @@ -3000,6 +3051,12 @@ "node": ">= 0.8" } }, + "node_modules/commander": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.11.0.tgz", + "integrity": "sha512-b0553uYA5YAEGgyYIGYROzKQ7X5RAqedkfjiZxwi0kL1g3bOaBNNZfYkzt/CL0umgD5wc9Jec2FbB98CjkMRvQ==", + "dev": true + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -10903,6 +10960,23 @@ "@babel/types": "^7.20.7" } }, + "@types/chrome-remote-interface": { + "version": "0.31.14", + "resolved": "https://registry.npmjs.org/@types/chrome-remote-interface/-/chrome-remote-interface-0.31.14.tgz", + "integrity": "sha512-H9hTcLu1y+Ms6GDPXXeGhgxaOSD69yEo674vjJw5EeW1tTwYo8fEkf7A9nWlnO6ArJsS7c41iZeX6mRDQ1LhEw==", + "dev": true, + "requires": { + "devtools-protocol": "0.0.927104" + }, + "dependencies": { + "devtools-protocol": { + "version": "0.0.927104", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.927104.tgz", + "integrity": "sha512-5jfffjSuTOv0Lz53wTNNTcCUV8rv7d82AhYcapj28bC2B5tDxEZzVb7k51cNxZP2KHw24QE+sW7ZuSeD9NfMpA==", + "dev": true + } + } + }, "@types/debug": { "version": "4.1.12", "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.12.tgz", @@ -11734,6 +11808,25 @@ "dev": true, "peer": true }, + "chrome-remote-interface": { + "version": "0.33.3", + "resolved": "https://registry.npmjs.org/chrome-remote-interface/-/chrome-remote-interface-0.33.3.tgz", + "integrity": "sha512-zNnn0prUL86Teru6UCAZ1yU1XeXljHl3gj7OrfPcarEfU62OUU4IujDPdTDW3dAWwRqN3ZMG/Chhkh2gPL/wiw==", + "dev": true, + "requires": { + "commander": "2.11.x", + "ws": "^7.2.0" + }, + "dependencies": { + "ws": { + "version": "7.5.10", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.10.tgz", + "integrity": "sha512-+dbF1tHwZpXcbOJdVOkzLDxZP1ailvSxM6ZweXTegylPny803bFhA+vqBYw4s31NSAk4S2Qz+AKXK9a4wkdjcQ==", + "dev": true, + "requires": {} + } + } + }, "chromium-bidi": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-5.1.0.tgz", @@ -11818,6 +11911,12 @@ "delayed-stream": "~1.0.0" } }, + "commander": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.11.0.tgz", + "integrity": "sha512-b0553uYA5YAEGgyYIGYROzKQ7X5RAqedkfjiZxwi0kL1g3bOaBNNZfYkzt/CL0umgD5wc9Jec2FbB98CjkMRvQ==", + "dev": true + }, "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", diff --git a/package.json b/package.json index 92f396fa..ccc6b60b 100644 --- a/package.json +++ b/package.json @@ -56,6 +56,7 @@ "@types/puppeteer": "^7.0.0", "@types/useragent": "^2.3.1", "browserslist": "^4.21.1", + "chrome-remote-interface": "^0.33.3", "eslint": "^9.23.0", "eslint-config-prettier": "^10.1.1", "fast-csv": "^5.0.0", diff --git a/packages/fingerprint-injector/package.json b/packages/fingerprint-injector/package.json index bdd9f964..b6d4571a 100644 --- a/packages/fingerprint-injector/package.json +++ b/packages/fingerprint-injector/package.json @@ -22,7 +22,8 @@ }, "peerDependencies": { "playwright": "^1.22.2", - "puppeteer": ">= 9.x" + "puppeteer": ">= 9.x", + "@types/chrome-remote-interface": "^0.31.14" }, "peerDependenciesMeta": { "playwright": { @@ -30,6 +31,9 @@ }, "puppeteer": { "optional": true + }, + "@types/chrome-remote-interface" : { + "optional": true } }, "scripts": { diff --git a/packages/fingerprint-injector/src/fingerprint-injector.ts b/packages/fingerprint-injector/src/fingerprint-injector.ts index 71ff881d..ff2d7253 100644 --- a/packages/fingerprint-injector/src/fingerprint-injector.ts +++ b/packages/fingerprint-injector/src/fingerprint-injector.ts @@ -1,4 +1,5 @@ import { readFileSync } from 'fs'; +import type CDP from 'chrome-remote-interface'; import { BrowserFingerprintWithHeaders, @@ -19,6 +20,13 @@ interface EnhancedFingerprint extends Fingerprint { historyLength: number; } +type AttachFingerprintToCDPparams = { + page: CDP.StableDomains['Page']; + network: CDP.StableDomains['Network']; + emulation: CDP.StableDomains['Emulation']; + browser: CDP.StableDomains['Browser']; +}; + declare function overrideInstancePrototype( instance: T, overrideObj: Partial, @@ -166,6 +174,53 @@ export class FingerprintInjector { ); } + /** + * Adds script that is evaluated before every document creation. + * Sets User-Agent and viewport using native puppeteer interface + * @param params AttachFingerprintToCDPparams `AttachFingerprintToCDPparams` object to be injected with the fingerprint. + * @param fingerprint Fingerprint from [`fingerprint-generator`](https://github.com/apify/fingerprint-generator). + */ + async attachFingerprintToCDP( + { page, network, emulation, browser }: AttachFingerprintToCDPparams, + browserFingerprintWithHeaders: BrowserFingerprintWithHeaders, + ): Promise { + const { fingerprint, headers } = browserFingerprintWithHeaders; + const enhancedFingerprint = this._enhanceFingerprint(fingerprint); + const { screen, userAgent } = enhancedFingerprint; + + await network.setUserAgentOverride({ userAgent }); + const { product: browserVersion } = await browser.getVersion(); + + if (!browserVersion.toLowerCase().includes('firefox')) { + page.setDeviceMetricsOverride({ + screenHeight: screen.height, + screenWidth: screen.width, + width: screen.width, + height: screen.height, + mobile: /phone|android|mobile/i.test(userAgent), + screenOrientation: + screen.height > screen.width + ? { angle: 0, type: 'portraitPrimary' } + : { angle: 90, type: 'landscapePrimary' }, + deviceScaleFactor: screen.devicePixelRatio, + }); + + await network.setExtraHTTPHeaders({ + headers: this.onlyInjectableHeaders(headers, browserVersion), + }); + + await emulation.setEmulatedMedia({ + features: [{ name: 'prefers-color-scheme', value: 'dark' }], + }); + } + + await page.addScriptToEvaluateOnNewDocument({ + source: this.getInjectableFingerprintFunction(enhancedFingerprint), + // @ts-ignore unfortunately types are too old, its essential to guarantee script is executed immediately + runImmediately: true, + }); + } + /** * Gets the override script that should be evaluated in the browser. */ diff --git a/test/fingerprint-injector/crossbrowser.test.ts b/test/fingerprint-injector/crossbrowser.test.ts index 53f77f5b..a1102d37 100644 --- a/test/fingerprint-injector/crossbrowser.test.ts +++ b/test/fingerprint-injector/crossbrowser.test.ts @@ -1,10 +1,12 @@ import { FingerprintInjector, + newCDPInjector, newInjectedContext, newInjectedPage, } from 'fingerprint-injector'; import playwright from 'playwright'; -import puppeteer from 'puppeteer'; +import puppeteer, { Browser } from 'puppeteer'; +import CDP from 'chrome-remote-interface'; function generateCartesianMatrix(A: any, B: any) { const matrix = []; @@ -81,3 +83,53 @@ describe('Puppeteer controlled instances', () => { }, ); }); +describe('CDP controller instances', () => { + const fingerprintBrowsers = [ + 'chrome', + 'firefox', + 'safari', + 'edge', + ] as const; + test.each(fingerprintBrowsers)( + `should inject %s fingerprint`, + async (fingerprintBrowser: (typeof fingerprintBrowsers)[number]) => { + const puppeteer_browser = await puppeteer.launch({ + browser: 'chrome', + debuggingPort: 9222, // need to specify it explicitly or puppeteer launcher won't open it up + headless: true, + }); + const webSocketDebuggerUrl = puppeteer_browser.wsEndpoint(); + const client = await CDP({ target: webSocketDebuggerUrl }); + const { Target } = client; + + // getting the default 'about:blank' page + const { targetInfos } = await Target.getTargets(); + const ctx_client = await CDP({ target: targetInfos[0].targetId }); + + const { Network, Page, Browser, Emulation } = ctx_client; + await Network.enable(); + await Page.enable(); + + await newCDPInjector( + { + network: Network, + page: Page, + browser: Browser, + emulation: Emulation, + }, + { + fingerprintOptions: { + browsers: [fingerprintBrowser], + }, + }, + ); + + const { frameId } = await Page.navigate({ + url: 'http://example.com', + }); + expect(frameId).toBeDefined(); + await client.close(); + await puppeteer_browser.close(); + }, + ); +}); diff --git a/test/fingerprint-injector/fingerprint-injector.test.ts b/test/fingerprint-injector/fingerprint-injector.test.ts index 28e80a2a..b29cbc5a 100644 --- a/test/fingerprint-injector/fingerprint-injector.test.ts +++ b/test/fingerprint-injector/fingerprint-injector.test.ts @@ -12,6 +12,7 @@ import { } from 'fingerprint-injector'; import playwright, { chromium, type Browser as PWBrowser } from 'playwright'; import puppeteer, { Browser as PPBrowser } from 'puppeteer'; +import CDP from 'chrome-remote-interface'; const cases = [ [ @@ -63,7 +64,24 @@ const cases = [ }, ], ], -]; + [ + 'CDP', + [ + { + name: 'Chrome', + launcher: puppeteer, + options: { + args: ['--no-sandbox', '--use-gl=desktop'], + channel: 'chrome', + debuggingPort: 9222, + }, + fingerprintGeneratorOptions: { + browsers: [{ name: 'chrome', minVersion: 90 }], + }, + }, + ], + ], +] as const; describe('FingerprintInjector', () => { let fpInjector: FingerprintInjector; @@ -76,7 +94,6 @@ describe('FingerprintInjector', () => { // eslint-disable-next-line dot-notation expect(fpInjector['utilsJs']).toBeTruthy(); }); - // @ts-expect-error test only describe.each(cases)('%s', (frameworkName, testCases) => { // @ts-expect-error test only describe.each(testCases)( @@ -90,6 +107,7 @@ describe('FingerprintInjector', () => { let fingerprintWithHeaders: BrowserFingerprintWithHeaders; let fingerprint: Fingerprint; let context: any; + let client: CDP.Client | undefined; beforeAll(async () => { fingerprintGenerator = new FingerprintGenerator({ @@ -135,6 +153,113 @@ describe('FingerprintInjector', () => { fingerprintWithHeaders, ); + response = await page.goto( + `file://${__dirname}/test.html`, + ); + } else if (frameworkName === 'CDP') { + browser = await launcher.launch({ + headless: false, + debuggingPort: 9222, + ...options, + }); + + const client = await CDP({ + target: browser.wsEndpoint(), + }); + + const { targetInfos } = + await client.Target.getTargets(); + + const ctx_client = await CDP({ + target: targetInfos[0].targetId, + }); + + const { Page, Network, Emulation, Runtime, Target } = + ctx_client; + + await Page.enable(); + await Network.enable(); + + // TODO: remove after testing, undefined on per-case runs + // fpInjector ??= new FingerprintInjector(); + + await fpInjector.attachFingerprintToCDP( + { + page: Page, + network: Network, + emulation: Emulation, + browser: client.Browser, + }, + fingerprintWithHeaders, + ); + + const requestHeaders = new Map< + string, + Record + >(); + Network.requestWillBeSent((params) => { + if (params.type === 'Document') { + let lowerCase: Record = {}; + for (const header of Object.keys( + params.request.headers, + )) { + lowerCase[header.toLowerCase()] = + params.request.headers[header]; + } + requestHeaders.set(params.frameId, lowerCase); + } + }); + + page = { + evaluate: async ( + fn: (...args: unknown[]) => unknown, + ...args: unknown[] + ) => { + const stringified = stringifyFunction(fn); + const { targetInfo: ti } = + await Target.getTargetInfo(); + const { sessionId } = + await Target.attachToTarget({ + targetId: ti.targetId, + flatten: true, + }); + ctx_client.send( + 'Runtime.enable', + undefined, + sessionId, + ); + const ctx = + await Runtime.executionContextCreated(); + const evaluated = await Runtime.callFunctionOn( + { + functionDeclaration: stringified, + executionContextId: ctx.context.id, + arguments: args.map((a) => ({ + value: a, + })), + awaitPromise: true, + + returnByValue: true, + }, + sessionId, + ); + await Runtime.disable(); + return evaluated.result.value; + }, + goto: async (url: string) => { + const { frameId } = await Page.navigate({ + url, + }); + await Page.loadEventFired(); + + return { + request: () => ({ + headers: () => + requestHeaders.get(frameId), + }), + }; + }, + }; response = await page.goto( `file://${__dirname}/test.html`, ); @@ -143,6 +268,9 @@ describe('FingerprintInjector', () => { }); afterAll(async () => { + if (client) { + await client.close(); + } if (browser) { await browser.close(); } @@ -408,7 +536,6 @@ describe('FingerprintInjector', () => { ); }); - // @ts-expect-error test only describe.each(cases)('%s', (frameworkName, testCases) => { // @ts-expect-error test only describe.each(testCases)( @@ -438,6 +565,51 @@ describe('FingerprintInjector', () => { await fpInjector.attachFingerprintToPuppeteer(page, fp); return page; } + if (frameworkName === 'CDP') { + const orig_close = browser.close; + const client = await CDP({ + target: (browser as PPBrowser).wsEndpoint(), + }); + const { targetInfos } = + await client.Target.getTargets(); + + const ctx_client = await CDP({ + target: targetInfos[0].targetId, + }); + + browser.close = async () => { + await client.close(); + await ctx_client.close(); + return orig_close.call(browser); + }; + + const { Page, Network, Emulation, Runtime } = + ctx_client; + await Page.enable(); + await Network.enable(); + await fpInjector.attachFingerprintToCDP( + { + page: Page, + network: Network, + emulation: Emulation, + browser: client.Browser, + }, + fp, + ); + return { + goto: async (url: string) => { + await Page.navigate({ url }); + }, + + $: async (selector: string) => { + const { result } = await Runtime.evaluate({ + expression: `document.querySelector('${selector}')`, + returnByValue: true, + }); + return result.value; + }, + }; + } throw new Error(`Unknown framework name ${frameworkName}`); }; @@ -555,3 +727,36 @@ describe('FingerprintInjector', () => { }); }); }); + +// from https://github.com/puppeteer/puppeteer/blob/main/packages/puppeteer-core/src/util/Function.ts#L30 +function stringifyFunction(fn: (...args: never) => unknown): string { + let value = fn.toString(); + try { + new Function(`(${value})`); + } catch (err) { + if ( + (err as Error).message.includes( + `Refused to evaluate a string as JavaScript because 'unsafe-eval' is not an allowed source of script in the following Content Security Policy directive`, + ) + ) { + // The content security policy does not allow Function eval. Let's + // assume the value might be valid as is. + return value; + } + // This means we might have a function shorthand (e.g. `test(){}`). Let's + // try prefixing. + let prefix = 'function '; + if (value.startsWith('async ')) { + prefix = `async ${prefix}`; + value = value.substring('async '.length); + } + value = `${prefix}${value}`; + try { + new Function(`(${value})`); + } catch { + // We tried hard to serialize, but there's a weird beast here. + throw new Error('Passed function cannot be serialized!'); + } + } + return value; +} diff --git a/test/generative-bayesian-network/testNetworkDefinition.zip b/test/generative-bayesian-network/testNetworkDefinition.zip index 6ec3ce56..f01070e2 100644 Binary files a/test/generative-bayesian-network/testNetworkDefinition.zip and b/test/generative-bayesian-network/testNetworkDefinition.zip differ