feat(core): DataDome headless bypass, anti-detection patches, CloakBrowser integration

jonzarecki · jonzarecki · commit 35bf2b4068c2 · 2026-03-29T09:35:21.000+02:00
- antiDetection config system (stripCOOP, patchPointerMedia, saveCookieDomains, useCloakBrowser)
- CloakBrowser lazy optional dependency for DataDome-protected sites (e.g. Booking.com)
- viewport: null globally — removes detectable 1280x720 Playwright default
- --blink-settings=primaryPointerType=4 + --window-size globally for all adapters
- about:blank startup fix: getStatus() navigates to loginUrl before isLoggedIn check
- create-adapter scaffold updated with selector stability guidance

Booking.com secure.booking.com now works in headless mode via CloakBrowser.
diff --git a/.cursor/hooks/state/continual-learning-index.json b/.cursor/hooks/state/continual-learning-index.json
@@ -54,8 +54,20 @@
       "lastProcessedAt": "2026-03-25T16:00:00.000Z"
     },
     "/Users/jzarecki/.cursor/projects/Users-jzarecki-Projects-session-mcp/agent-transcripts/672ffd14-9bc9-4c91-9eb7-9759b40e8893/672ffd14-9bc9-4c91-9eb7-9759b40e8893.jsonl": {
-      "mtimeMs": 1774474577000,
-      "lastProcessedAt": "2026-03-25T16:00:00.000Z"
+      "mtimeMs": 1774522118000,
+      "lastProcessedAt": "2026-03-26T00:00:00.000Z"
+    },
+    "/Users/jzarecki/.cursor/projects/Users-jzarecki-Projects-session-mcp/agent-transcripts/d03e9a64-5187-4bee-88df-279763fa977e/d03e9a64-5187-4bee-88df-279763fa977e.jsonl": {
+      "mtimeMs": 1774474691000,
+      "lastProcessedAt": "2026-03-26T00:00:00.000Z"
+    },
+    "/Users/jzarecki/.cursor/projects/Users-jzarecki-Projects-session-mcp/agent-transcripts/8511b06d-07c8-42f2-8f30-5b6762c2fdc6/8511b06d-07c8-42f2-8f30-5b6762c2fdc6.jsonl": {
+      "mtimeMs": 1774523258000,
+      "lastProcessedAt": "2026-03-26T00:00:00.000Z"
+    },
+    "/Users/jzarecki/.cursor/projects/Users-jzarecki-Projects-session-mcp/agent-transcripts/9bec9e6f-a179-401c-90cc-6d64720fbfee/9bec9e6f-a179-401c-90cc-6d64720fbfee.jsonl": {
+      "mtimeMs": 1774534587000,
+      "lastProcessedAt": "2026-03-26T00:00:00.000Z"
     }
   }
 }
diff --git a/.cursor/hooks/state/continual-learning.json b/.cursor/hooks/state/continual-learning.json
@@ -1,8 +1,8 @@
 {
   "version": 1,
-  "lastRunAtMs": 1774519514313,
-  "turnsSinceLastRun": 5,
-  "lastTranscriptMtimeMs": 1774519513950,
-  "lastProcessedGenerationId": "492daf12-4802-4a02-a01e-a454c517b849",
+  "lastRunAtMs": 1774534549568,
+  "turnsSinceLastRun": 6,
+  "lastTranscriptMtimeMs": 1774534549508,
+  "lastProcessedGenerationId": "785251f5-f6dd-46fd-bd02-6b91246cd354",
   "trialStartedAtMs": null
 }
diff --git a/AGENTS.md b/AGENTS.md
@@ -30,7 +30,7 @@ Durable facts and correction patterns for this workspace. Updated by continual-l
 - Browser mode switching (`headless` / `watch` / `paused`), screenshot, page state, and navigate are consolidated into a single `browser` MCP tool with an `action` parameter — user explicitly asked to reduce tool count ("too many tools"); do NOT revert to 5 separate management tools
 - Management tools bypass the LockManager; regular automation tools go through it
 - "Raw" Playwright access means exposing the CDP WebSocket URL (`wsEndpoint()`) of each adapter's browser — external agents (Claude Code, Cursor) attach to the already-authenticated session and write their own Playwright scripts via shell
-- The Playwright skill pattern: AI writes a script to `/tmp`, executes it via shell — no `run(code)` MCP tool needed
+- The Playwright skill pattern: AI writes a script to `/tmp`, executes it via shell — primary approach for shell-capable clients (Cursor, Claude Code); opt-in `run_script` MCP tool planned for clients without shell access (Claude Desktop)
 - MCP resources use `page://${site}/snapshot` (site name dynamic) — user pushed back when the URI appeared to hardcode the adapter name
 - Testing utilities (`createTestAdapterServer`, `createTestMcpClient`) live at `@browserkit/core/testing` subpath — a separate harness package was explicitly rejected ("I don't think we need it, it should be in either adapter or in core")
 - Real Chrome (`channel: "chrome"`) is required for Google-based adapters — Playwright's bundled Chromium is blocked by Google's login with "This browser or app may not be secure". `isLoggedIn` must NOT navigate during login polling or it redirects the user away from the sign-in page.
@@ -41,6 +41,7 @@ Durable facts and correction patterns for this workspace. Updated by continual-l
 - Framework navigates to `adapter.loginUrl` before calling `isLoggedIn()` when browser is at `about:blank` — adapters do NOT need to handle this themselves
 - `warm_up_browser()` (visiting google/wiki/github before login) was evaluated from stickerdaniel's code — decided as "nice to have" for first-time login, not adopted yet
 - `browserkit login <site>` is blocked by the `CI=1` env var that Cursor sets — must run as `CI="" node packages/core/dist/cli.js login <site>` to open a headed browser from within Cursor terminal
+- `browser` tool `snapshot` action is planned — returns incremental aria-snapshot diff, more token-efficient than screenshots; `page-snapshot` MCP resource already exists, the action adds diff support. Inspired by `SawyerHood/dev-browser`.
 
 ## Design Process Preferences
 
diff --git a/browserkit.config.js b/browserkit.config.js
@@ -11,5 +11,12 @@ export default {
       channel: "chrome",
     },
     "/Users/jzarecki/Projects/session-mcp/packages/adapter-linkedin/dist/index.js": { port: 52744, channel: "chrome" },
+    "/Users/jzarecki/Projects/browserkit-adapter-booking/dist/index.js": {
+      port: 52745,
+      // channel: "chrome" not used — CloakBrowser uses its own Chromium binary
+      antiDetection: {
+        useCloakBrowser: true,  // CloakBrowser's 33 C++ patches handle everything DataDome checks
+      },
+    },
   },
 };
diff --git a/packages/core/package.json b/packages/core/package.json
@@ -28,8 +28,9 @@
   },
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.10.2",
-    "pino": "^9.6.0",
     "patchright": "^1.51.1",
+    "pino": "^9.6.0",
+    "playwright-core": "^1.58.2",
     "zod": "^3.24.2"
   },
   "devDependencies": {
@@ -46,6 +47,7 @@
     "README.md"
   ],
   "optionalDependencies": {
-    "@browserkit/adapter-hackernews": "workspace:*"
+    "@browserkit/adapter-hackernews": "workspace:*",
+    "cloakbrowser": "^0.3.18"
   }
 }
diff --git a/packages/core/src/adapter-server.ts b/packages/core/src/adapter-server.ts
@@ -57,6 +57,7 @@ export async function createAdapterServer(
     debugPort: adapterConfig.debugPort,
     deviceEmulation: adapterConfig.deviceEmulation,
     channel: adapterConfig.channel,
+    antiDetection: adapterConfig.antiDetection,
   };
 
   let lastCallAt: Date | undefined;
diff --git a/packages/core/src/session-manager.ts b/packages/core/src/session-manager.ts
@@ -114,6 +114,25 @@ export class SessionManager {
   async closeSite(site: string): Promise<void> {
     const entry = this.sessions.get(site);
     if (!entry) return;
+    // Save sticky session cookies (e.g. DataDome's `datadome` cookie) to disk
+    // before closing, so they can be restored on the next headless launch.
+    if (entry.config.antiDetection?.saveCookieDomains?.length) {
+      try {
+        const allCookies = await entry.context.cookies();
+        const toSave = allCookies.filter((c) =>
+          (entry.config.antiDetection?.saveCookieDomains ?? []).some(
+            (d) => c.domain.endsWith(d.replace(/^\./, ""))
+          )
+        );
+        if (toSave.length > 0) {
+          const savePath = path.join(this.dataDir, "profiles", site, "saved-session-cookies.json");
+          fs.writeFileSync(savePath, JSON.stringify(toSave, null, 2), { mode: 0o600 });
+          log.info({ site, cookieCount: toSave.length }, "saved sticky session cookies");
+        }
+      } catch (err) {
+        log.warn({ site, err }, "failed to save sticky session cookies");
+      }
+    }
     try { await entry.context.close(); } catch (err) { log.warn({ site, err }, "close error"); }
     this.sessions.delete(site);
   }
@@ -178,17 +197,125 @@ export class SessionManager {
       const antiAutomationArgs = [
         "--disable-blink-features=AutomationControlled",
         "--disable-infobars",
+        // Declare a primary pointer (mouse) and available pointer types at the Blink level.
+        // Without this, headless Chrome reports pointer type = none, which causes CSS
+        // (pointer: fine) to return false — a key DataDome headless detection signal.
+        // Type 4 = kFine (mouse), type 1 = kNone, type 2 = kCoarse (touch).
+        "--blink-settings=primaryPointerType=4,availablePointerTypes=4,primaryHoverType=2,availableHoverTypes=2",
+        // Set a realistic window size so screen.width/height are non-zero.
+        "--window-size=1920,1080",
       ];
-      context = await chromium.launchPersistentContext(profileDir, {
-        headless: !headed,
-        slowMo,
-        args: [...debugArgs, ...antiAutomationArgs],
-        ...(config.channel ? { channel: config.channel } : {}),
-        ...devicePreset,
-      });
+
+      if (config.antiDetection?.useCloakBrowser) {
+        // CloakBrowser: stealth Chromium with 33 C++-level patches for DataDome.
+        // Uses its own Chromium binary — cannot share profiles with real Chrome.
+        // Profile dir is sibling to the standard profile: <site>-cloak
+        // Lazy import — only loaded when useCloakBrowser is configured, so adapters
+        // that don't need it never trigger the ~140MB binary download on install.
+        const { launchPersistentContext: cloakLaunch } = await import("cloakbrowser");
+        const cloakProfileDir = this.getProfileDir(`${config.site}-cloak`);
+        fs.mkdirSync(cloakProfileDir, { recursive: true, mode: 0o700 });
+        log.info({ site: config.site, cloakProfileDir }, "launching CloakBrowser for DataDome bypass");
+        context = await cloakLaunch({
+          userDataDir: cloakProfileDir,
+          headless: !headed,
+          humanize: true,
+          args: antiAutomationArgs,
+        }) as unknown as BrowserContext;
+      } else {
+        context = await chromium.launchPersistentContext(profileDir, {
+          headless: !headed,
+          slowMo,
+          // viewport: null tells Chrome to use its natural window size rather than
+          // Playwright's default 1280x720 — the fixed Playwright viewport is a known
+          // bot-detection signal (e.g. DataDome's c.js checks for it).
+          // Device presets override this with the device's specific viewport.
+          viewport: null,
+          args: [...debugArgs, ...antiAutomationArgs],
+          ...(config.channel ? { channel: config.channel } : {}),
+          ...devicePreset,
+        });
+      }
     }
 
     const page = await context.newPage();
+
+    // ── Anti-detection patches (DataDome / similar challenge-based protection) ─
+    // Applied after context creation so they cover the first navigation too.
+    //
+    // stripCOOP: strip Cross-Origin-Opener-Policy and Cross-Origin-Embedder-Policy
+    //   response headers. Prevents Chromium from restarting its renderer process on
+    //   COOP-protected pages, which would reset --blink-settings pointer emulation.
+    //   Without this fix, (pointer:fine) flips to false mid-navigation on secure.booking.com.
+    //
+    // patchPointerMedia: override window.matchMedia so (pointer:fine) = true in headless.
+    //   DataDome's c.js challenge script reads this signal to classify headless Chrome as a bot.
+    //   Uses context.addInitScript so it runs in every frame and survives cross-origin navigations.
+    if (config.antiDetection?.stripCOOP) {
+      await context.route("**/*", async (route) => {
+        try {
+          const response = await route.fetch();
+          const headers = { ...response.headers() };
+          delete headers["cross-origin-opener-policy"];
+          delete headers["cross-origin-embedder-policy"];
+          await route.fulfill({ response, headers });
+        } catch {
+          await route.continue();
+        }
+      });
+    }
+    if (config.antiDetection?.patchPointerMedia) {
+      await context.addInitScript(() => {
+        // 1. Patch window.matchMedia so (pointer:fine) returns true in headless.
+        //    DataDome's c.js challenge reads this to classify headless Chrome as a bot.
+        const orig = window.matchMedia.bind(window);
+        window.matchMedia = (query: string): MediaQueryList => {
+          const mql = orig(query);
+          const q = query.replace(/\s+/g, "").toLowerCase();
+          if (q.includes("pointer:fine"))   return Object.assign(Object.create(mql), { matches: true  });
+          if (q.includes("pointer:none"))   return Object.assign(Object.create(mql), { matches: false });
+          if (q.includes("pointer:coarse")) return Object.assign(Object.create(mql), { matches: false });
+          return mql;
+        };
+
+        // 2. Patch window.outerHeight / outerWidth to simulate a real browser toolbar.
+        //    In headless Chrome, outerHeight === innerHeight (no toolbar rendered).
+        //    DataDome's c.js checks this: zero toolbar height = headless signal.
+        //    Real Chrome on macOS has ~74px of toolbar (address bar + tabs).
+        const TOOLBAR_HEIGHT = 74;
+        try {
+          Object.defineProperty(window, "outerHeight", {
+            get: () => window.innerHeight + TOOLBAR_HEIGHT,
+            configurable: true,
+          });
+          Object.defineProperty(window, "outerWidth", {
+            get: () => window.innerWidth,
+            configurable: true,
+          });
+        } catch { /* ignore — may already be non-configurable */ }
+      });
+    }
+
+    // ── Restore sticky session cookies ────────────────────────────────────────
+    // DataDome and similar protection systems use session cookies (no expiry) that
+    // Chrome doesn't persist to disk. We save them manually before closing and
+    // restore them here so headless launches don't start from scratch every time.
+    if (config.antiDetection?.saveCookieDomains?.length) {
+      const savePath = path.join(this.dataDir, "profiles", config.site, "saved-session-cookies.json");
+      if (fs.existsSync(savePath)) {
+        try {
+          const saved = JSON.parse(fs.readFileSync(savePath, "utf8")) as Array<Record<string, unknown>>;
+          if (saved.length > 0) {
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+          await context.addCookies(saved as any);
+            log.info({ site: config.site, cookieCount: saved.length }, "restored sticky session cookies");
+          }
+        } catch (err) {
+          log.warn({ site: config.site, err }, "failed to restore sticky session cookies");
+        }
+      }
+    }
+
     this.sessions.set(config.site, { context, page, config, mode, slowMoMs, debugPort: config.debugPort });
     log.info({ site: config.site, mode, debugPort: config.debugPort }, "browser session ready");
     return page;
diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts
@@ -88,6 +88,15 @@ export interface SessionConfig {
    * crashes in headed mode.
    */
   channel?: string | undefined;
+  /**
+   * Anti-bot-detection patches. See AdapterConfig.antiDetection for docs.
+   */
+  antiDetection?: {
+    stripCOOP?: boolean | undefined;
+    patchPointerMedia?: boolean | undefined;
+    saveCookieDomains?: string[] | undefined;
+    useCloakBrowser?: boolean | undefined;
+  } | undefined;
 }
 
 // ─── Adapter config (per-entry in browserkit.config.ts) ────────────────────
@@ -122,6 +131,38 @@ export interface AdapterConfig {
    * Accepts: "chrome" | "chrome-beta" | "msedge" | undefined (default: Playwright Chromium)
    */
   channel?: string | undefined;
+  /**
+   * Anti-bot-detection patches for sites using DataDome or similar challenge-based
+   * protection (e.g. Booking.com's secure.booking.com).
+   *
+   * stripCOOP: Strip Cross-Origin-Opener-Policy and Cross-Origin-Embedder-Policy
+   *   response headers via a context-level route interceptor. Prevents Chromium from
+   *   restarting its renderer process mid-navigation, which would reset Playwright's
+   *   --blink-settings pointer emulation and cause (pointer:fine) to flip to false.
+   *
+   * patchPointerMedia: Override window.matchMedia in every frame so that
+   *   (pointer:fine) returns true in headless. DataDome's challenge script (c.js)
+   *   reads this signal to classify headless Chrome as a bot.
+   */
+  antiDetection?: {
+    stripCOOP?: boolean | undefined;
+    patchPointerMedia?: boolean | undefined;
+    /**
+     * Domains whose cookies should be saved to disk before the browser closes
+     * and restored on the next launch. Use for session cookies that DataDome
+     * or similar systems set but Chrome doesn't persist natively (e.g. `datadome`).
+     * Example: [".booking.com", "captcha-delivery.com"]
+     */
+    saveCookieDomains?: string[] | undefined;
+    /**
+     * Use CloakBrowser (stealth Chromium with 33 C++-level patches) instead of
+     * Patchright. Downloads ~140MB on first use, cached at ~/.cloakbrowser/.
+     * Required for sites using DataDome (e.g. Booking.com's secure subdomain).
+     * NOTE: incompatible with channel:"chrome" — uses its own Chromium binary.
+     * Profiles created with real Chrome will not work; use a separate profileDir.
+     */
+    useCloakBrowser?: boolean | undefined;
+  } | undefined;
 }
 
 // ─── Handoff ─────────────────────────────────────────────────────────────────
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml

Original file line number	Diff line number	Diff line change
`@@ -1,8 +1,8 @@`
`1`	`1`	`{`
`2`	`2`	`"version": 1,`
`3`		`- "lastRunAtMs": 1774519514313,`
`4`		`- "turnsSinceLastRun": 5,`
`5`		`- "lastTranscriptMtimeMs": 1774519513950,`
`6`		`- "lastProcessedGenerationId": "492daf12-4802-4a02-a01e-a454c517b849",`
	`3`	`+ "lastRunAtMs": 1774534549568,`
	`4`	`+ "turnsSinceLastRun": 6,`
	`5`	`+ "lastTranscriptMtimeMs": 1774534549508,`
	`6`	`+ "lastProcessedGenerationId": "785251f5-f6dd-46fd-bd02-6b91246cd354",`
`7`	`7`	`"trialStartedAtMs": null`
`8`	`8`	`}`