fix: separate crawlers from apps that merely name the vendor

CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## Unreleased
+
+- Detect headless Chrome. The list carried `chromeheadless`, which is the Karma
+  launcher label; Chrome itself reports `HeadlessChrome/<version>`.
+- Detect Scrapy.
+- Stop flagging Play Store WebViews as bots. The bare `google` pattern matched any
+  user-agent carrying `Channel/googleplay`, which is why TikTok needed a hard-coded
+  exemption; the pattern is now the documented Google crawler tokens and the
+  exemption is gone. `botName` for Googlebot is now `googlebot` rather than `google`.
+- Distinguish the Pinterest crawler (`Pinterest/0.2`) from the Pinterest in-app
+  browser (`[Pinterest/iOS]`), which was reported as a bot.
+
 ## 2.2.1 - 2026-06-30
 
 - Updated dependency metadata and lockfile entries for recent transitive security advisories.

CONTRIBUTING.md

@@ -23,7 +23,11 @@ const BOTS = [
 
 Add the new bot pattern to the `BOTS` array. Follow these best practices:
 
-- **Use regex patterns**, not exact matches (e.g., use `'google'` instead of `'Googlebot'`)
+- **Match the crawler, not the vendor.** A pattern must not match a browser that merely
+  mentions the vendor. `'google'` once matched every Play Store WebView carrying
+  `Channel/googleplay`, and `'pinterest'` matched the Pinterest in-app browser. Prefer the
+  crawler token (`'googleother'`, `'pinterestbot'`), or a lookahead that pins the shape the
+  crawler uses (`'pinterest(?=\\/\\d)'` matches `Pinterest/0.2` but not `[Pinterest/iOS]`)
 - **Handle variations** - include common variations (e.g., both `'phantom\\.js'` and `'phantomjs'`)
 - **Escape special characters** properly (dots, spaces, etc.)
 - **Keep patterns simple** - avoid overly complex regex that might miss variations

src/express-useragent.ts

@@ -34,6 +34,7 @@ const BOTS = [
   'baiduspider',
   'bingbot',
   'chromeheadless',
+  'headlesschrome',
   'cloudflare',
   'cloudinary',
   'crawler',
@@ -45,7 +46,16 @@ const BOTS = [
   'facebookexternalhit',
   'facebot',
   'flipboard',
-  'google',
+  'apis-google',
+  'feedfetcher-google',
+  'google\\spage\\sspeed',
+  'google\\sweb\\spreview',
+  'google-inspectiontool',
+  'google-read-aloud',
+  'google-site-verification',
+  'googleother',
+  'googleweblight',
+  'storebot-google',
   'googlebot',
   'gsa-crawler',
   'gurujibot',
@@ -61,9 +71,11 @@ const BOTS = [
   'phantom\\.js',
   'phantomjs',
   'pingdom',
-  'pinterest',
+  'pinterest(?=\\/\\d)',
+  'pinterestbot',
   'python',
   'rtlnieuws',
+  'scrapy',
   'skypeuripreview',
   'slackbot',
   'slurp',
@@ -780,16 +792,6 @@ export class UserAgent {
     if (match) {
       const botIdentifier = match[1];
 
-      // Handle false positives - TikTok WebView contains "googleplay" but isn't a bot
-      if (
-        botIdentifier === 'google' &&
-        (source.includes('tiktok') || source.includes('trill') || source.includes('bytedance'))
-      ) {
-        this.Agent.isBot = false;
-        this.Agent.botName = '';
-        return;
-      }
-
       // For all bots, return boolean true and store bot name (fixes issues #168, #138)
       this.Agent.isBot = true;
       this.Agent.botName = botIdentifier;

tests/bots.test.ts

@@ -470,3 +470,145 @@ describe('Failing Bot detection cases', () => {
     });
   });
 });
+
+interface DetectionCase {
+  name: string;
+  source: string;
+  isBot: boolean;
+  botName?: string;
+}
+
+// Headless Chrome reports "HeadlessChrome/<version>"; "chromeheadless" is the
+// Karma launcher label and never appears in a user-agent string.
+const headlessAndScraperCases: DetectionCase[] = [
+  {
+    name: 'Headless Chrome',
+    source:
+      'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/120.0.0.0 Safari/537.36',
+    isBot: true,
+    botName: 'headlesschrome',
+  },
+  {
+    name: 'Scrapy',
+    source: 'Scrapy/2.11 (+https://scrapy.org)',
+    isBot: true,
+    botName: 'scrapy',
+  },
+];
+
+// "google" as a bare substring matched every Play Store WebView that carries
+// "Channel/googleplay". TikTok was special-cased for exactly this reason; the
+// same bug hit every other app shipping that token.
+const googleCases: DetectionCase[] = [
+  {
+    name: 'Play Store WebView of an unrelated app is not a bot',
+    source:
+      'Mozilla/5.0 (Linux; Android 12; RMX3085 Build/SP1A.210812.016; wv) AppleWebKit/537.36 ' +
+      '(KHTML, like Gecko) Version/4.0 Chrome/107.0.5304.105 Mobile Safari/537.36 JsSdk/1.0 ' +
+      'NetType/WIFI Channel/googleplay AppName/shopmate app_version/5.2.1',
+    isBot: false,
+  },
+  {
+    name: 'TikTok WebView stays human without the special case',
+    source:
+      'Mozilla/5.0 (Linux; Android 8.1.0; CPH1901 Build/OPM1.171019.026; wv) AppleWebKit/537.36 ' +
+      '(KHTML, like Gecko) Version/4.0 Chrome/91.0.4472.120 Mobile Safari/537.36 trill_200005 ' +
+      'JsSdk/1.0 NetType/WIFI Channel/googleplay AppName/trill app_version/20.0.5 ' +
+      'BytedanceWebview/d8a21c6',
+    isBot: false,
+  },
+  {
+    name: 'Googlebot is still a bot',
+    source: 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
+    isBot: true,
+    botName: 'googlebot',
+  },
+  {
+    name: 'Google Page Speed Insights is still a bot',
+    source:
+      'Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/537.36 ' +
+      '(KHTML, like Gecko; Google Page Speed Insights) Version/8.0 Mobile/12F70 Safari/600.1.4',
+    isBot: true,
+  },
+  {
+    name: 'GoogleOther crawler is a bot',
+    source: 'Mozilla/5.0 (compatible; GoogleOther)',
+    isBot: true,
+    botName: 'googleother',
+  },
+];
+
+// Pinterest's crawler sends "Pinterest/0.2"; its in-app browser sends
+// "[Pinterest/iOS]". The digit after the slash separates them.
+const pinterestCases: DetectionCase[] = [
+  {
+    name: 'Pinterest crawler',
+    source: 'Pinterest/0.2 (+https://www.pinterest.com/bot.html)',
+    isBot: true,
+    botName: 'pinterest',
+  },
+  {
+    name: 'Pinterestbot',
+    source: 'Mozilla/5.0 (compatible; Pinterestbot/1.0; +https://www.pinterest.com/bot.html)',
+    isBot: true,
+    botName: 'pinterestbot',
+  },
+  {
+    name: 'Pinterest in-app browser on iOS is not a bot',
+    source:
+      'Mozilla/5.0 (iPhone; CPU iPhone OS 17_1 like Mac OS X) AppleWebKit/605.1.15 ' +
+      '(KHTML, like Gecko) Mobile/15E148 [Pinterest/iOS]',
+    isBot: false,
+  },
+  {
+    name: 'Pinterest in-app browser on Android is not a bot',
+    source:
+      'Mozilla/5.0 (Linux; Android 13; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko) ' +
+      'Chrome/120.0.0.0 Mobile Safari/537.36 [Pinterest/Android]',
+    isBot: false,
+  },
+];
+
+const humanCases: DetectionCase[] = [
+  {
+    name: 'Chrome on Windows',
+    source:
+      'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) ' +
+      'Chrome/120.0.0.0 Safari/537.36',
+    isBot: false,
+  },
+  {
+    name: 'Chrome on Android',
+    source:
+      'Mozilla/5.0 (Linux; Android 13; SM-S918B) AppleWebKit/537.36 (KHTML, like Gecko) ' +
+      'Chrome/120.0.0.0 Mobile Safari/537.36',
+    isBot: false,
+  },
+  {
+    name: 'Safari on iPhone',
+    source:
+      'Mozilla/5.0 (iPhone; CPU iPhone OS 17_1 like Mac OS X) AppleWebKit/605.1.15 ' +
+      '(KHTML, like Gecko) Version/17.1 Mobile/15E148 Safari/604.1',
+    isBot: false,
+  },
+];
+
+describe.each([
+  ['Headless browsers and scrapers', headlessAndScraperCases],
+  ['Google crawlers versus Google-adjacent apps', googleCases],
+  ['Pinterest crawler versus in-app browser', pinterestCases],
+  ['Ordinary browsers', humanCases],
+])('%s', (_group, groupCases) => {
+  groupCases.forEach(({ name, source, isBot, botName }) => {
+    it(name, () => {
+      const agent = useragent.parse(source);
+      expect(agent.isBot).toBe(isBot);
+      if (botName !== undefined) {
+        expect(agent.botName).toBe(botName);
+      }
+      if (!isBot) {
+        expect(agent.botName).toBe('');
+      }
+    });
+  });
+});