From e522eb719724812a31911deb5368ab99d1cbb21b Mon Sep 17 00:00:00 2001 From: Aleksejs Gordejevs Date: Sat, 22 Aug 2026 10:05:52 +0300 Subject: [PATCH] fix: separate crawlers from apps that merely name the vendor Three bot-detection defects, each verified by a test that fails without the corresponding change. Headless Chrome went undetected. The list carried "chromeheadless", which is the Karma launcher label; Chrome itself reports "HeadlessChrome/". Scrapy was absent entirely. The bare "google" pattern matched any user-agent carrying "Channel/googleplay" - that is, the WebView of any app shipped through the Play Store. TikTok already had a hard-coded exemption for exactly this, which covered one app and left the rest of the class flagged as crawlers. The pattern is replaced by the documented Google crawler tokens, and the exemption is removed as unreachable. A side effect: botName for Googlebot is now "googlebot" rather than "google". "pinterest" matched both the crawler and the in-app browser. The crawler sends "Pinterest/0.2", the browser sends "[Pinterest/iOS]"; a lookahead on the digit separates them. CONTRIBUTING advised using "google" over "Googlebot", which is the guidance that produced the first false positive, so that bullet is rewritten. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 12 ++++ CONTRIBUTING.md | 6 +- src/express-useragent.ts | 26 +++---- tests/bots.test.ts | 142 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 173 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 432b31f..e58ca55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## Unreleased + +- Detect headless Chrome. The list carried `chromeheadless`, which is the Karma + launcher label; Chrome itself reports `HeadlessChrome/`. +- Detect Scrapy. +- Stop flagging Play Store WebViews as bots. The bare `google` pattern matched any + user-agent carrying `Channel/googleplay`, which is why TikTok needed a hard-coded + exemption; the pattern is now the documented Google crawler tokens and the + exemption is gone. `botName` for Googlebot is now `googlebot` rather than `google`. +- Distinguish the Pinterest crawler (`Pinterest/0.2`) from the Pinterest in-app + browser (`[Pinterest/iOS]`), which was reported as a bot. + ## 2.2.1 - 2026-06-30 - Updated dependency metadata and lockfile entries for recent transitive security advisories. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4c9eb55..11ffee7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -23,7 +23,11 @@ const BOTS = [ Add the new bot pattern to the `BOTS` array. Follow these best practices: -- **Use regex patterns**, not exact matches (e.g., use `'google'` instead of `'Googlebot'`) +- **Match the crawler, not the vendor.** A pattern must not match a browser that merely + mentions the vendor. `'google'` once matched every Play Store WebView carrying + `Channel/googleplay`, and `'pinterest'` matched the Pinterest in-app browser. Prefer the + crawler token (`'googleother'`, `'pinterestbot'`), or a lookahead that pins the shape the + crawler uses (`'pinterest(?=\\/\\d)'` matches `Pinterest/0.2` but not `[Pinterest/iOS]`) - **Handle variations** - include common variations (e.g., both `'phantom\\.js'` and `'phantomjs'`) - **Escape special characters** properly (dots, spaces, etc.) - **Keep patterns simple** - avoid overly complex regex that might miss variations diff --git a/src/express-useragent.ts b/src/express-useragent.ts index 473e624..ff21909 100644 --- a/src/express-useragent.ts +++ b/src/express-useragent.ts @@ -34,6 +34,7 @@ const BOTS = [ 'baiduspider', 'bingbot', 'chromeheadless', + 'headlesschrome', 'cloudflare', 'cloudinary', 'crawler', @@ -45,7 +46,16 @@ const BOTS = [ 'facebookexternalhit', 'facebot', 'flipboard', - 'google', + 'apis-google', + 'feedfetcher-google', + 'google\\spage\\sspeed', + 'google\\sweb\\spreview', + 'google-inspectiontool', + 'google-read-aloud', + 'google-site-verification', + 'googleother', + 'googleweblight', + 'storebot-google', 'googlebot', 'gsa-crawler', 'gurujibot', @@ -61,9 +71,11 @@ const BOTS = [ 'phantom\\.js', 'phantomjs', 'pingdom', - 'pinterest', + 'pinterest(?=\\/\\d)', + 'pinterestbot', 'python', 'rtlnieuws', + 'scrapy', 'skypeuripreview', 'slackbot', 'slurp', @@ -780,16 +792,6 @@ export class UserAgent { if (match) { const botIdentifier = match[1]; - // Handle false positives - TikTok WebView contains "googleplay" but isn't a bot - if ( - botIdentifier === 'google' && - (source.includes('tiktok') || source.includes('trill') || source.includes('bytedance')) - ) { - this.Agent.isBot = false; - this.Agent.botName = ''; - return; - } - // For all bots, return boolean true and store bot name (fixes issues #168, #138) this.Agent.isBot = true; this.Agent.botName = botIdentifier; diff --git a/tests/bots.test.ts b/tests/bots.test.ts index 32aeb47..9aa9f93 100644 --- a/tests/bots.test.ts +++ b/tests/bots.test.ts @@ -470,3 +470,145 @@ describe('Failing Bot detection cases', () => { }); }); }); + +interface DetectionCase { + name: string; + source: string; + isBot: boolean; + botName?: string; +} + +// Headless Chrome reports "HeadlessChrome/"; "chromeheadless" is the +// Karma launcher label and never appears in a user-agent string. +const headlessAndScraperCases: DetectionCase[] = [ + { + name: 'Headless Chrome', + source: + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/120.0.0.0 Safari/537.36', + isBot: true, + botName: 'headlesschrome', + }, + { + name: 'Scrapy', + source: 'Scrapy/2.11 (+https://scrapy.org)', + isBot: true, + botName: 'scrapy', + }, +]; + +// "google" as a bare substring matched every Play Store WebView that carries +// "Channel/googleplay". TikTok was special-cased for exactly this reason; the +// same bug hit every other app shipping that token. +const googleCases: DetectionCase[] = [ + { + name: 'Play Store WebView of an unrelated app is not a bot', + source: + 'Mozilla/5.0 (Linux; Android 12; RMX3085 Build/SP1A.210812.016; wv) AppleWebKit/537.36 ' + + '(KHTML, like Gecko) Version/4.0 Chrome/107.0.5304.105 Mobile Safari/537.36 JsSdk/1.0 ' + + 'NetType/WIFI Channel/googleplay AppName/shopmate app_version/5.2.1', + isBot: false, + }, + { + name: 'TikTok WebView stays human without the special case', + source: + 'Mozilla/5.0 (Linux; Android 8.1.0; CPH1901 Build/OPM1.171019.026; wv) AppleWebKit/537.36 ' + + '(KHTML, like Gecko) Version/4.0 Chrome/91.0.4472.120 Mobile Safari/537.36 trill_200005 ' + + 'JsSdk/1.0 NetType/WIFI Channel/googleplay AppName/trill app_version/20.0.5 ' + + 'BytedanceWebview/d8a21c6', + isBot: false, + }, + { + name: 'Googlebot is still a bot', + source: 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)', + isBot: true, + botName: 'googlebot', + }, + { + name: 'Google Page Speed Insights is still a bot', + source: + 'Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/537.36 ' + + '(KHTML, like Gecko; Google Page Speed Insights) Version/8.0 Mobile/12F70 Safari/600.1.4', + isBot: true, + }, + { + name: 'GoogleOther crawler is a bot', + source: 'Mozilla/5.0 (compatible; GoogleOther)', + isBot: true, + botName: 'googleother', + }, +]; + +// Pinterest's crawler sends "Pinterest/0.2"; its in-app browser sends +// "[Pinterest/iOS]". The digit after the slash separates them. +const pinterestCases: DetectionCase[] = [ + { + name: 'Pinterest crawler', + source: 'Pinterest/0.2 (+https://www.pinterest.com/bot.html)', + isBot: true, + botName: 'pinterest', + }, + { + name: 'Pinterestbot', + source: 'Mozilla/5.0 (compatible; Pinterestbot/1.0; +https://www.pinterest.com/bot.html)', + isBot: true, + botName: 'pinterestbot', + }, + { + name: 'Pinterest in-app browser on iOS is not a bot', + source: + 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_1 like Mac OS X) AppleWebKit/605.1.15 ' + + '(KHTML, like Gecko) Mobile/15E148 [Pinterest/iOS]', + isBot: false, + }, + { + name: 'Pinterest in-app browser on Android is not a bot', + source: + 'Mozilla/5.0 (Linux; Android 13; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko) ' + + 'Chrome/120.0.0.0 Mobile Safari/537.36 [Pinterest/Android]', + isBot: false, + }, +]; + +const humanCases: DetectionCase[] = [ + { + name: 'Chrome on Windows', + source: + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) ' + + 'Chrome/120.0.0.0 Safari/537.36', + isBot: false, + }, + { + name: 'Chrome on Android', + source: + 'Mozilla/5.0 (Linux; Android 13; SM-S918B) AppleWebKit/537.36 (KHTML, like Gecko) ' + + 'Chrome/120.0.0.0 Mobile Safari/537.36', + isBot: false, + }, + { + name: 'Safari on iPhone', + source: + 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_1 like Mac OS X) AppleWebKit/605.1.15 ' + + '(KHTML, like Gecko) Version/17.1 Mobile/15E148 Safari/604.1', + isBot: false, + }, +]; + +describe.each([ + ['Headless browsers and scrapers', headlessAndScraperCases], + ['Google crawlers versus Google-adjacent apps', googleCases], + ['Pinterest crawler versus in-app browser', pinterestCases], + ['Ordinary browsers', humanCases], +])('%s', (_group, groupCases) => { + groupCases.forEach(({ name, source, isBot, botName }) => { + it(name, () => { + const agent = useragent.parse(source); + expect(agent.isBot).toBe(isBot); + if (botName !== undefined) { + expect(agent.botName).toBe(botName); + } + if (!isBot) { + expect(agent.botName).toBe(''); + } + }); + }); +});