diff --git a/CHANGELOG.md b/CHANGELOG.md index 432b31f..e58ca55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## Unreleased + +- Detect headless Chrome. The list carried `chromeheadless`, which is the Karma + launcher label; Chrome itself reports `HeadlessChrome/`. +- Detect Scrapy. +- Stop flagging Play Store WebViews as bots. The bare `google` pattern matched any + user-agent carrying `Channel/googleplay`, which is why TikTok needed a hard-coded + exemption; the pattern is now the documented Google crawler tokens and the + exemption is gone. `botName` for Googlebot is now `googlebot` rather than `google`. +- Distinguish the Pinterest crawler (`Pinterest/0.2`) from the Pinterest in-app + browser (`[Pinterest/iOS]`), which was reported as a bot. + ## 2.2.1 - 2026-06-30 - Updated dependency metadata and lockfile entries for recent transitive security advisories. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4c9eb55..11ffee7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -23,7 +23,11 @@ const BOTS = [ Add the new bot pattern to the `BOTS` array. Follow these best practices: -- **Use regex patterns**, not exact matches (e.g., use `'google'` instead of `'Googlebot'`) +- **Match the crawler, not the vendor.** A pattern must not match a browser that merely + mentions the vendor. `'google'` once matched every Play Store WebView carrying + `Channel/googleplay`, and `'pinterest'` matched the Pinterest in-app browser. Prefer the + crawler token (`'googleother'`, `'pinterestbot'`), or a lookahead that pins the shape the + crawler uses (`'pinterest(?=\\/\\d)'` matches `Pinterest/0.2` but not `[Pinterest/iOS]`) - **Handle variations** - include common variations (e.g., both `'phantom\\.js'` and `'phantomjs'`) - **Escape special characters** properly (dots, spaces, etc.) - **Keep patterns simple** - avoid overly complex regex that might miss variations diff --git a/src/express-useragent.ts b/src/express-useragent.ts index 473e624..ff21909 100644 --- a/src/express-useragent.ts +++ b/src/express-useragent.ts @@ -34,6 +34,7 @@ const BOTS = [ 'baiduspider', 'bingbot', 'chromeheadless', + 'headlesschrome', 'cloudflare', 'cloudinary', 'crawler', @@ -45,7 +46,16 @@ const BOTS = [ 'facebookexternalhit', 'facebot', 'flipboard', - 'google', + 'apis-google', + 'feedfetcher-google', + 'google\\spage\\sspeed', + 'google\\sweb\\spreview', + 'google-inspectiontool', + 'google-read-aloud', + 'google-site-verification', + 'googleother', + 'googleweblight', + 'storebot-google', 'googlebot', 'gsa-crawler', 'gurujibot', @@ -61,9 +71,11 @@ const BOTS = [ 'phantom\\.js', 'phantomjs', 'pingdom', - 'pinterest', + 'pinterest(?=\\/\\d)', + 'pinterestbot', 'python', 'rtlnieuws', + 'scrapy', 'skypeuripreview', 'slackbot', 'slurp', @@ -780,16 +792,6 @@ export class UserAgent { if (match) { const botIdentifier = match[1]; - // Handle false positives - TikTok WebView contains "googleplay" but isn't a bot - if ( - botIdentifier === 'google' && - (source.includes('tiktok') || source.includes('trill') || source.includes('bytedance')) - ) { - this.Agent.isBot = false; - this.Agent.botName = ''; - return; - } - // For all bots, return boolean true and store bot name (fixes issues #168, #138) this.Agent.isBot = true; this.Agent.botName = botIdentifier; diff --git a/tests/bots.test.ts b/tests/bots.test.ts index 32aeb47..9aa9f93 100644 --- a/tests/bots.test.ts +++ b/tests/bots.test.ts @@ -470,3 +470,145 @@ describe('Failing Bot detection cases', () => { }); }); }); + +interface DetectionCase { + name: string; + source: string; + isBot: boolean; + botName?: string; +} + +// Headless Chrome reports "HeadlessChrome/"; "chromeheadless" is the +// Karma launcher label and never appears in a user-agent string. +const headlessAndScraperCases: DetectionCase[] = [ + { + name: 'Headless Chrome', + source: + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/120.0.0.0 Safari/537.36', + isBot: true, + botName: 'headlesschrome', + }, + { + name: 'Scrapy', + source: 'Scrapy/2.11 (+https://scrapy.org)', + isBot: true, + botName: 'scrapy', + }, +]; + +// "google" as a bare substring matched every Play Store WebView that carries +// "Channel/googleplay". TikTok was special-cased for exactly this reason; the +// same bug hit every other app shipping that token. +const googleCases: DetectionCase[] = [ + { + name: 'Play Store WebView of an unrelated app is not a bot', + source: + 'Mozilla/5.0 (Linux; Android 12; RMX3085 Build/SP1A.210812.016; wv) AppleWebKit/537.36 ' + + '(KHTML, like Gecko) Version/4.0 Chrome/107.0.5304.105 Mobile Safari/537.36 JsSdk/1.0 ' + + 'NetType/WIFI Channel/googleplay AppName/shopmate app_version/5.2.1', + isBot: false, + }, + { + name: 'TikTok WebView stays human without the special case', + source: + 'Mozilla/5.0 (Linux; Android 8.1.0; CPH1901 Build/OPM1.171019.026; wv) AppleWebKit/537.36 ' + + '(KHTML, like Gecko) Version/4.0 Chrome/91.0.4472.120 Mobile Safari/537.36 trill_200005 ' + + 'JsSdk/1.0 NetType/WIFI Channel/googleplay AppName/trill app_version/20.0.5 ' + + 'BytedanceWebview/d8a21c6', + isBot: false, + }, + { + name: 'Googlebot is still a bot', + source: 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)', + isBot: true, + botName: 'googlebot', + }, + { + name: 'Google Page Speed Insights is still a bot', + source: + 'Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/537.36 ' + + '(KHTML, like Gecko; Google Page Speed Insights) Version/8.0 Mobile/12F70 Safari/600.1.4', + isBot: true, + }, + { + name: 'GoogleOther crawler is a bot', + source: 'Mozilla/5.0 (compatible; GoogleOther)', + isBot: true, + botName: 'googleother', + }, +]; + +// Pinterest's crawler sends "Pinterest/0.2"; its in-app browser sends +// "[Pinterest/iOS]". The digit after the slash separates them. +const pinterestCases: DetectionCase[] = [ + { + name: 'Pinterest crawler', + source: 'Pinterest/0.2 (+https://www.pinterest.com/bot.html)', + isBot: true, + botName: 'pinterest', + }, + { + name: 'Pinterestbot', + source: 'Mozilla/5.0 (compatible; Pinterestbot/1.0; +https://www.pinterest.com/bot.html)', + isBot: true, + botName: 'pinterestbot', + }, + { + name: 'Pinterest in-app browser on iOS is not a bot', + source: + 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_1 like Mac OS X) AppleWebKit/605.1.15 ' + + '(KHTML, like Gecko) Mobile/15E148 [Pinterest/iOS]', + isBot: false, + }, + { + name: 'Pinterest in-app browser on Android is not a bot', + source: + 'Mozilla/5.0 (Linux; Android 13; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko) ' + + 'Chrome/120.0.0.0 Mobile Safari/537.36 [Pinterest/Android]', + isBot: false, + }, +]; + +const humanCases: DetectionCase[] = [ + { + name: 'Chrome on Windows', + source: + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) ' + + 'Chrome/120.0.0.0 Safari/537.36', + isBot: false, + }, + { + name: 'Chrome on Android', + source: + 'Mozilla/5.0 (Linux; Android 13; SM-S918B) AppleWebKit/537.36 (KHTML, like Gecko) ' + + 'Chrome/120.0.0.0 Mobile Safari/537.36', + isBot: false, + }, + { + name: 'Safari on iPhone', + source: + 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_1 like Mac OS X) AppleWebKit/605.1.15 ' + + '(KHTML, like Gecko) Version/17.1 Mobile/15E148 Safari/604.1', + isBot: false, + }, +]; + +describe.each([ + ['Headless browsers and scrapers', headlessAndScraperCases], + ['Google crawlers versus Google-adjacent apps', googleCases], + ['Pinterest crawler versus in-app browser', pinterestCases], + ['Ordinary browsers', humanCases], +])('%s', (_group, groupCases) => { + groupCases.forEach(({ name, source, isBot, botName }) => { + it(name, () => { + const agent = useragent.parse(source); + expect(agent.isBot).toBe(isBot); + if (botName !== undefined) { + expect(agent.botName).toBe(botName); + } + if (!isBot) { + expect(agent.botName).toBe(''); + } + }); + }); +});