Update pattern (#248)

omrilotan · Mar 18, 2024 · c123374 · c123374
1 parent b489f2c
commit c123374
Show file tree

Hide file tree

Showing 5 changed files with 23 additions and 4 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,10 @@
 # Changelog
 
+## [5.1.2](https://github.com/omrilotan/isbot/compare/v5.1.1...v5.1.2)
+
+- Add bots: Cypress, Detectify, InternetMeasurement, BuiltWith
+- Recognise browser: Zip Recruiter job search app, Ecosia android in-app browser
+
 ## [5.1.1](https://github.com/omrilotan/isbot/compare/v5.1.0...v5.1.1)
 
 - Reduce pattern size by introducing the substring ".com" and improve generic pattern

diff --git a/fixtures/browsers.yml b/fixtures/browsers.yml
@@ -159,6 +159,9 @@ DuckDuckGo:
   - Mozilla/5.0 (Linux; Android 10) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/84.0.4147.111 Mobile DuckDuckGo/5 Safari/537.36
   - Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 DuckDuckGo/7
   - Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 DuckDuckGo/7
+Ecosia:
+  - Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Mobile Safari/537.36 (Ecosia [email protected])
+  - Mozilla/5.0 (Linux; Android 8.0.0; Moto Z2 Play) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.41 Mobile Safari/537.36 (Ecosia [email protected])
 Edge:
   - Mozilla/5.0 (iPhone; CPU iPhone OS 13_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0 EdgiOS/44.10.19 Mobile/15E148 Safari/605.1.15
   - Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.92 Safari/537.36 Edg/81.0.416.45
@@ -176,7 +179,6 @@ Electron Application:
   - Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Atom/1.45.0 Chrome/69.0.3497.128 Electron/4.2.7 Safari/537.36
   - Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) discord/0.0.306 Chrome/78.0.3904.130 Electron/7.1.11 Safari/537.36
   - Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) electron/1.0.0 Chrome/53.0.2785.113 Electron/1.4.3 Safari/537.36
-  - Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Cypress/3.3.2 Chrome/61.0.3163.100 Electron/2.0.18 Safari/537.36
 Elinks:
   - ELinks/0.12~pre6-1ubuntu1 (textmode; Ubuntu; Linux 3.11.0-13-generic i686; 100x25-2)
 "ESPN: Sports News & Highlights App":
@@ -677,6 +679,9 @@ Yandex:
   - Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 YaBrowser/17.3.1.873 (beta) Yowser/2.5 Safari/537.36
 Yandex Browser:
   - Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.95 YaBrowser/13.10.1500.9323 Safari/537.36
+Zip Recruiter (Job Search phone app):
+  - Job Search/24.4.0 Mozilla/5.0 (Linux; Android 14; SM-S918U Build/UP1A.231005.007; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/122.0.6261.119 Mobile Safari/537.36
+  - Job Search/95.0 (iPhone; CPU iOS 17_3_1 like Mac OS X)
 ZZZ Glitches and Misidentified Browsers - These browsers are legit user agent even though they are included in some bot listings (not all bots identify themselves):
   - (iPhone; CPU iPhone OS 16_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.4 Mobile/15E148 DuckDuckGo/7 Safari/605.1.15
   - ; Android 11) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/96.0.4664.45 Mobile DuckDuckGo/5 Safari/537.36
@@ -695,5 +700,7 @@ ZZZ Glitches and Misidentified Browsers - These browsers are legit user agent ev
   - User-Agent:Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.74 Safari/537.36 Edg/90.0.818.62
   - User-Agent:Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0 Safari /537.36
 ZZZ Insignificat bots - These bots have very low appearance rate and are not worth blocking:
+  - Mozilla/5.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322) 360JK yunjiankong 427691
   - Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; Banca Caboto s.p.a.)
+  - Mozilla/5.0 (Unknown; Linux x86_64) AppleWebKit/538.1 (KHTML, like Gecko) HLB/1.247
   - Opera/9.70 (Linux armv7l ; U; turbotabbee/TSV2.0/1.02Q; fr) Presto/2.2
diff --git a/fixtures/crawlers.yml b/fixtures/crawlers.yml
@@ -109,6 +109,9 @@ BUbiNG:
   - BUbiNG (+http://law.di.unimi.it/BUbiNG.html)
 Buck:
   - Buck/2.2; (+https://app.hypefactors.com/media-monitoring/about.html)
+BuiltWith:
+  - BW/1.1; bit.ly/3eZNDnO
+  - BW/1.1; rb.gy/oupwis
 Burp Collaborator Scanner:
   - Godzilla/17.0 (Unknown Operator; Nexus 5X Build/MMB29P) NoWebKit/5.36 (DOM, like Rhino; n7vbji2k0wt3rsbev55blgx17sdk8lwek97zvo.burpcollaborator.net) TOR/540.5.35487 Torrent 2654.76
   - http://5iojs2zo8rkhlrrwmm7oyl75twztnlo9ex9kz8o.burpcollaborator.net/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36

diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "isbot",
-  "version": "5.1.1",
+  "version": "5.1.2",
   "description": "🤖/👨‍🦰 Recognise bots/crawlers/spiders using the user agent string.",
   "keywords": [
     "bot",

diff --git a/src/patterns.json b/src/patterns.json
@@ -5,10 +5,10 @@
   "(?:^|[^g])news",
   "(?<! (?:channel/|google/))google(?!(app|/google| pixel))",
   "(?<! cu)bot(?:[^\\w]|_|$)",
-  "(?<! ya(?:yandex)?)search",
+  "(?<!(?: ya| yandex|^job) ?)search",
   "(?<!(?:lib))http",
   "(?<![hg]m)score",
-  "@",
+  "(?<!android)@",
   "\\(\\)",
   "\\.com",
   "^12345",
@@ -22,6 +22,7 @@
   "^avsdevicesdk/",
   "^biglotron",
   "^bot",
+  "^bw/",
   "^clamav[ /]",
   "^client/",
   "^cobweb/",
@@ -89,9 +90,11 @@
   "classifier",
   "cloud",
   "crawl",
+  "cypress/",
   "dareboost",
   "datanyze",
   "dejaclick",
+  "detect",
   "dmbrowser",
   "download",
   "evc-batch/",
@@ -112,6 +115,7 @@
   "library",
   "mail\\.ru/",
   "manager",
+  "measure",
   "neustar wpm",
   "node",
   "nutch",