From 077cd1585293a4e0d352e0102122d8e1dd6fd7da Mon Sep 17 00:00:00 2001 From: omrilotan Date: Tue, 2 Apr 2024 16:02:19 +0300 Subject: [PATCH 1/3] Recognise search providers inapp browser --- CHANGELOG.md | 5 +++++ fixtures/browsers.yml | 7 ++++++- fixtures/crawlers.yml | 3 --- package.json | 2 +- src/patterns.json | 2 +- 5 files changed, 13 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index af207a1..a25bb5d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## [5.1.4](https://github.com/omrilotan/isbot/compare/v5.1.3...v5.1.4) + +- Recognise search providers inapp browsers +- Ignore Crosswalk project: An old project that is no longer maintained and has insignificant usage + ## [5.1.3](https://github.com/omrilotan/isbot/compare/v5.1.2...v5.1.3) - Recognise browsers: Ecosia ios in-app browser, Phantom in-app browser diff --git a/fixtures/browsers.yml b/fixtures/browsers.yml index 86fc9fb..135df62 100644 --- a/fixtures/browsers.yml +++ b/fixtures/browsers.yml @@ -365,7 +365,9 @@ Motorola Internet: - MOT-VE240/00.72 UP.Browser/7.2.7.5.548 (GUI) MMP/2.0 Novarra-Vision/8.0 Mozilla Android Components: - MozacFetch/49.0.20200702190156 -Naver Whale: +Naver: + - Mozilla/5.0 (iPhone; CPU iPhone OS 17_2_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Mobile/15E148 Safari/605.1 NAVER(inapp; search; 2000; 12.3.6; 14PRO) + - Mozilla/5.0 (Linux; Android 8.0.0; SM-N950N Build/R16NW; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/79.0.3945.88 Mobile Safari/537.36 NAVER(inapp; search; 1000; 11.8.4; 11) - Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.57 Whale/3.14.133.23 Safari/537.36 NCSA Mosaic: - NCSA_Mosaic/2.7b5 (X11;Linux 2.6.7 i686) libwww/2.12 modified @@ -703,6 +705,9 @@ ZZZ Glitches and Misidentified Browsers - These browsers are legit user agent ev - User-Agent:Mozilla/5.0 (Macintosh; Intel Mac OS X 11_3_1) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/92.0 Safari /535.7 - User-Agent:Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.74 Safari/537.36 Edg/90.0.818.62 - User-Agent:Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0 Safari /537.36 +ZZZ Insignificat bots - Crosswalk project (deprecated): + - Mozilla/5.0 (Linux; Android 11;SM-G9866N Build/PR1A.2007820.012; wv) AppleWebKit/537.36 (KHTML,linke Gecko) Version/4.0 Chrome/80.0.3987.163 Whale/1.0.0.0 Crosswalk/25.80.14.26 Mobile Safari/537.36 NAVER(inapp; search; 900; 11.2.5) + - Mozilla/5.0 (Linux; Android 12; SM-G975N Build/SP1A.210812.016; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/90.0.4430.232 Whale/1.0.0.0 Crosswalk/26.90.3.21 Mobile Safari/537.36 NAVER(inapp; search; 1010; 11.11.3) ZZZ Insignificat bots - These bots have very low appearance rate and are not worth blocking: - Mozilla/5.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322) 360JK yunjiankong 427691 - Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; Banca Caboto s.p.a.) diff --git a/fixtures/crawlers.yml b/fixtures/crawlers.yml index 3910a89..8e58d3b 100644 --- a/fixtures/crawlers.yml +++ b/fixtures/crawlers.yml @@ -470,9 +470,6 @@ Nagios check_http: - check_http/v1.5 (nagios-plugins 1.5) NalezenCzBot: - NalezenCzBot/1.0 (http://www.nalezen.cz/about-crawler) -Naver Search: - - Mozilla/5.0 (Linux; Android 12; SM-G975N Build/SP1A.210812.016; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/90.0.4430.232 Whale/1.0.0.0 Crosswalk/26.90.3.21 Mobile Safari/537.36 NAVER(inapp; search; 1010; 11.11.3) - - Mozilla/5.0 (Linux; Android 8.0.0; SM-N950N Build/R16NW; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/79.0.3945.88 Mobile Safari/537.36 NAVER(inapp; search; 1000; 11.8.4; 11) nbertaupete95: - Mozilla/5.0/Firefox/42.0 - nbertaupete95(at)gmail.com Netcraft Survey Bot: diff --git a/package.json b/package.json index d0af848..5b84695 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "isbot", - "version": "5.1.3", + "version": "5.1.4", "description": "🤖/👨‍🦰 Recognise bots/crawlers/spiders using the user agent string.", "keywords": [ "bot", diff --git a/src/patterns.json b/src/patterns.json index 52686f7..4a355bd 100644 --- a/src/patterns.json +++ b/src/patterns.json @@ -5,7 +5,7 @@ "(?:^|[^g])news", "(? Date: Tue, 2 Apr 2024 16:20:26 +0300 Subject: [PATCH 2/3] Pattern: analyzer --- CHANGELOG.md | 1 + fixtures/crawlers.yml | 2 ++ src/patterns.json | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a25bb5d..c2d9b47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - Recognise search providers inapp browsers - Ignore Crosswalk project: An old project that is no longer maintained and has insignificant usage +- PDRL Analyzer ## [5.1.3](https://github.com/omrilotan/isbot/compare/v5.1.2...v5.1.3) diff --git a/fixtures/crawlers.yml b/fixtures/crawlers.yml index 8e58d3b..fb0f049 100644 --- a/fixtures/crawlers.yml +++ b/fixtures/crawlers.yml @@ -894,6 +894,8 @@ ZoomBot (seozoom.it): - zoombot (linkbot 1.0 http://suite.seozoom.it/bot.html) ZumBot: - ZumBot/1.0 (ZUM Search; http://help.zum.com/inquiry) +PDRL: + - "pdrl.fm Analyzer / 1.0.0" ZZZ Miscellaneous Glitches and Errornous User Agent Strings: - "123" - diff --git a/src/patterns.json b/src/patterns.json index 4a355bd..b62647f 100644 --- a/src/patterns.json +++ b/src/patterns.json @@ -35,7 +35,6 @@ "^facebook", "^getright/", "^gozilla/", - "^hatena", "^hobbit", "^hotzonu", "^hwcdn/", @@ -76,6 +75,7 @@ "^zdm/\\d", "^zoom marketplace/", "^{{.*}}$", + "analyzer", "archive", "ask jeeves/teoma", "bit\\.ly/", From 0ceac1636b238e2f542fa0b73ccb9987541bd710 Mon Sep 17 00:00:00 2001 From: omrilotan Date: Tue, 2 Apr 2024 13:21:17 +0000 Subject: [PATCH 3/3] =?UTF-8?q?=F0=9F=98=8E=20Autofix?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fixtures/crawlers.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fixtures/crawlers.yml b/fixtures/crawlers.yml index fb0f049..b95076f 100644 --- a/fixtures/crawlers.yml +++ b/fixtures/crawlers.yml @@ -534,6 +534,8 @@ Pageburst: - Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; pageburst) Chrome/111.0.5563.146 Safari/537.36 PaperLiBot: - Mozilla/5.0 (compatible; PaperLiBot/2.1; http://support.paper.li/entries/20023257-what-is-paper-li) +PDRL: + - pdrl.fm Analyzer / 1.0.0 PerimeterX: - PerimeterX Integration Services PetalBot: @@ -894,8 +896,6 @@ ZoomBot (seozoom.it): - zoombot (linkbot 1.0 http://suite.seozoom.it/bot.html) ZumBot: - ZumBot/1.0 (ZUM Search; http://help.zum.com/inquiry) -PDRL: - - "pdrl.fm Analyzer / 1.0.0" ZZZ Miscellaneous Glitches and Errornous User Agent Strings: - "123" -