Skip to content

Commit 06de36f

Browse files
chiycchiyc
committed
Squashed commit of the following:
commit a22d888b1568c5a41fdd7c5b6615f465918320e4 Author: Stefan Valentin <[email protected]> Date: Mon Jan 8 16:53:02 2024 +0000 PR feedback cleanup commit 221b136e738b06e9c97410c3f16ff075a3a9a33c Author: Stefan Valentin <[email protected]> Date: Thu Dec 14 14:38:02 2023 -0500 Move isBlockedUA tests to unit tests commit c040f52d4a03705f72d7c3b8719f54e0abe3f7b1 Author: Stefan Valentin <[email protected]> Date: Wed Nov 22 11:17:16 2023 -0500 Revert test file change commit ca8b3feeadb34242e6ca052dcc31fc1a58a86035 Author: Stefan Valentin <[email protected]> Date: Tue Nov 21 20:27:43 2023 +0000 Update built files commit 58352b7 Author: Stefan Valentin <[email protected]> Date: Tue Nov 21 15:11:02 2023 -0500 Ignore AhrefsSiteAudit Crawler - Fix integrations tests.html
1 parent 038d2fc commit 06de36f

File tree

12 files changed

+46
-35
lines changed

12 files changed

+46
-35
lines changed

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -8,3 +8,4 @@ tunnel.log
88
*.swp
99
*.ps1
1010
*.bundle.js
11+
.DS_Store

‎dist/mixpanel.amd.js

+1
Original file line numberDiff line numberDiff line change
@@ -901,6 +901,7 @@ define(function () { 'use strict';
901901
// sending false tracking data
902902
var BLOCKED_UA_STRS = [
903903
'ahrefsbot',
904+
'ahrefssiteaudit',
904905
'baiduspider',
905906
'bingbot',
906907
'bingpreview',

‎dist/mixpanel.cjs.js

+1
Original file line numberDiff line numberDiff line change
@@ -901,6 +901,7 @@ _.UUID = (function() {
901901
// sending false tracking data
902902
var BLOCKED_UA_STRS = [
903903
'ahrefsbot',
904+
'ahrefssiteaudit',
904905
'baiduspider',
905906
'bingbot',
906907
'bingpreview',

‎dist/mixpanel.globals.js

+1
Original file line numberDiff line numberDiff line change
@@ -902,6 +902,7 @@
902902
// sending false tracking data
903903
var BLOCKED_UA_STRS = [
904904
'ahrefsbot',
905+
'ahrefssiteaudit',
905906
'baiduspider',
906907
'bingbot',
907908
'bingpreview',

‎dist/mixpanel.min.js

+1-1
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎dist/mixpanel.umd.js

+1
Original file line numberDiff line numberDiff line change
@@ -905,6 +905,7 @@
905905
// sending false tracking data
906906
var BLOCKED_UA_STRS = [
907907
'ahrefsbot',
908+
'ahrefssiteaudit',
908909
'baiduspider',
909910
'bingbot',
910911
'bingpreview',

‎examples/commonjs-browserify/bundle.js

+1
Original file line numberDiff line numberDiff line change
@@ -915,6 +915,7 @@ _.UUID = (function() {
915915
// sending false tracking data
916916
var BLOCKED_UA_STRS = [
917917
'ahrefsbot',
918+
'ahrefssiteaudit',
918919
'baiduspider',
919920
'bingbot',
920921
'bingpreview',

‎examples/es2015-babelify/bundle.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -5665,7 +5665,7 @@ _.UUID = (function () {
56655665
// _.isBlockedUA()
56665666
// This is to block various web spiders from executing our JS and
56675667
// sending false tracking data
5668-
var BLOCKED_UA_STRS = ['ahrefsbot', 'baiduspider', 'bingbot', 'bingpreview', 'chrome-lighthouse', 'facebookexternal', 'petalbot', 'pinterest', 'screaming frog', 'yahoo! slurp', 'yandexbot',
5668+
var BLOCKED_UA_STRS = ['ahrefsbot', 'ahrefssiteaudit', 'baiduspider', 'bingbot', 'bingpreview', 'chrome-lighthouse', 'facebookexternal', 'petalbot', 'pinterest', 'screaming frog', 'yahoo! slurp', 'yandexbot',
56695669

56705670
// a whole bunch of goog-specific crawlers
56715671
// https://developers.google.com/search/docs/advanced/crawling/overview-google-crawlers

‎examples/umd-webpack/bundle.js

+1
Original file line numberDiff line numberDiff line change
@@ -968,6 +968,7 @@
968968
// sending false tracking data
969969
var BLOCKED_UA_STRS = [
970970
'ahrefsbot',
971+
'ahrefssiteaudit',
971972
'baiduspider',
972973
'bingbot',
973974
'bingpreview',

‎src/utils.js

+1
Original file line numberDiff line numberDiff line change
@@ -899,6 +899,7 @@ _.UUID = (function() {
899899
// sending false tracking data
900900
var BLOCKED_UA_STRS = [
901901
'ahrefsbot',
902+
'ahrefssiteaudit',
902903
'baiduspider',
903904
'bingbot',
904905
'bingpreview',

‎tests/test.js

+1-33
Original file line numberDiff line numberDiff line change
@@ -3479,39 +3479,7 @@
34793479
same(mixpanel._.info.browserVersion(a), 5.2);
34803480
});
34813481

3482-
test('blocked user agents', function() {
3483-
var bot_user_agents = [
3484-
"Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)",
3485-
"Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)",
3486-
"Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)",
3487-
"Mozilla/5.0 (Windows Phone 8.1; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 530) like Gecko (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)",
3488-
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534+ (KHTML, like Gecko) BingPreview/1.0b",
3489-
"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
3490-
"Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)",
3491-
"Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)",
3492-
"facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)",
3493-
"facebookexternalhit/1.1",
3494-
"Mozilla/5.0 (compatible;PetalBot;+http://aspiegel.com/petalbot)",
3495-
"Mozilla/5.0(Linux;Android7.0;) AppleWebKit/537.36(KHTML,likeGecko) MobileSafari/537.36(compatible;PetalBot;+http://aspiegel.com/petalbot)",
3496-
"Mozilla/5.0 (compatible; Pinterestbot/1.0; +http://www.pinterest.com/bot.html)",
3497-
"APIs-Google (+https://developers.google.com/webmasters/APIs-Google.html)",
3498-
"Mediapartners-Google",
3499-
"Mozilla/5.0 (Linux; Android 5.0; SM-G920A) AppleWebKit (KHTML, like Gecko) Chrome Mobile Safari (compatible; AdsBot-Google-Mobile; +http://www.google.com/mobile/adsbot.html)",
3500-
"FeedFetcher-Google; (+http://www.google.com/feedfetcher.html)",
3501-
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 Safari/537.36 (compatible; Google-Read-Aloud; +/search/docs/advanced/crawling/overview-google-crawlers)",
3502-
"Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012; DuplexWeb-Google/1.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Mobile Safari/537.36",
3503-
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36 Google Favicon",
3504-
"Mozilla/5.0 (Linux; Android 4.2.1; en-us; Nexus 5 Build/JOP40D) AppleWebKit/535.19 (KHTML, like Gecko; googleweblight) Chrome/38.0.1025.166 Mobile Safari/535.19",
3505-
"Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012; Storebot-Google/1.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Mobile Safari/537.36",
3506-
"Screaming Frog SEO Spider/12.3",
3507-
"Mozilla/5.0 (Linux; Android 7.0; Moto G (4)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4695.0 Mobile Safari/537.36 Chrome-Lighthouse"
3508-
];
3509-
_.each(bot_user_agents, function(ua) {
3510-
ok(mixpanel._.isBlockedUA(ua), ua);
3511-
});
3512-
});
3513-
3514-
mpmodule("mixpanel.reset");
3482+
mpmodule('mixpanel.reset');
35153483

35163484
test('reset generates new distinct_id', 2, function() {
35173485
var id = '1234';

‎tests/unit/utils.js

+35
Original file line numberDiff line numberDiff line change
@@ -202,3 +202,38 @@ describe(`_.info helper methods`, function() {
202202
});
203203
});
204204
});
205+
206+
describe('_.isBlockedUA', function() {
207+
[
208+
'Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)',
209+
'Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)',
210+
'Mozilla/5.0 (compatible; AhrefsSiteAudit/6.1; +http://ahrefs.com/robot/site-audit)', // Desktop
211+
'Mozilla/5.0 (Linux; Android 13) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.5359.128 Mobile Safari/537.36 (compatible; AhrefsSiteAudit/6.1; +http://ahrefs.com/robot/site-audit)', // Mobile
212+
'Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)',
213+
'Mozilla/5.0 (Windows Phone 8.1; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 530) like Gecko (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)',
214+
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534+ (KHTML, like Gecko) BingPreview/1.0b',
215+
'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
216+
'Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)',
217+
'Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)',
218+
'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)',
219+
'facebookexternalhit/1.1',
220+
'Mozilla/5.0 (compatible;PetalBot;+http://aspiegel.com/petalbot)',
221+
'Mozilla/5.0(Linux;Android7.0;) AppleWebKit/537.36(KHTML,likeGecko) MobileSafari/537.36(compatible;PetalBot;+http://aspiegel.com/petalbot)',
222+
'Mozilla/5.0 (compatible; Pinterestbot/1.0; +http://www.pinterest.com/bot.html)',
223+
'APIs-Google (+https://developers.google.com/webmasters/APIs-Google.html)',
224+
'Mediapartners-Google',
225+
'Mozilla/5.0 (Linux; Android 5.0; SM-G920A) AppleWebKit (KHTML, like Gecko) Chrome Mobile Safari (compatible; AdsBot-Google-Mobile; +http://www.google.com/mobile/adsbot.html)',
226+
'FeedFetcher-Google; (+http://www.google.com/feedfetcher.html)',
227+
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 Safari/537.36 (compatible; Google-Read-Aloud; +/search/docs/advanced/crawling/overview-google-crawlers)',
228+
'Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012; DuplexWeb-Google/1.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Mobile Safari/537.36',
229+
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36 Google Favicon',
230+
'Mozilla/5.0 (Linux; Android 4.2.1; en-us; Nexus 5 Build/JOP40D) AppleWebKit/535.19 (KHTML, like Gecko; googleweblight) Chrome/38.0.1025.166 Mobile Safari/535.19',
231+
'Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012; Storebot-Google/1.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Mobile Safari/537.36',
232+
'Screaming Frog SEO Spider/12.3',
233+
'Mozilla/5.0 (Linux; Android 7.0; Moto G (4)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4695.0 Mobile Safari/537.36 Chrome-Lighthouse',
234+
].forEach((ua) => {
235+
it(`should block bot user agent: ${ua}`, () => {
236+
expect(_.isBlockedUA(ua)).to.be.true;
237+
});
238+
});
239+
});

0 commit comments

Comments
 (0)