diff --git a/AUTO-CONFIGURATION.md b/AUTO-CONFIGURATION.md index 23c9e9b252c..d5714cfb62f 100755 --- a/AUTO-CONFIGURATION.md +++ b/AUTO-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2024.10.4835 +#### Version: V4.2024.10.4836 #### Bad Referrers Blocked: [7104](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [667](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [668](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/MANUAL-CONFIGURATION.md b/MANUAL-CONFIGURATION.md index f3c63076b71..7b79c41553a 100755 --- a/MANUAL-CONFIGURATION.md +++ b/MANUAL-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2024.10.4835 +#### Version: V4.2024.10.4836 #### Bad Referrers Blocked: [7104](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [667](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [668](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/README.md b/README.md index e64b9cadb7d..3e1f2541ad9 100755 --- a/README.md +++ b/README.md @@ -14,9 +14,9 @@ ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic even Fake Googlebots from ever reaching your web sites. [PLEASE SEE: Definition of Bad Bots](#define-bad-bots) _______________ -#### Version: V4.2024.10.4835 +#### Version: V4.2024.10.4836 #### Bad Referrers Blocked: [7104](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [667](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [668](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/_generator_lists/bad-user-agents.list b/_generator_lists/bad-user-agents.list index d677d0ba565..40001c1d6bf 100755 --- a/_generator_lists/bad-user-agents.list +++ b/_generator_lists/bad-user-agents.list @@ -1,4 +1,3 @@ -everyfeed-spider 01h4x.com 360Spider 404checker @@ -608,6 +607,7 @@ demandbase-bot domainsproject.org eCatch evc-batch +everyfeed-spider facebookscraper gopher heritrix diff --git a/_google_analytics_ghost_spam/README.md b/_google_analytics_ghost_spam/README.md index e3d7c82d9cd..ece9f70a324 100644 --- a/_google_analytics_ghost_spam/README.md +++ b/_google_analytics_ghost_spam/README.md @@ -3,9 +3,9 @@ # EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM _______________ -#### Version: V4.2024.10.4835 +#### Version: V4.2024.10.4836 #### Bad Referrers Blocked: [7104](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [667](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [668](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf index a3aaacfcd7a..b70423ac717 100755 --- a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf +++ b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2024.10.4835 -### Updated: Fri Oct 25 22:01:17 UTC 2024 +### Version: V4.2024.10.4836 +### Updated: Sat Oct 26 05:47:29 UTC 2024 ### Bad Referrer Count: 7104 -### Bad Bot Count: 667 +### Bad Bot Count: 668 ################################################### ### VERSION INFORMATION ## @@ -760,6 +760,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)domainsproject.org(?:\b)" 3; "~*(?:\b)eCatch(?:\b)" 3; "~*(?:\b)evc-batch(?:\b)" 3; + "~*(?:\b)everyfeed-spider(?:\b)" 3; "~*(?:\b)facebookscraper(?:\b)" 3; "~*(?:\b)gopher(?:\b)" 3; "~*(?:\b)heritrix(?:\b)" 3; diff --git a/conf.d/globalblacklist-testing-version.conf b/conf.d/globalblacklist-testing-version.conf index 7db0472c0eb..9fc805cb38d 100644 --- a/conf.d/globalblacklist-testing-version.conf +++ b/conf.d/globalblacklist-testing-version.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2024.10.4835 -### Updated: Fri Oct 25 22:01:18 UTC 2024 +### Version: V3.2024.10.4836 +### Updated: Sat Oct 26 05:47:29 UTC 2024 ### Bad Referrer Count: 7104 -### Bad Bot Count: 667 +### Bad Bot Count: 668 ################################################### ### VERSION INFORMATION ## @@ -757,6 +757,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)domainsproject.org(?:\b)" 3; "~*(?:\b)eCatch(?:\b)" 3; "~*(?:\b)evc-batch(?:\b)" 3; + "~*(?:\b)everyfeed-spider(?:\b)" 3; "~*(?:\b)facebookscraper(?:\b)" 3; "~*(?:\b)gopher(?:\b)" 3; "~*(?:\b)heritrix(?:\b)" 3; diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf index a3aaacfcd7a..b70423ac717 100644 --- a/conf.d/globalblacklist.conf +++ b/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2024.10.4835 -### Updated: Fri Oct 25 22:01:17 UTC 2024 +### Version: V4.2024.10.4836 +### Updated: Sat Oct 26 05:47:29 UTC 2024 ### Bad Referrer Count: 7104 -### Bad Bot Count: 667 +### Bad Bot Count: 668 ################################################### ### VERSION INFORMATION ## @@ -760,6 +760,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)domainsproject.org(?:\b)" 3; "~*(?:\b)eCatch(?:\b)" 3; "~*(?:\b)evc-batch(?:\b)" 3; + "~*(?:\b)everyfeed-spider(?:\b)" 3; "~*(?:\b)facebookscraper(?:\b)" 3; "~*(?:\b)gopher(?:\b)" 3; "~*(?:\b)heritrix(?:\b)" 3; diff --git a/dev-tools/_robots_input/robots-input.txt b/dev-tools/_robots_input/robots-input.txt index 568cbc36c2d..5bc63a04889 100644 --- a/dev-tools/_robots_input/robots-input.txt +++ b/dev-tools/_robots_input/robots-input.txt @@ -607,6 +607,7 @@ demandbase-bot domainsproject.org eCatch evc-batch +everyfeed-spider facebookscraper gopher heritrix diff --git a/dev-tools/buildnumber b/dev-tools/buildnumber index 11336821115..1bb19682327 100644 --- a/dev-tools/buildnumber +++ b/dev-tools/buildnumber @@ -1 +1 @@ -4835 +4836 diff --git a/dev-tools/globalblacklist-testing.template b/dev-tools/globalblacklist-testing.template index 7db0472c0eb..9fc805cb38d 100644 --- a/dev-tools/globalblacklist-testing.template +++ b/dev-tools/globalblacklist-testing.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2024.10.4835 -### Updated: Fri Oct 25 22:01:18 UTC 2024 +### Version: V3.2024.10.4836 +### Updated: Sat Oct 26 05:47:29 UTC 2024 ### Bad Referrer Count: 7104 -### Bad Bot Count: 667 +### Bad Bot Count: 668 ################################################### ### VERSION INFORMATION ## @@ -757,6 +757,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)domainsproject.org(?:\b)" 3; "~*(?:\b)eCatch(?:\b)" 3; "~*(?:\b)evc-batch(?:\b)" 3; + "~*(?:\b)everyfeed-spider(?:\b)" 3; "~*(?:\b)facebookscraper(?:\b)" 3; "~*(?:\b)gopher(?:\b)" 3; "~*(?:\b)heritrix(?:\b)" 3; diff --git a/dev-tools/globalblacklist.template b/dev-tools/globalblacklist.template index a3aaacfcd7a..b70423ac717 100755 --- a/dev-tools/globalblacklist.template +++ b/dev-tools/globalblacklist.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2024.10.4835 -### Updated: Fri Oct 25 22:01:17 UTC 2024 +### Version: V4.2024.10.4836 +### Updated: Sat Oct 26 05:47:29 UTC 2024 ### Bad Referrer Count: 7104 -### Bad Bot Count: 667 +### Bad Bot Count: 668 ################################################### ### VERSION INFORMATION ## @@ -760,6 +760,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)domainsproject.org(?:\b)" 3; "~*(?:\b)eCatch(?:\b)" 3; "~*(?:\b)evc-batch(?:\b)" 3; + "~*(?:\b)everyfeed-spider(?:\b)" 3; "~*(?:\b)facebookscraper(?:\b)" 3; "~*(?:\b)gopher(?:\b)" 3; "~*(?:\b)heritrix(?:\b)" 3; diff --git a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index 1242e9bb048..1dea6b79e9b 100644 --- a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -172.183.122.254 0; +172.171.255.193 0; diff --git a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf index a3aaacfcd7a..b70423ac717 100644 --- a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2024.10.4835 -### Updated: Fri Oct 25 22:01:17 UTC 2024 +### Version: V4.2024.10.4836 +### Updated: Sat Oct 26 05:47:29 UTC 2024 ### Bad Referrer Count: 7104 -### Bad Bot Count: 667 +### Bad Bot Count: 668 ################################################### ### VERSION INFORMATION ## @@ -760,6 +760,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)domainsproject.org(?:\b)" 3; "~*(?:\b)eCatch(?:\b)" 3; "~*(?:\b)evc-batch(?:\b)" 3; + "~*(?:\b)everyfeed-spider(?:\b)" 3; "~*(?:\b)facebookscraper(?:\b)" 3; "~*(?:\b)gopher(?:\b)" 3; "~*(?:\b)heritrix(?:\b)" 3; diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 0d8dca8c3cc..99575d41d90 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.48 1; +10.1.0.64 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index 5d111b732fb..0ddde472877 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.48 0; +10.1.0.64 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index a3aaacfcd7a..b70423ac717 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2024.10.4835 -### Updated: Fri Oct 25 22:01:17 UTC 2024 +### Version: V4.2024.10.4836 +### Updated: Sat Oct 26 05:47:29 UTC 2024 ### Bad Referrer Count: 7104 -### Bad Bot Count: 667 +### Bad Bot Count: 668 ################################################### ### VERSION INFORMATION ## @@ -760,6 +760,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)domainsproject.org(?:\b)" 3; "~*(?:\b)eCatch(?:\b)" 3; "~*(?:\b)evc-batch(?:\b)" 3; + "~*(?:\b)everyfeed-spider(?:\b)" 3; "~*(?:\b)facebookscraper(?:\b)" 3; "~*(?:\b)gopher(?:\b)" 3; "~*(?:\b)heritrix(?:\b)" 3; diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 0d8dca8c3cc..99575d41d90 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.48 1; +10.1.0.64 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index 5d111b732fb..0ddde472877 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.48 0; +10.1.0.64 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index a3aaacfcd7a..b70423ac717 100644 --- a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2024.10.4835 -### Updated: Fri Oct 25 22:01:17 UTC 2024 +### Version: V4.2024.10.4836 +### Updated: Sat Oct 26 05:47:29 UTC 2024 ### Bad Referrer Count: 7104 -### Bad Bot Count: 667 +### Bad Bot Count: 668 ################################################### ### VERSION INFORMATION ## @@ -760,6 +760,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)domainsproject.org(?:\b)" 3; "~*(?:\b)eCatch(?:\b)" 3; "~*(?:\b)evc-batch(?:\b)" 3; + "~*(?:\b)everyfeed-spider(?:\b)" 3; "~*(?:\b)facebookscraper(?:\b)" 3; "~*(?:\b)gopher(?:\b)" 3; "~*(?:\b)heritrix(?:\b)" 3; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index 0d8dca8c3cc..99575d41d90 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.48 1; +10.1.0.64 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf index 8a5c58f474b..ced3f3f353a 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf @@ -668,6 +668,7 @@ "~*(?:\b)domainsproject.org(?:\b|)" 0; "~*(?:\b)eCatch(?:\b|)" 0; "~*(?:\b)evc-batch(?:\b|)" 0; + "~*(?:\b)everyfeed-spider(?:\b|)" 0; "~*(?:\b)facebookscraper(?:\b|)" 0; "~*(?:\b)gopher(?:\b|)" 0; "~*(?:\b)heritrix(?:\b|)" 0; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index 5d111b732fb..0ddde472877 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.48 0; +10.1.0.64 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index a3aaacfcd7a..b70423ac717 100644 --- a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2024.10.4835 -### Updated: Fri Oct 25 22:01:17 UTC 2024 +### Version: V4.2024.10.4836 +### Updated: Sat Oct 26 05:47:29 UTC 2024 ### Bad Referrer Count: 7104 -### Bad Bot Count: 667 +### Bad Bot Count: 668 ################################################### ### VERSION INFORMATION ## @@ -760,6 +760,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)domainsproject.org(?:\b)" 3; "~*(?:\b)eCatch(?:\b)" 3; "~*(?:\b)evc-batch(?:\b)" 3; + "~*(?:\b)everyfeed-spider(?:\b)" 3; "~*(?:\b)facebookscraper(?:\b)" 3; "~*(?:\b)gopher(?:\b)" 3; "~*(?:\b)heritrix(?:\b)" 3; diff --git a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index 1242e9bb048..1dea6b79e9b 100644 --- a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -172.183.122.254 0; +172.171.255.193 0; diff --git a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf index 216c0b6273b..7be4c1fe46a 100644 --- a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2024.10.4835 -### Updated: Fri Oct 25 22:01:17 UTC 2024 +### Version: V4.2024.10.4836 +### Updated: Sat Oct 26 05:47:29 UTC 2024 ### Bad Referrer Count: 7104 -### Bad Bot Count: 667 +### Bad Bot Count: 668 ################################################### ### VERSION INFORMATION ## @@ -760,6 +760,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)domainsproject.org(?:\b)" 3; "~*(?:\b)eCatch(?:\b)" 3; "~*(?:\b)evc-batch(?:\b)" 3; + "~*(?:\b)everyfeed-spider(?:\b)" 3; "~*(?:\b)facebookscraper(?:\b)" 3; "~*(?:\b)gopher(?:\b)" 3; "~*(?:\b)heritrix(?:\b)" 3; diff --git a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf index 1242e9bb048..1dea6b79e9b 100644 --- a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -172.183.122.254 0; +172.171.255.193 0; diff --git a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf index 59a27b7aea3..72359bfa103 100644 --- a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2024.10.4835 -### Updated: Fri Oct 25 22:01:17 UTC 2024 +### Version: V4.2024.10.4836 +### Updated: Sat Oct 26 05:47:29 UTC 2024 ### Bad Referrer Count: 7104 -### Bad Bot Count: 667 +### Bad Bot Count: 668 ################################################### ### VERSION INFORMATION ## @@ -760,6 +760,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)domainsproject.org(?:\b)" 3; "~*(?:\b)eCatch(?:\b)" 3; "~*(?:\b)evc-batch(?:\b)" 3; + "~*(?:\b)everyfeed-spider(?:\b)" 3; "~*(?:\b)facebookscraper(?:\b)" 3; "~*(?:\b)gopher(?:\b)" 3; "~*(?:\b)heritrix(?:\b)" 3; diff --git a/dev-tools/test_units/blacklist-ips.conf b/dev-tools/test_units/blacklist-ips.conf index 0d8dca8c3cc..99575d41d90 100644 --- a/dev-tools/test_units/blacklist-ips.conf +++ b/dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.48 1; +10.1.0.64 1; 127.0.0.1 1; diff --git a/dev-tools/test_units/blacklist-user-agents.conf b/dev-tools/test_units/blacklist-user-agents.conf index 8a5c58f474b..ced3f3f353a 100644 --- a/dev-tools/test_units/blacklist-user-agents.conf +++ b/dev-tools/test_units/blacklist-user-agents.conf @@ -668,6 +668,7 @@ "~*(?:\b)domainsproject.org(?:\b|)" 0; "~*(?:\b)eCatch(?:\b|)" 0; "~*(?:\b)evc-batch(?:\b|)" 0; + "~*(?:\b)everyfeed-spider(?:\b|)" 0; "~*(?:\b)facebookscraper(?:\b|)" 0; "~*(?:\b)gopher(?:\b|)" 0; "~*(?:\b)heritrix(?:\b|)" 0; diff --git a/dev-tools/test_units/random-bots-for-test-quick.list b/dev-tools/test_units/random-bots-for-test-quick.list index cb1e5be4166..f58d2408280 100644 --- a/dev-tools/test_units/random-bots-for-test-quick.list +++ b/dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -DigitalPebble -Lftp -LinkWalker -Nutch -Sucker -TeleportPro -TheNomad -Turnitin -URLy Warning -clark-crawler +BDFetch +Hloader +MIDown tool +Musobot +Navroad +RankActive +RebelMouse +Sottopop +adscanner +satoristudio.net diff --git a/dev-tools/test_units/random-bots-for-whitelist-test.list b/dev-tools/test_units/random-bots-for-whitelist-test.list index e621b282a58..4b08d7a7b7e 100644 --- a/dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,100 +1,100 @@ -360Spider -ALittle Client -ASPSeek -Aspiegel -BDCbot +404enemy +AIBOT +Abonti +AfD-Verbotsverfahren +Atomseobot +AwarioSmartBot +BDFetch BLEXBot +BackDoorBot Backlink-Ceck -Bigfoot +Badass +Blackboard Blow -Bullseye -BunnySlippers -ChinaClaw -Citoid -Claritybot -CyotekWebCopy -DIIbot -DSearch +BuiltWith +CATExplorador +CensysInspect +CheTeam +CheeseBot +Cocolyzebot +Copyscape +Crawling at Home Project +Crescent +Curious +Deusu Digincore -Download Wonder -FDM -Freeuploader -GrapeFX -HTMLparser -Heritrix +DomainCrawler +FHscan +FemtosearchBot +FrontPage +FyberSpider +G-i-g-a-b-o-t +GPTBot +GetWeb +GridBot +HTTrack Humanlinks -HybridBot +IRLbot Iblog -Intelliseek -Internet Ninja -Jetty -Lanshanbot -LieBaoFast -Lightspeedsystems -MJ12bot -MSIECrawler -Mass Downloader -Mata Hari -Meanpathbot -Mediatoolkitbot -Mojeek -MolokaiBot -Nessus -NetAnts -Nettrack -NextGenSearchBot -Nibbler -Ninja -OpenLinkProfiler +JOC Web Spider +JustView +Jyxobot +LNSpiderguy +LWP::Simple +Ltx71 +Masscan +Morfeus Fucking Scanner +Mr.4x3 +Net Vampire +Nikto +NimbleCrawler +Offline Explorer OrangeSpider -Page Analyzer -PeoplePal -Petalbot -Picscout -ProPowerBot -Reaper -SalesIntelligent -ScanAlert -SearchmetricsBot -SentiBot -Seomoz -SiteCheckerBotCrawler +OutclicksBot +Pandalytics +Pi-Monster +ProWebWalker +Pu_iN +PxBroker +PyCurl +Rankivabot +RankurBot +SMTBot +ScoutJet +Scrapy +SeobilityBot SiteExplorer -SiteLockSpider -SmartDownload -SocialRankIOBot -Spammen -Spider_Bot -Sqworm -Steeler -Swiftbot -T0PHackTeam -Telesoft -Telesphoreo -Toweyabot -Turingos -VeriCiteCrawler -Virusdie -WBSearchBot -WWW-Collector-E -WWWOFFLE -Web Auto -WebStripper -WebSucker -Whacker +Sosospider +SpaceBison +Spinn3r +SuperBot +T8Abot +Thumbor +TinyTestBot +Web Fuck +WebCopier +WebLeacher +WebSauger +WebZIP +Whack +Whatweb +Widow +Wotbox +Xaldon WebSpider Xenu -Zermelo +YoudaoBot Zeus -cognitiveseo -dataforseobot -evc-batch -imagesift.com -iubenda-radar -linkfluence -magpie-crawler -mediawords -page scorer -probe-image-size -ripz -sexsearcher -zgrab +anthropic-ai +archive.org_bot +instabid +ips-agent +isitwp.com +lwp-request +omgili +s1z.ru +satoristudio.net +seocompany.store +sogouspider +trendiction.de +webgains-bot +webpros.com diff --git a/dev-tools/test_units/random-referrers-for-test-quick.list b/dev-tools/test_units/random-referrers-for-test-quick.list index 5a4c8df6450..c77f48b59c8 100644 --- a/dev-tools/test_units/random-referrers-for-test-quick.list +++ b/dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -carina-sy.de -compliance-george.top -divci-hry.info -eloconcream.blogspot.com -keywordsking.com -kupit-adenu.ru -manuscript.su -onlywoman.org -szamponrevita.pl -urccvfmc.bloger.index.hr +bookmark4you.com.biz +cashonads.com +compliance-barak.xyz +gg-arena.ru +mindbox.co.za +ok-ua.info +slowmac.tech +trafficjunky.net +uhdtv.website +urengoy.pro diff --git a/dev-tools/test_units/random-referrers-for-whitelist-test.list b/dev-tools/test_units/random-referrers-for-whitelist-test.list index 7d11e914e88..504682cc265 100644 --- a/dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,100 +1,100 @@ -1millionusd.xyz -aa81bf391151884adfa3dd677e41f94be1.com -actulite.com -afftrack001.com -alibestsale.com -arcteryxsale.online -autoua.top -ayakino.net -babieca.com -beachpics.com -bear.gotcher.us -belstaffstore.online -bestmouthwash.club -bitcoinpile.com -bocoarchives.org -brainboostingsupplements.org -brandbucket.com -buigas.00it.com -chcu.net -cheapmoncler.win -chinese-amezon.com -dealitright.click -diegolopezcastan.com -drunkmoms.net -elektrozigaretten2.yn.lt -ezigarettekaufen1.hpage.com -f07.de -farmprofi.net -fickluder69.com -formseo.com -freejabs.com -fugarif.ga -gallery.rennlist.com -girls-ufa.ru -handicapbathtubarea.com -highland-homes.com -histock.info -hostcritique.com -hostingclub.lk -humanorightswatch.org -ikearugs.xyz -ikritikimou.gr -intimshop-fantasy.ru -jav-fetish.site -keenoutlet.online -kustanay.kz -law-check-nine.xyz -lion.playtap.us -liveinternet.ro -localmatchbook.com -lonely-mature.com -marketingvici.com +4replicawatch.net +66cpwgln.space +aaa38852e886ac4af1a3cff9b47cab6272.com +academiacsmendoza.org +ads-cool.pro +adviceforum.info +androids-store.com +arkkivoltti.net +asociatia-tipografilor-transilvania.ro +azazu.ru +bausparen.bz.it +bonkers.name +bonzbuddy.com +buzz-porno.info +centrumcoachingu.com +cialischsrx.com +clean-start.net +compliance-george.top +compliance-irvin.xyz +compliance-willy.top +custom-product-labels.com +data1.scopich.com +dealighted.com +domaincdn.xyz +drpornogratisx.xxx +ekto.ee +eluxer.net +epngo.bz +essayservicewriting.org +eyessurgery.ru +fefo.gdn +filmania-x.ru +freefoto.ca +gazporno.com +ge0ip.net +globalscam.ga +growboxbl.ru +gunsvicceadadebt.tk +hdseriale.pl +housediz.com +housemilan.ru +humbmosquina.tk +ilmexico.com +images-free.net +imagez.co +imediadesk.com +intim-uslugi.info +japan-bearings.ru +jerseywholesalechinabiz.com +katadhin.co +kchaxton.com +keywordglobal.co.za +keywordsuggests.com +lflash.ru maximilitary.ru -mixed-wrestling.ru -mlf.hordo.win -mobplayer.ru -moncleronline.xyz -muabancantho.info -mypornfree.ru -na-telefon.biz -notfastfood.ru -novodigs.com -nuit-artisanale.com -nuker.com -oxford-book.com.ua -parajumpersstore.online -pathwhelp.org -physfunc.ru -pojdelo.weebly.com -produkto.net -prohoster.info -redirectme.net -russia-tao.ru -samolet.fr -sexspornotub.com -sexuria.net -sharpchallenge.com -soft-terminal.ru -special-porn.com -studworks.org -sugarkun.com -summerlinhomes411.info -teenforporn.com -top-l2.com -trahic.ru -url-extractor.xyz +mesbuta.info +moroccosurfadventures.com +mrcsa.com.au +mrlmedia.net +myfreemp3.eu +myfreetutorials.com +najaden.no +no-rx.info +ownshop.cf +pandarastore.top +parlament.biz +phormchina.com +piccshare.com +portadd.men +privatefx-in.ru +pussysaga.com +remont-ustanovka-tehniki.ru +robot-forex.biz +rus-pornuha.com +ruscoininvest.company +safe-app.net +seozoom.it +shareyards.com +shemalegalls.blogporn.in +sim-service.net +snip.to +spacash.com +staff.prairiesouth.ca +strag-invest.ru +tako3.com +thebluffs.com +tizanidinehcl4mgisitanarcotic.blogspot.com +tpu.ru +treasuretrack-a.akamaihd.net +tubeoffline.com +v24s.net vbikse.com -viandpet.com -vikistars.com -vpnhowto.info -waterpurifier.club -wineitudes.wordpress.com -wormix-cheats.ru -www.ehscloud.cn -xxx-treker.ru -xxxguitars.com -xxxkaz.org -zed21.net -zeroredirect8.com -zryydi.com +villacoloniale.com +vote-up.ru +wholesalecheapjerseysfree.com +womensplay.net +www.timer4web.com +x-true.info +xn--80aafb2a.xn--p1ai +zmoda.hostreo.com diff --git a/dev-tools/test_units/whitelist-ips.conf b/dev-tools/test_units/whitelist-ips.conf index 5d111b732fb..0ddde472877 100644 --- a/dev-tools/test_units/whitelist-ips.conf +++ b/dev-tools/test_units/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.48 0; +10.1.0.64 0; 127.0.0.1 0; diff --git a/dev-tools/tmp/robots.txt b/dev-tools/tmp/robots.txt index 34e1babf05e..a0b049d89bb 100644 --- a/dev-tools/tmp/robots.txt +++ b/dev-tools/tmp/robots.txt @@ -6,9 +6,9 @@ ### Version Information # ################################################### -### Version: V4.2024.10.4835 -### Updated: Fri Oct 25 22:01:18 UTC 2024 -### Bad Bot Count: 667 +### Version: V4.2024.10.4836 +### Updated: Sat Oct 26 05:47:30 UTC 2024 +### Bad Bot Count: 668 ################################################### ### Version Information ## @@ -1233,6 +1233,8 @@ User-agent: eCatch Disallow:/ User-agent: evc-batch Disallow:/ +User-agent: everyfeed-spider +Disallow:/ User-agent: facebookscraper Disallow:/ User-agent: gopher diff --git a/robots.txt/robots.txt b/robots.txt/robots.txt index 34e1babf05e..a0b049d89bb 100644 --- a/robots.txt/robots.txt +++ b/robots.txt/robots.txt @@ -6,9 +6,9 @@ ### Version Information # ################################################### -### Version: V4.2024.10.4835 -### Updated: Fri Oct 25 22:01:18 UTC 2024 -### Bad Bot Count: 667 +### Version: V4.2024.10.4836 +### Updated: Sat Oct 26 05:47:30 UTC 2024 +### Bad Bot Count: 668 ################################################### ### Version Information ## @@ -1233,6 +1233,8 @@ User-agent: eCatch Disallow:/ User-agent: evc-batch Disallow:/ +User-agent: everyfeed-spider +Disallow:/ User-agent: facebookscraper Disallow:/ User-agent: gopher