From 7edc6bce62c0138a5fd7f75f647accae911c8ec4 Mon Sep 17 00:00:00 2001 From: Christoph Date: Sat, 29 Jun 2024 00:29:01 +0200 Subject: [PATCH] Add 'DOC/README.bad_bot'. --- DOC/README.bad_bot | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 DOC/README.bad_bot diff --git a/DOC/README.bad_bot b/DOC/README.bad_bot new file mode 100644 index 0000000..14bc791 --- /dev/null +++ b/DOC/README.bad_bot @@ -0,0 +1,18 @@ +# ---- +# Deny User-Agent (i.e. Bbots not recognizing robots.txt +# ---- + +# Add to .htavvess +# +SetEnvIfNoCase User-Agent ".*MJ12bot.*" bad_bot +SetEnvIfNoCase User-Agent ".*Baiduspider.*" bad_bot +SetEnvIfNoCase User-Agent ".*Vagabondo.*" bad_bot +SetEnvIfNoCase User-Agent ".*lwp-trivial.*" bad_bot +SetEnvIfNoCase User-Agent ".*libwww.*" bad_bot +SetEnvIfNoCase User-Agent ".*Wget.*" bad_bot +SetEnvIfNoCase User-Agent ".*XoviBot.*" bad_bot +SetEnvIfNoCase User-Agent ".*xovibot.*" bad_bot +SetEnvIfNoCase User-Agent ".*AhrefsBot.*" bad_bot +SetEnvIfNoCase User-Agent ".*ClaudeBot.*" bad_bot +SetEnvIfNoCase User-Agent "SemrushBot" bad_bot +Deny from env=bad_bot