diff --git a/DOC/README.bad_bot b/DOC/README.bad_bot new file mode 100644 index 0000000..14bc791 --- /dev/null +++ b/DOC/README.bad_bot @@ -0,0 +1,18 @@ +# ---- +# Deny User-Agent (i.e. Bbots not recognizing robots.txt +# ---- + +# Add to .htavvess +# +SetEnvIfNoCase User-Agent ".*MJ12bot.*" bad_bot +SetEnvIfNoCase User-Agent ".*Baiduspider.*" bad_bot +SetEnvIfNoCase User-Agent ".*Vagabondo.*" bad_bot +SetEnvIfNoCase User-Agent ".*lwp-trivial.*" bad_bot +SetEnvIfNoCase User-Agent ".*libwww.*" bad_bot +SetEnvIfNoCase User-Agent ".*Wget.*" bad_bot +SetEnvIfNoCase User-Agent ".*XoviBot.*" bad_bot +SetEnvIfNoCase User-Agent ".*xovibot.*" bad_bot +SetEnvIfNoCase User-Agent ".*AhrefsBot.*" bad_bot +SetEnvIfNoCase User-Agent ".*ClaudeBot.*" bad_bot +SetEnvIfNoCase User-Agent "SemrushBot" bad_bot +Deny from env=bad_bot