19 lines
652 B
Plaintext
19 lines
652 B
Plaintext
# ----
|
|
# Deny User-Agent (i.e. Bbots not recognizing robots.txt
|
|
# ----
|
|
|
|
# Add to .htavvess
|
|
#
|
|
SetEnvIfNoCase User-Agent ".*MJ12bot.*" bad_bot
|
|
SetEnvIfNoCase User-Agent ".*Baiduspider.*" bad_bot
|
|
SetEnvIfNoCase User-Agent ".*Vagabondo.*" bad_bot
|
|
SetEnvIfNoCase User-Agent ".*lwp-trivial.*" bad_bot
|
|
SetEnvIfNoCase User-Agent ".*libwww.*" bad_bot
|
|
SetEnvIfNoCase User-Agent ".*Wget.*" bad_bot
|
|
SetEnvIfNoCase User-Agent ".*XoviBot.*" bad_bot
|
|
SetEnvIfNoCase User-Agent ".*xovibot.*" bad_bot
|
|
SetEnvIfNoCase User-Agent ".*AhrefsBot.*" bad_bot
|
|
SetEnvIfNoCase User-Agent ".*ClaudeBot.*" bad_bot
|
|
SetEnvIfNoCase User-Agent "SemrushBot" bad_bot
|
|
Deny from env=bad_bot
|