X-Git-Url: https://git.mdrn.pl/wolnelektury.git/blobdiff_plain/5105e426102784140df92c6a6ce1ffdf70866a4e..45696bbb638e45271f2c108616f7a9427cf87f7d:/src/wolnelektury/utils.py

diff --git a/src/wolnelektury/utils.py b/src/wolnelektury/utils.py
index d20039cd2..2657a5542 100644
--- a/src/wolnelektury/utils.py
+++ b/src/wolnelektury/utils.py
@@ -155,3 +155,14 @@ class UnicodeCSVWriter(object):
 # the original re.escape messes with unicode
 def re_escape(s):
     return re.sub(r"[(){}\[\].*?|^$\\+-]", r"\\\g<0>", s)
+
+
+BOT_BITS = ['bot', 'slurp', 'spider', 'facebook', 'crawler', 'parser', 'http']
+
+
+def is_crawler(request):
+    user_agent = request.META.get('HTTP_USER_AGENT')
+    if not user_agent:
+        return True
+    user_agent = user_agent.lower()
+    return any(bot_bit in user_agent for bot_bit in BOT_BITS)