large boost for multiple results

[wolnelektury.git] / src / wolnelektury / utils.py

diff --git a/src/wolnelektury/utils.py b/src/wolnelektury/utils.py
index 8c5ead6..2657a55 100644 (file)
--- a/src/wolnelektury/utils.py
+++ b/src/wolnelektury/utils.py
@@ -12,6 +12,7 @@ from functools import wraps
 import pytz
 from inspect import getargspec
 
+import re
 from django.core.mail import send_mail
 from django.http import HttpResponse
 from django.template import RequestContext
@@ -149,3 +150,19 @@ class UnicodeCSVWriter(object):
     def writerows(self, rows):
         for row in rows:
             self.writerow(row)
+
+
+# the original re.escape messes with unicode
+def re_escape(s):
+    return re.sub(r"[(){}\[\].*?|^$\\+-]", r"\\\g<0>", s)
+
+
+BOT_BITS = ['bot', 'slurp', 'spider', 'facebook', 'crawler', 'parser', 'http']
+
+
+def is_crawler(request):
+    user_agent = request.META.get('HTTP_USER_AGENT')
+    if not user_agent:
+        return True
+    user_agent = user_agent.lower()
+    return any(bot_bit in user_agent for bot_bit in BOT_BITS)