+
+
+BOT_BITS = ['bot', 'slurp', 'spider', 'facebook', 'crawler', 'parser', 'http']
+
+
+def is_crawler(request):
+ user_agent = request.META.get('HTTP_USER_AGENT')
+ if not user_agent:
+ return True
+ user_agent = user_agent.lower()
+ return any(bot_bit in user_agent for bot_bit in BOT_BITS)