-BOT_BITS = ['bot', 'slurp', 'spider', 'facebook', 'crawler', 'parser', 'http']
-
-
-def is_crawler(request):
- user_agent = request.META.get('HTTP_USER_AGENT')
- if not user_agent:
- return True
- user_agent = user_agent.lower()
- return any(bot_bit in user_agent for bot_bit in BOT_BITS)
-
-