From: Radek Czajka <rczajka@rczajka.pl>
Date: Thu, 26 Mar 2020 09:58:12 +0000 (+0100)
Subject: Remove crawler restriction.
X-Git-Url: https://git.mdrn.pl/wolnelektury.git/commitdiff_plain/51e143456755a74d038bf0250a999aed6d41b45e?ds=inline;hp=099458fdc499ac939a42af27c0df30afcdff55e9

Remove crawler restriction.
---

diff --git a/src/catalogue/views.py b/src/catalogue/views.py
index 2ef0d3bf0..aeed403ab 100644
--- a/src/catalogue/views.py
+++ b/src/catalogue/views.py
@@ -5,7 +5,6 @@ from collections import OrderedDict
 import random
 
 from django.conf import settings
-from django.http.response import HttpResponseForbidden
 from django.template.loader import render_to_string
 from django.shortcuts import get_object_or_404, render, redirect
 from django.http import HttpResponse, HttpResponseRedirect, Http404, HttpResponsePermanentRedirect
@@ -28,7 +27,6 @@ from catalogue.helpers import get_top_level_related_tags
 from catalogue.models import Book, Collection, Tag, Fragment
 from catalogue.utils import split_tags
 from catalogue.models.tag import prefetch_relations
-from wolnelektury.utils import is_crawler
 
 staff_required = user_passes_test(lambda user: user.is_staff)
 
@@ -224,9 +222,6 @@ def tagged_object_list(request, tags, list_type):
     except ResponseInstead as e:
         return e.response
 
-    if is_crawler(request) and len(tags) > 1:
-        return HttpResponseForbidden('address removed from crawling. check robots.txt')
-
     if list_type == 'gallery' and any(tag.category == 'set' for tag in tags):
         raise Http404
 
diff --git a/src/wolnelektury/utils.py b/src/wolnelektury/utils.py
index b0b972657..15319a11a 100644
--- a/src/wolnelektury/utils.py
+++ b/src/wolnelektury/utils.py
@@ -159,17 +159,6 @@ def re_escape(s):
     return re.sub(r"[(){}\[\].*?|^$\\+-]", r"\\\g<0>", s)
 
 
-BOT_BITS = ['bot', 'slurp', 'spider', 'facebook', 'crawler', 'parser', 'http']
-
-
-def is_crawler(request):
-    user_agent = request.META.get('HTTP_USER_AGENT')
-    if not user_agent:
-        return True
-    user_agent = user_agent.lower()
-    return any(bot_bit in user_agent for bot_bit in BOT_BITS)
-
-
 def get_cached_render_key(instance, property_name, language=None):
     if language is None:
         language = get_language()