From d2b0cc75d0835ad5384bec638fea6ea244eadb40 Mon Sep 17 00:00:00 2001 From: Radek Czajka Date: Mon, 30 Dec 2019 23:24:59 +0100 Subject: [PATCH] Add findable flag. --- requirements/requirements.txt | 2 +- src/catalogue/api/views.py | 4 +++- src/catalogue/feeds.py | 1 + src/catalogue/helpers.py | 4 ++-- .../management/commands/importbooks.py | 8 +++++++- src/catalogue/migrations/0027_book_findable.py | 18 ++++++++++++++++++ src/catalogue/models/book.py | 16 ++++++++++------ src/catalogue/templatetags/catalogue_tags.py | 8 ++++---- src/catalogue/views.py | 15 +++++++++------ src/dictionary/models.py | 5 ++++- src/oai/handlers.py | 4 ++-- src/opds/views.py | 2 +- src/search/mock_search.py | 2 +- src/search/views.py | 2 +- src/wolnelektury/views.py | 6 +++--- 15 files changed, 67 insertions(+), 30 deletions(-) create mode 100644 src/catalogue/migrations/0027_book_findable.py diff --git a/requirements/requirements.txt b/requirements/requirements.txt index f217ccaff..fbf7a4bd7 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,7 +1,7 @@ -i https://py.mdrn.pl/simple/ # django -Django==2.2.8 +Django==2.2.9 fnpdjango==0.4 docutils diff --git a/src/catalogue/api/views.py b/src/catalogue/api/views.py index c70a73ed0..89fafd0d2 100644 --- a/src/catalogue/api/views.py +++ b/src/catalogue/api/views.py @@ -70,6 +70,7 @@ class BookList(ListAPIView): books = Book.tagged.with_all(tags) else: books = Book.objects.all() + books = books.filter(findable=True) books = order_books(books, new_api) if not Membership.is_active_for(self.request.user): @@ -187,6 +188,7 @@ class FilterBookList(ListAPIView): after = self.request.query_params.get('after') count = int(self.request.query_params.get('count', 50)) books = order_books(Book.objects.distinct(), new_api) + books = books.filter(findable=True) if is_lektura is not None: books = books.filter(has_audience=is_lektura) if is_audiobook is not None: @@ -293,7 +295,7 @@ class FragmentList(ListAPIView): ) except ValueError: raise Http404 - return Fragment.tagged.with_all(tags).select_related('book') + return Fragment.tagged.with_all(tags).filter(book__findable=True).select_related('book') @vary_on_auth # Because of 'liked'. diff --git a/src/catalogue/feeds.py b/src/catalogue/feeds.py index d33b79963..8658c8c55 100644 --- a/src/catalogue/feeds.py +++ b/src/catalogue/feeds.py @@ -39,6 +39,7 @@ class AudiobookFeed(Feed): def items(self, args): objects = models.BookMedia.objects.order_by('-uploaded_at') + objects = objects.filter(book__findable=True) if type == 'all': objects = objects.filter(type__in=('mp3', 'ogg', 'daisy')) else: diff --git a/src/catalogue/helpers.py b/src/catalogue/helpers.py index 796ed0483..c4b926783 100644 --- a/src/catalogue/helpers.py +++ b/src/catalogue/helpers.py @@ -77,7 +77,7 @@ def update_counters(): count_for_book(child, count_by_combination, combs_for_child) count_by_combination = defaultdict(lambda: 0) - for b in Book.objects.filter(parent=None): + for b in Book.objects.filter(findable=True, parent=None): count_for_book(b, count_by_combination) next_combinations = defaultdict(set) @@ -101,7 +101,7 @@ def update_counters(): def get_audiobook_tags(): audiobook_tag_ids = cache.get('audiobook_tags') if audiobook_tag_ids is None: - books_with_audiobook = Book.objects.filter(media__type__in=('mp3', 'ogg'))\ + books_with_audiobook = Book.objects.filter(findable=True, media__type__in=('mp3', 'ogg'))\ .distinct().values_list('pk', flat=True) audiobook_tag_ids = Tag.objects.filter( items__content_type=ContentType.objects.get_for_model(Book), diff --git a/src/catalogue/management/commands/importbooks.py b/src/catalogue/management/commands/importbooks.py index b8a9aa7bf..e5e7c89fa 100644 --- a/src/catalogue/management/commands/importbooks.py +++ b/src/catalogue/management/commands/importbooks.py @@ -32,6 +32,10 @@ class Command(BaseCommand): '-S', '--no-search-index', action='store_false', dest='search_index', default=True, help='Skip indexing imported works for search') + parser.add_argument( + '-F', '--not-findable', action='store_false', + dest='findable', default=True, + help='Set book as not findable.') parser.add_argument( '-p', '--picture', action='store_true', dest='import_picture', default=False, help='Import pictures') @@ -46,7 +50,9 @@ class Command(BaseCommand): file_base, ext = os.path.splitext(file_path) book = Book.from_xml_file(file_path, overwrite=options.get('force'), dont_build=dont_build, - search_index_tags=False) + search_index_tags=False, + findable=options.get('findable'), + ) for ebook_format in Book.ebook_formats: if os.path.isfile(file_base + '.' + ebook_format): getattr(book, '%s_file' % ebook_format).save( diff --git a/src/catalogue/migrations/0027_book_findable.py b/src/catalogue/migrations/0027_book_findable.py new file mode 100644 index 000000000..64e86434f --- /dev/null +++ b/src/catalogue/migrations/0027_book_findable.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.9 on 2019-12-30 21:57 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('catalogue', '0026_book_preview_key'), + ] + + operations = [ + migrations.AddField( + model_name='book', + name='findable', + field=models.BooleanField(db_index=True, default=True, verbose_name='findable'), + ), + ] diff --git a/src/catalogue/models/book.py b/src/catalogue/models/book.py index 5dbda4770..534f00bae 100644 --- a/src/catalogue/models/book.py +++ b/src/catalogue/models/book.py @@ -74,6 +74,7 @@ class Book(models.Model): preview = models.BooleanField(_('preview'), default=False) preview_until = models.DateField(_('preview until'), blank=True, null=True) preview_key = models.CharField(max_length=32, blank=True, null=True) + findable = models.BooleanField(_('findable'), default=True, db_index=True) # files generated during publication cover = EbookField( @@ -391,7 +392,7 @@ class Book(models.Model): format_) field_name = "%s_file" % format_ - books = Book.objects.filter(parent=None).exclude(**{field_name: ""}).exclude(preview=True) + books = Book.objects.filter(parent=None).exclude(**{field_name: ""}).exclude(preview=True).exclude(findable=False) paths = [(pretty_file_name(b), getattr(b, field_name).path) for b in books.iterator()] return create_zip(paths, app_settings.FORMAT_ZIPS[format_]) @@ -401,6 +402,8 @@ class Book(models.Model): return create_zip(paths, "%s_%s" % (self.slug, format_)) def search_index(self, book_info=None, index=None, index_tags=True, commit=True): + if not self.findable: + return if index is None: from search.index import Index index = Index() @@ -455,7 +458,7 @@ class Book(models.Model): @classmethod def from_text_and_meta(cls, raw_file, book_info, overwrite=False, dont_build=None, search_index=True, - search_index_tags=True, remote_gallery_url=None, days=0): + search_index_tags=True, remote_gallery_url=None, days=0, findable=True): if dont_build is None: dont_build = set() dont_build = set.union(set(dont_build), set(app_settings.DONT_BUILD)) @@ -493,6 +496,7 @@ class Book(models.Model): if book.preview: book.xml_file.set_readable(False) + book.findable = findable book.language = book_info.language book.title = book_info.title if book_info.variant_of: @@ -557,7 +561,7 @@ class Book(models.Model): if format_ not in dont_build: getattr(book, '%s_file' % format_).build_delay() - if not settings.NO_SEARCH_INDEX and search_index: + if not settings.NO_SEARCH_INDEX and search_index and findable: tasks.index_book.delay(book.id, book_info=book_info, index_tags=search_index_tags) for child in notify_cover_changed: @@ -644,7 +648,7 @@ class Book(models.Model): def other_versions(self): """Find other versions (i.e. in other languages) of the book.""" - return type(self).objects.filter(common_slug=self.common_slug).exclude(pk=self.pk) + return type(self).objects.filter(common_slug=self.common_slug, findable=True).exclude(pk=self.pk) def parents(self): books = [] @@ -681,7 +685,7 @@ class Book(models.Model): """ objects = cls.tagged.with_all(tags) - return objects.exclude(ancestor__in=objects) + return objects.filter(findable=True).exclude(ancestor__in=objects) @classmethod def book_list(cls, book_filter=None): @@ -692,7 +696,7 @@ class Book(models.Model): """ books_by_parent = {} - books = cls.objects.order_by('parent_number', 'sort_key').only('title', 'parent', 'slug', 'extra_info') + books = cls.objects.filter(findable=True).order_by('parent_number', 'sort_key').only('title', 'parent', 'slug', 'extra_info') if book_filter: books = books.filter(book_filter).distinct() diff --git a/src/catalogue/templatetags/catalogue_tags.py b/src/catalogue/templatetags/catalogue_tags.py index 69ff8d358..d9b0faadc 100644 --- a/src/catalogue/templatetags/catalogue_tags.py +++ b/src/catalogue/templatetags/catalogue_tags.py @@ -379,7 +379,7 @@ def related_books(context, instance, limit=6, random=1, taken=0): # Reserve one spot for an image. max_books -= 1 - books_qs = Book.objects.all() + books_qs = Book.objects.filter(findable=True) if not is_picture: books_qs = books_qs.exclude(common_slug=instance.common_slug).exclude(ancestor=instance) books = Book.tagged.related_to(instance, books_qs)[:max_books] @@ -458,7 +458,7 @@ def catalogue_random_book(exclude_ids): from .. import app_settings if random() < app_settings.RELATED_RANDOM_PICTURE_CHANCE: return None - queryset = Book.objects.exclude(pk__in=exclude_ids) + queryset = Book.objects.filter(findable=True).exclude(pk__in=exclude_ids) count = queryset.count() if count: return queryset[randint(0, count - 1)] @@ -473,9 +473,9 @@ def choose_fragment(book=None, tag_ids=None): else: if tag_ids is not None: tags = Tag.objects.filter(pk__in=tag_ids) - fragments = Fragment.tagged.with_all(tags).order_by().only('id') + fragments = Fragment.tagged.with_all(tags).filter(book__findable=True).order_by().only('id') else: - fragments = Fragment.objects.all().order_by().only('id') + fragments = Fragment.objects.filter(book__findable=True).order_by().only('id') fragment_count = fragments.count() fragment = fragments[randint(0, fragment_count - 1)] if fragment_count else None return fragment diff --git a/src/catalogue/views.py b/src/catalogue/views.py index 2c6692f19..b110b01c0 100644 --- a/src/catalogue/views.py +++ b/src/catalogue/views.py @@ -35,7 +35,7 @@ staff_required = user_passes_test(lambda user: user.is_staff) def catalogue(request): return render(request, 'catalogue/catalogue.html', { - 'books': Book.objects.filter(parent=None), + 'books': Book.objects.filter(findable=True, parent=None), 'pictures': Picture.objects.all(), 'collections': Collection.objects.all(), 'active_menu_item': 'all_works', @@ -146,7 +146,7 @@ def object_list(request, objects, fragments=None, related_tags=None, tags=None, def literature(request): - books = Book.objects.filter(parent=None) + books = Book.objects.filter(parent=None, findable=True) return object_list(request, books, related_tags=get_top_level_related_tags([])) @@ -155,9 +155,9 @@ def gallery(request): def audiobooks(request): - audiobooks = Book.objects.filter(media__type__in=('mp3', 'ogg')).distinct() + audiobooks = Book.objects.filter(findable=True, media__type__in=('mp3', 'ogg')).distinct() return object_list(request, audiobooks, list_type='audiobooks', extra={ - 'daisy': Book.objects.filter(media__type='daisy').distinct(), + 'daisy': Book.objects.filter(findable=True, media__type='daisy').distinct(), }) @@ -205,6 +205,8 @@ def theme_list(request, tags, list_type): # TODO: Pictures on shelves not supported yet. books = Book.tagged.with_all(shelf_tags).order_by() fragments = fragments.filter(Q(book__in=books) | Q(book__ancestor__in=books)) + else: + fragments = fragments.filter(book__findable=True) if not fragments and len(tags) == 1 and list_type == 'books': if PictureArea.tagged.with_any(tags).exists() or Picture.tagged.with_any(tags).exists(): @@ -237,15 +239,16 @@ def tagged_object_list(request, tags, list_type): if any(tag.category == 'set' for tag in tags): params = {'objects': books} else: + books = books.filter(findable=True) params = { - 'objects': Book.tagged_top_level(tags), + 'objects': Book.tagged_top_level(tags).filter(findable=True), 'fragments': Fragment.objects.filter(book__in=books), 'related_tags': get_top_level_related_tags(tags), } elif list_type == 'gallery': params = {'objects': Picture.tagged.with_all(tags)} elif list_type == 'audiobooks': - audiobooks = Book.objects.filter(media__type__in=('mp3', 'ogg')).distinct() + audiobooks = Book.objects.filter(findable=True, media__type__in=('mp3', 'ogg')).distinct() params = { 'objects': Book.tagged.with_all(tags, audiobooks), 'extra': { diff --git a/src/dictionary/models.py b/src/dictionary/models.py index 5c3d2b998..d395b231f 100644 --- a/src/dictionary/models.py +++ b/src/dictionary/models.py @@ -46,6 +46,8 @@ class NoteSource(models.Model): @task(ignore_result=True) def build_notes(book): + if not book.findable: + return task_logger.info(book.slug) with transaction.atomic(): book.notesource_set.all().delete() @@ -82,5 +84,6 @@ def build_notes(book): def notes_from_book(sender, instance, **kwargs): - build_notes.delay(instance) + if instance.findable: + build_notes.delay(instance) Book.html_built.connect(notes_from_book) diff --git a/src/oai/handlers.py b/src/oai/handlers.py index 356f51b57..54a0a20bc 100644 --- a/src/oai/handlers.py +++ b/src/oai/handlers.py @@ -68,7 +68,7 @@ class Catalogue(common.ResumptionOAIPMH): year_zero = timezone.make_aware(datetime(1990, 1, 1, 0, 0, 0), timezone.utc) try: - earliest_change = Book.objects.filter(preview=False).order_by('changed_at')[0].changed_at + earliest_change = Book.objects.filter(findable=True, preview=False).order_by('changed_at')[0].changed_at except IndexError: earliest_change = year_zero @@ -132,7 +132,7 @@ class Catalogue(common.ResumptionOAIPMH): raise error.NoSetHierarchyError("Wolne Lektury does not support sets.") # books = Book.tagged.with_all([tag]) else: - books = Book.objects.filter(preview=False) + books = Book.objects.filter(findable=True, preview=False) deleted = Deleted.objects.exclude(slug__exact='') books = books.order_by('changed_at') diff --git a/src/opds/views.py b/src/opds/views.py index 6d04c6c2a..8e929c6bc 100644 --- a/src/opds/views.py +++ b/src/opds/views.py @@ -434,7 +434,7 @@ class SearchFeed(AcquisitionFeed): results = q.execute() book_scores = dict([(r['book_id'], r['score']) for r in results]) - books = Book.objects.filter(id__in=set([r['book_id'] for r in results])) + books = Book.objects.filter(findable=True, id__in=set([r['book_id'] for r in results])) books = list(books) books.sort(reverse=True, key=lambda book: book_scores[book.id]) return books diff --git a/src/search/mock_search.py b/src/search/mock_search.py index 6c430400e..118078f9d 100644 --- a/src/search/mock_search.py +++ b/src/search/mock_search.py @@ -23,7 +23,7 @@ class Search(Mock): def _find_some_books(query_terms=None, max_results=20): from .index import SearchResult - qs = Book.objects.order_by('?') + qs = Book.objects.filter(findable=True).order_by('?') results = [] for book in qs[:randint(1, max_results)]: doc = { diff --git a/src/search/views.py b/src/search/views.py index 2fe94f431..4d792e439 100644 --- a/src/search/views.py +++ b/src/search/views.py @@ -92,7 +92,7 @@ def hint(request): 'id': b.id, 'url': b.get_absolute_url() } - for b in Book.objects.filter(title__iregex='\m' + prefix)[:limit-len(data)] + for b in Book.objects.filter(findable=True, title__iregex='\m' + prefix)[:limit-len(data)] ] callback = request.GET.get('callback', None) if callback: diff --git a/src/wolnelektury/views.py b/src/wolnelektury/views.py index 2736acc5c..415a4fd1b 100644 --- a/src/wolnelektury/views.py +++ b/src/wolnelektury/views.py @@ -27,7 +27,7 @@ from wolnelektury.forms import RegistrationForm, SocialSignupForm @never_cache def main_page(request): ctx = { - 'last_published': Book.objects.exclude(cover_thumb='').filter(parent=None).order_by('-created_at')[:6], + 'last_published': Book.objects.exclude(cover_thumb='').filter(findable=True, parent=None).order_by('-created_at')[:6], 'theme_books': [], } @@ -35,7 +35,7 @@ def main_page(request): if Fragment.objects.exists(): while True: ctx['theme'] = Tag.objects.filter(category='theme').order_by('?')[:1][0] - tf = Fragment.tagged.with_any([ctx['theme']]).select_related('book').order_by('?')[:100] + tf = Fragment.tagged.with_any([ctx['theme']]).select_related('book').filter(book__findable=True).order_by('?')[:100] if not tf: continue ctx['theme_fragment'] = tf[0] @@ -52,7 +52,7 @@ def main_page(request): except IndexError: pass - ctx['best'] = Book.objects.order_by('?')[:5] + ctx['best'] = Book.objects.filter(findable=True).order_by('?')[:5] return render(request, "main_page.html", ctx) -- 2.20.1