From dfd584e3b136d770bf56569030d10712a8722569 Mon Sep 17 00:00:00 2001 From: Radek Czajka Date: Wed, 10 Sep 2014 09:20:52 +0200 Subject: [PATCH] Add Book.ancestor m2m. Remove *.*_counter, *.related_info, Tag.*_count model fields. --- apps/api/handlers.py | 36 +-- apps/api/management/commands/mobileinit.py | 2 +- apps/catalogue/fields.py | 12 +- apps/catalogue/helpers.py | 253 ++++++++++++++++++ .../management/commands/checkintegrity.py | 14 +- .../migrations/0002_book_ancestor.py | 80 ++++++ apps/catalogue/models/book.py | 240 +++++------------ apps/catalogue/models/listeners.py | 19 -- apps/catalogue/models/tag.py | 45 +--- apps/catalogue/tasks.py | 2 - .../templates/catalogue/book_short.html | 68 ++--- .../templates/catalogue/inline_tag_list.html | 4 +- .../catalogue/search_multiple_hits.html | 2 +- .../templates/catalogue/tag_list.html | 4 +- apps/catalogue/templatetags/catalogue_tags.py | 58 ++-- apps/catalogue/tests/book_import.py | 32 ++- apps/catalogue/tests/tags.py | 28 +- apps/catalogue/utils.py | 9 +- apps/catalogue/views.py | 116 +++----- apps/funding/models.py | 4 +- apps/lesmianator/models.py | 8 +- apps/newtagging/models.py | 14 +- apps/opds/views.py | 18 +- .../0002_remove_picture__related_info.py | 18 ++ apps/picture/models.py | 94 +------ .../templates/picture/picture_wide.html | 4 +- apps/picture/templatetags/picture_tags.py | 2 - apps/picture/tests/picture_import.py | 2 +- apps/reporting/utils.py | 2 +- apps/search/index.py | 2 +- apps/search/templatetags/search_tags.py | 12 +- apps/social/templatetags/social_tags.py | 5 +- apps/social/utils.py | 6 +- 33 files changed, 610 insertions(+), 605 deletions(-) create mode 100644 apps/catalogue/helpers.py create mode 100644 apps/catalogue/migrations/0002_book_ancestor.py create mode 100644 apps/picture/migrations/0002_remove_picture__related_info.py diff --git a/apps/api/handlers.py b/apps/api/handlers.py index e1792af13..a3a5ce599 100644 --- a/apps/api/handlers.py +++ b/apps/api/handlers.py @@ -19,7 +19,6 @@ from api.helpers import timestamp from api.models import Deleted from catalogue.forms import BookImportForm from catalogue.models import Book, Tag, BookMedia, Fragment, Collection -from catalogue.utils import related_tag_name from picture.models import Picture from picture.forms import PictureImportForm from wolnelektury.utils import tz @@ -54,10 +53,11 @@ def read_tags(tags, allowed): :raises: ValueError when tags can't be found """ if not tags: - return [] + return [], [] tags = tags.strip('/').split('/') real_tags = [] + books = [] while tags: category = tags.pop(0) slug = tags.pop(0) @@ -70,15 +70,14 @@ def read_tags(tags, allowed): if not category in allowed: raise ValueError('Category not allowed.') - # !^%@#$^#! if category == 'book': - slug = 'l-' + slug + books.append(Book.objects.get(slug=slug)) try: real_tags.append(Tag.objects.get(category=category, slug=slug)) except Tag.DoesNotExist: raise ValueError('Tag not found') - return real_tags + return real_tags, books # RESTful handlers @@ -186,7 +185,7 @@ class AnonymousBooksHandler(AnonymousBaseHandler, BookDetails): are returned. """ try: - tags = read_tags(tags, allowed=book_tag_categories) + tags, ancestors_ = read_tags(tags, allowed=book_tag_categories) except ValueError: return rc.NOT_FOUND @@ -247,7 +246,7 @@ def _tags_getter(category): def _tag_getter(category): @classmethod def get_tag(cls, book): - return ", ".join(related_tag_name(t) for t in book.related_info()['tags'].get(category, [])) + return ', '.join(tag.name for tag in book.tags.filter(category=category)) return get_tag for plural, singular in category_singular.items(): setattr(BookDetails, plural, _tags_getter(singular)) @@ -369,7 +368,7 @@ class TagsHandler(BaseHandler, TagDetails): except KeyError, e: return rc.NOT_FOUND - tags = Tag.objects.filter(category=category_sng).exclude(book_count=0) + tags = Tag.objects.filter(category=category_sng).exclude(items=None) if tags.exists(): return tags else: @@ -433,7 +432,7 @@ class FragmentsHandler(BaseHandler, FragmentDetails): """ try: - tags = read_tags(tags, allowed=self.categories) + tags, ancestors = read_tags(tags, allowed=self.categories) except ValueError: return rc.NOT_FOUND fragments = Fragment.tagged.with_all(tags).select_related('book') @@ -514,7 +513,7 @@ class CatalogueHandler(BaseHandler): obj[field] = book.get_absolute_url() elif field == 'tags': - obj[field] = [t.id for t in book.tags.exclude(category__in=('book', 'set')).iterator()] + obj[field] = [t.id for t in book.tags.exclude(category='set').iterator()] elif field == 'author': obj[field] = ", ".join(t.name for t in book.tags.filter(category='author').iterator()) @@ -627,13 +626,16 @@ class CatalogueHandler(BaseHandler): for tag in Tag.objects.filter(category__in=categories, changed_at__gte=since, - changed_at__lt=until).iterator(): - # only serve non-empty tags - if tag.book_count: - tag_d = cls.tag_dict(tag, fields) - updated.append(tag_d) - elif tag.created_at < since: - deleted.append(tag.id) + changed_at__lt=until + ).exclude(items=None).iterator(): + tag_d = cls.tag_dict(tag, fields) + updated.append(tag_d) + for tag in Tag.objects.filter(category__in=categories, + created_at__lt=since, + changed_at__gte=since, + changed_at__lt=until, + items=None).iterator(): + deleted.append(tag.id) if updated: changes['updated'] = updated diff --git a/apps/api/management/commands/mobileinit.py b/apps/api/management/commands/mobileinit.py index 3d88a7744..57b41aa46 100755 --- a/apps/api/management/commands/mobileinit.py +++ b/apps/api/management/commands/mobileinit.py @@ -24,7 +24,7 @@ class Command(BaseCommand): for b in Book.objects.all(): add_book(db, b) for t in Tag.objects.exclude( - category__in=('book', 'set', 'theme')).exclude(book_count=0): + category__in=('book', 'set', 'theme')).exclude(items=None): # only add non-empty tags add_tag(db, t) db.commit() diff --git a/apps/catalogue/fields.py b/apps/catalogue/fields.py index 8ed628e0e..d5cec2e62 100644 --- a/apps/catalogue/fields.py +++ b/apps/catalogue/fields.py @@ -149,7 +149,6 @@ class BuildHtml(BuildEbook): if html_output: meta_tags = list(book.tags.filter( category__in=('author', 'epoch', 'genre', 'kind'))) - book_tag = book.book_tag() lang = book.language lang = LANGUAGES_3TO2.get(lang, lang) @@ -162,13 +161,6 @@ class BuildHtml(BuildEbook): fieldfile.field.attname: fieldfile }) - # get ancestor l-tags for adding to new fragments - ancestor_tags = [] - p = book.parent - while p: - ancestor_tags.append(p.book_tag()) - p = p.parent - # Extract fragments closed_fragments, open_fragments = html.extract_fragments(fieldfile.path) for fragment in closed_fragments.values(): @@ -211,11 +203,9 @@ class BuildHtml(BuildEbook): book=book, text=text, short_text=short_text) new_fragment.save() - new_fragment.tags = set(meta_tags + themes + [book_tag] + ancestor_tags) - book.fix_tree_tags() + new_fragment.tags = set(meta_tags + themes) book.html_built.send(sender=book) return True - book.fix_tree_tags() return False @BuildEbook.register('cover_thumb') diff --git a/apps/catalogue/helpers.py b/apps/catalogue/helpers.py new file mode 100644 index 000000000..ddfa4828e --- /dev/null +++ b/apps/catalogue/helpers.py @@ -0,0 +1,253 @@ +from django.db import connection +from django.contrib.contenttypes.models import ContentType +from django.utils.translation import get_language +from picture.models import Picture, PictureArea +from catalogue.models import Fragment, Tag, Book + + +def _get_tag_relations_sql(tags): + select = """ + SELECT Rx.object_id, Rx.content_type_id + FROM catalogue_tag_relation Rx""" + joins = [] + where = ['WHERE Rx.tag_id = %d' % tags[0].pk] + for i, tag in enumerate(tags[1:]): + joins.append('INNER JOIN catalogue_tag_relation TR%(i)d ' + 'ON TR%(i)d.object_id = Rx.object_id ' + 'AND TR%(i)d.content_type_id = Rx.content_type_id' % {'i': i}) + where.append('AND TR%d.tag_id = %d' % (i, tag.pk)) + return " ".join([select] + joins + where) + + + +def get_related_tags(tags): + # Get Tag fields for constructing tags in a raw query. + tag_fields = ('id', 'category', 'slug', 'sort_key', 'name_%s' % get_language()) + tag_fields = ', '.join( + 'T.%s' % connection.ops.quote_name(field) + for field in tag_fields) + tag_ids = tuple(t.pk for t in tags) + + # This is based on fragments/areas sharing their works tags + qs = Tag.objects.raw(''' + SELECT ''' + tag_fields + ''', COUNT(T.id) count + FROM ( + -- R: TagRelations of all objects tagged with the given tags. + WITH R AS ( + ''' + _get_tag_relations_sql(tags) + ''' + ) + + SELECT ''' + tag_fields + ''', MAX(R4.object_id) ancestor + + FROM R R1 + + -- R2: All tags of the found objects. + JOIN catalogue_tag_relation R2 + ON R2.object_id = R1.object_id + AND R2.content_type_id = R1.content_type_id + + -- Tag data for output. + JOIN catalogue_tag T + ON T.id=R2.tag_id + + -- Special case for books: + -- We want to exclude from output all the relations + -- between a book and a tag, if there's a relation between + -- the the book's ancestor and the tag in the result. + LEFT JOIN catalogue_book_ancestor A + ON A.from_book_id = R1.object_id + AND R1.content_type_id = %s + LEFT JOIN catalogue_tag_relation R3 + ON R3.tag_id = R2.tag_id + AND R3.content_type_id = R1.content_type_id + AND R3.object_id = A.to_book_id + LEFT JOIN R R4 + ON R4.object_id = R3.object_id + AND R4.content_type_id = R3.content_type_id + + WHERE + -- Exclude from the result the tags we started with. + R2.tag_id NOT IN %s + -- Special case for books: exclude descendants. + -- AND R4.object_id IS NULL + AND ( + -- Only count fragment tags on fragments + -- and book tags for books. + (R2.content_type_id IN %s AND T.category IN %s) + OR + (R2.content_type_id IN %s AND T.category IN %s) + ) + + GROUP BY T.id, R2.object_id, R2.content_type_id + + ) T + -- Now group by tag and count occurencies. + WHERE ancestor IS NULL + GROUP BY ''' + tag_fields + ''' + ORDER BY T.sort_key + ''', params=( + ContentType.objects.get_for_model(Book).pk, + tag_ids, + tuple(ContentType.objects.get_for_model(model).pk + for model in (Fragment, PictureArea)), + ('theme', 'object'), + tuple(ContentType.objects.get_for_model(model).pk + for model in (Book, Picture)), + ('author', 'epoch', 'genre', 'kind'), + )) + return qs + + +def get_fragment_related_tags(tags): + tag_fields = ', '.join( + 'T.%s' % (connection.ops.quote_name(field.column)) + for field in Tag._meta.fields) + + tag_ids = tuple(t.pk for t in tags) + # This is based on fragments/areas sharing their works tags + return Tag.objects.raw(''' + SELECT T.*, COUNT(T.id) count + FROM ( + + SELECT T.* + + -- R1: TagRelations of all objects tagged with the given tags. + FROM ( + ''' + _get_tag_relations_sql(tags) + ''' + ) R1 + + -- R2: All tags of the found objects. + JOIN catalogue_tag_relation R2 + ON R2.object_id = R1.object_id + AND R2.content_type_id = R1.content_type_id + + -- Tag data for output. + JOIN catalogue_tag T + ON T.id = R2.tag_id + + WHERE + -- Exclude from the result the tags we started with. + R2.tag_id NOT IN %s + GROUP BY T.id, R2.object_id, R2.content_type_id + + ) T + -- Now group by tag and count occurencies. + GROUP BY ''' + tag_fields + ''' + ORDER BY T.sort_key + ''', params=( + tag_ids, + )) + + +def tags_usage_for_books(categories): + tag_fields = ', '.join( + 'T.%s' % (connection.ops.quote_name(field.column)) + for field in Tag._meta.fields) + + # This is based on fragments/areas sharing their works tags + return Tag.objects.raw(''' + SELECT T.*, COUNT(T.id) count + FROM ( + SELECT T.* + + FROM catalogue_tag_relation R1 + + -- Tag data for output. + JOIN catalogue_tag T + ON T.id=R1.tag_id + + -- We want to exclude from output all the relations + -- between a book and a tag, if there's a relation between + -- the the book's ancestor and the tag in the result. + LEFT JOIN catalogue_book_ancestor A + ON A.from_book_id=R1.object_id + LEFT JOIN catalogue_tag_relation R3 + ON R3.tag_id = R1.tag_id + AND R3.content_type_id = R1.content_type_id + AND R3.object_id = A.to_book_id + + WHERE + R1.content_type_id = %s + -- Special case for books: exclude descendants. + AND R3.object_id IS NULL + AND T.category IN %s + + -- TODO: + -- Shouldn't it just be 'distinct'? + -- Maybe it's faster this way. + GROUP BY T.id, R1.object_id, R1.content_type_id + + ) T + -- Now group by tag and count occurencies. + GROUP BY ''' + tag_fields + ''' + ORDER BY T.sort_key + ''', params=( + ContentType.objects.get_for_model(Book).pk, + tuple(categories), + )) + + +def tags_usage_for_works(categories): + tag_fields = ', '.join( + 'T.%s' % (connection.ops.quote_name(field.column)) + for field in Tag._meta.fields) + + return Tag.objects.raw(''' + SELECT T.*, COUNT(T.id) count + FROM ( + + SELECT T.* + + FROM catalogue_tag_relation R1 + + -- Tag data for output. + JOIN catalogue_tag T + ON T.id = R1.tag_id + + -- Special case for books: + -- We want to exclude from output all the relations + -- between a book and a tag, if there's a relation between + -- the the book's ancestor and the tag in the result. + LEFT JOIN catalogue_book_ancestor A + ON A.from_book_id = R1.object_id + AND R1.content_type_id = %s + LEFT JOIN catalogue_tag_relation R3 + ON R3.tag_id = R1.tag_id + AND R3.content_type_id = R1.content_type_id + AND R3.object_id = A.to_book_id + + WHERE + R1.content_type_id IN %s + -- Special case for books: exclude descendants. + AND R3.object_id IS NULL + AND T.category IN %s + + -- TODO: + -- Shouldn't it just be 'distinct'? + -- Maybe it's faster this way. + GROUP BY T.id, R1.object_id, R1.content_type_id + + ) T + -- Now group by tag and count occurencies. + GROUP BY ''' + tag_fields + ''' + ORDER BY T.sort_key + + ''', params=( + ContentType.objects.get_for_model(Book).pk, + tuple(ContentType.objects.get_for_model(model).pk for model in (Book, Picture)), + categories, + )) + + +def tags_usage_for_fragments(categories): + return Tag.objects.raw(''' + SELECT t.*, count(t.id) + from catalogue_tag_relation r + join catalogue_tag t + on t.id = r.tag_id + where t.category IN %s + group by t.id + order by t.sort_key + ''', params=( + categories, + )) diff --git a/apps/catalogue/management/commands/checkintegrity.py b/apps/catalogue/management/commands/checkintegrity.py index 0892a7815..51fcd94c7 100644 --- a/apps/catalogue/management/commands/checkintegrity.py +++ b/apps/catalogue/management/commands/checkintegrity.py @@ -42,20 +42,20 @@ class Command(BaseCommand): print "To resolve: republish parent book." print - # Check for parent l-tags. + # Check for ancestry. parents = [] parent = book.parent while parent: parents.append(parent) parent = parent.parent - ltags = [b.book_tag() for b in parents] - if set(ltags) != set(book.tags.filter(category='book')): + ancestors = list(book.ancestor.all()) + if set(ancestors) != set(parents): if options['verbose']: - print "Wrong book tags for book:", book - print "Is: ", ", ".join(sorted(t.slug for t in book.tags.filter(category='book'))) - print "Should be:", ", ".join(sorted(t.slug for t in ltags)) + print "Wrong ancestry for book:", book + print "Is: ", ", ".join(ancestors) + print "Should be:", ", ".join(parents) if not options['dry_run']: - book.tags = ltags + list(book.tags.exclude(category='book')) + book.fix_tree_tags() if options['verbose']: print "Fixed." if options['verbose']: diff --git a/apps/catalogue/migrations/0002_book_ancestor.py b/apps/catalogue/migrations/0002_book_ancestor.py new file mode 100644 index 000000000..4aa582819 --- /dev/null +++ b/apps/catalogue/migrations/0002_book_ancestor.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +def fix_tree_tags(apps, schema_editor): + """Fixes the ancestry cache.""" + # TODO: table names + from django.db import connection, transaction + if connection.vendor == 'postgres': + cursor = connection.cursor() + cursor.execute(""" + WITH RECURSIVE ancestry AS ( + SELECT book.id, book.parent_id + FROM catalogue_book AS book + WHERE book.parent_id IS NOT NULL + UNION + SELECT ancestor.id, book.parent_id + FROM ancestry AS ancestor, catalogue_book AS book + WHERE ancestor.parent_id = book.id + AND book.parent_id IS NOT NULL + ) + INSERT INTO catalogue_book_ancestor + (from_book_id, to_book_id) + SELECT id, parent_id + FROM ancestry + ORDER BY id; + """) + else: + Book = apps.get_model("catalogue", "Book") + for b in Book.objects.exclude(parent=None): + parent = b.parent + while parent is not None: + b.ancestor.add(parent) + parent = parent.parent + + +def remove_book_tags(apps, schema_editor): + Tag = apps.get_model("catalogue", "Tag") + Book = apps.get_model("catalogue", "Book") + Tag.objects.filter(category='book').delete() + + +class Migration(migrations.Migration): + + dependencies = [ + ('catalogue', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='book', + name='ancestor', + field=models.ManyToManyField(related_name=b'descendant', null=True, editable=False, to='catalogue.Book', blank=True), + preserve_default=True, + ), + + migrations.RunPython(fix_tree_tags), + migrations.RunPython(remove_book_tags), + + migrations.AlterField( + model_name='tag', + name='category', + field=models.CharField(db_index=True, max_length=50, verbose_name='Category', choices=[(b'author', 'author'), (b'epoch', 'period'), (b'kind', 'form'), (b'genre', 'genre'), (b'theme', 'motif'), (b'set', 'set'), (b'thing', 'thing')]), + ), + + migrations.RemoveField( + model_name='tag', + name='book_count', + ), + migrations.RemoveField( + model_name='tag', + name='picture_count', + ), + migrations.RemoveField( + model_name='book', + name='_related_info', + ), + ] diff --git a/apps/catalogue/models/book.py b/apps/catalogue/models/book.py index 8f5f107dc..37d9b71b2 100644 --- a/apps/catalogue/models/book.py +++ b/apps/catalogue/models/book.py @@ -3,10 +3,11 @@ # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information. # from collections import OrderedDict +from random import randint import re from django.conf import settings from django.core.cache import caches -from django.db import models +from django.db import connection, models, transaction from django.db.models import permalink import django.dispatch from django.contrib.contenttypes.fields import GenericRelation @@ -17,7 +18,7 @@ from fnpdjango.storage import BofhFileSystemStorage from catalogue import constants from catalogue.fields import EbookField from catalogue.models import Tag, Fragment, BookMedia -from catalogue.utils import create_zip, split_tags, related_tag_name +from catalogue.utils import create_zip, split_tags from catalogue import app_settings from catalogue import tasks from newtagging import managers @@ -72,8 +73,8 @@ class Book(models.Model): parent = models.ForeignKey('self', blank=True, null=True, related_name='children') - - _related_info = jsonfield.JSONField(blank=True, null=True, editable=False) + ancestor = models.ManyToManyField('self', blank=True, null=True, + editable=False, related_name='descendant', symmetrical=False) objects = models.Manager() tagged = managers.ModelTaggedItemManager(Tag) @@ -127,18 +128,6 @@ class Book(models.Model): def language_name(self): return dict(settings.LANGUAGES).get(self.language_code(), "") - def book_tag_slug(self): - return ('l-' + self.slug)[:120] - - def book_tag(self): - slug = self.book_tag_slug() - book_tag, created = Tag.objects.get_or_create(slug=slug, category='book') - if created: - book_tag.name = self.title[:50] - book_tag.sort_key = self.title.lower() - book_tag.save() - return book_tag - def has_media(self, type_): if type_ in Book.formats: return bool(getattr(self, "%s_file" % type_)) @@ -167,7 +156,6 @@ class Book(models.Model): if self.id is None: return - type(self).objects.filter(pk=self.pk).update(_related_info=None) # Fragment.short_html relies on book's tags, so reset it here too for fragm in self.fragments.all().iterator(): fragm.reset_short_html() @@ -336,6 +324,8 @@ class Book(models.Model): if old_cover: notify_cover_changed.append(child) + cls.fix_tree_tags() + # No saves beyond this point. # Build cover. @@ -362,43 +352,38 @@ class Book(models.Model): cls.published.send(sender=book) return book - def fix_tree_tags(self): - """Fixes the l-tags on the book's subtree. - - Makes sure that: - * the book has its parents book-tags, - * its fragments have the book's and its parents book-tags, - * runs those for every child book too, - * touches all relevant tags, - * resets tag and theme counter on the book and its ancestry. - """ - def fix_subtree(book, parent_tags): - affected_tags = set(book.tags) - book.tags = list(book.tags.exclude(category='book')) + parent_tags - sub_parent_tags = parent_tags + [book.book_tag()] - for frag in book.fragments.all(): - affected_tags.update(frag.tags) - frag.tags = list(frag.tags.exclude(category='book') - ) + sub_parent_tags - for child in book.children.all(): - affected_tags.update(fix_subtree(child, sub_parent_tags)) - return affected_tags - - parent_tags = [] - parent = self.parent - while parent is not None: - parent_tags.append(parent.book_tag()) - parent = parent.parent - - affected_tags = fix_subtree(self, parent_tags) - for tag in affected_tags: - tasks.touch_tag(tag) - - book = self - while book is not None: - book.reset_tag_counter() - book.reset_theme_counter() - book = book.parent + @classmethod + def fix_tree_tags(cls): + """Fixes the ancestry cache.""" + # TODO: table names + with transaction.atomic(): + cursor = connection.cursor() + if connection.vendor == 'postgres': + cursor.execute("TRUNCATE catalogue_book_ancestor") + cursor.execute(""" + WITH RECURSIVE ancestry AS ( + SELECT book.id, book.parent_id + FROM catalogue_book AS book + WHERE book.parent_id IS NOT NULL + UNION + SELECT ancestor.id, book.parent_id + FROM ancestry AS ancestor, catalogue_book AS book + WHERE ancestor.parent_id = book.id + AND book.parent_id IS NOT NULL + ) + INSERT INTO catalogue_book_ancestor + (from_book_id, to_book_id) + SELECT id, parent_id + FROM ancestry + ORDER BY id; + """) + else: + cursor.execute("DELETE FROM catalogue_book_ancestor") + for b in cls.objects.exclude(parent=None): + parent = b.parent + while parent is not None: + b.ancestor.add(parent) + parent = parent.parent def cover_info(self, inherit=True): """Returns a dictionary to serve as fallback for BookInfo. @@ -419,6 +404,11 @@ class Book(models.Model): info = parent_info return info + def related_themes(self): + return Tag.objects.usage_for_queryset( + Fragment.objects.filter(models.Q(book=self) | models.Q(book__ancestor=self)), + counts=True).filter(category='theme') + def parent_cover_changed(self): """Called when parent book's cover image is changed.""" if not self.cover_info(inherit=False): @@ -435,116 +425,19 @@ class Book(models.Model): """Find other versions (i.e. in other languages) of the book.""" return type(self).objects.filter(common_slug=self.common_slug).exclude(pk=self.pk) - def related_info(self): - """Keeps info about related objects (tags, media) in cache field.""" - if self._related_info is not None: - return self._related_info - else: - rel = {'tags': {}, 'media': {}} - - tags = self.tags.filter(category__in=( - 'author', 'kind', 'genre', 'epoch')) - tags = split_tags(tags) - for category in tags: - cat = [] - for tag in tags[category]: - tag_info = {'slug': tag.slug, 'name': tag.name} - for lc, ln in settings.LANGUAGES: - tag_name = getattr(tag, "name_%s" % lc) - if tag_name: - tag_info["name_%s" % lc] = tag_name - cat.append(tag_info) - rel['tags'][category] = cat - - for media_format in BookMedia.formats: - rel['media'][media_format] = self.has_media(media_format) - - book = self - parents = [] - while book.parent: - parents.append((book.parent.title, book.parent.slug)) - book = book.parent - parents = parents[::-1] - if parents: - rel['parents'] = parents - - if self.pk: - type(self).objects.filter(pk=self.pk).update(_related_info=rel) - return rel - - def related_themes(self): - theme_counter = self.theme_counter - book_themes = list(Tag.objects.filter(pk__in=theme_counter.keys())) - for tag in book_themes: - tag.count = theme_counter[tag.pk] - return book_themes - - def reset_tag_counter(self): - if self.id is None: - return - - cache_key = "Book.tag_counter/%d" % self.id - permanent_cache.delete(cache_key) - if self.parent: - self.parent.reset_tag_counter() - - @property - def tag_counter(self): - if self.id: - cache_key = "Book.tag_counter/%d" % self.id - tags = permanent_cache.get(cache_key) - else: - tags = None - - if tags is None: - tags = {} - for child in self.children.all().order_by().iterator(): - for tag_pk, value in child.tag_counter.iteritems(): - tags[tag_pk] = tags.get(tag_pk, 0) + value - for tag in self.tags.exclude(category__in=('book', 'theme', 'set')).order_by().iterator(): - tags[tag.pk] = 1 - - if self.id: - permanent_cache.set(cache_key, tags) - return tags - - def reset_theme_counter(self): - if self.id is None: - return - - cache_key = "Book.theme_counter/%d" % self.id - permanent_cache.delete(cache_key) - if self.parent: - self.parent.reset_theme_counter() - - @property - def theme_counter(self): - if self.id: - cache_key = "Book.theme_counter/%d" % self.id - tags = permanent_cache.get(cache_key) - else: - tags = None - - if tags is None: - tags = {} - for fragment in Fragment.tagged.with_any([self.book_tag()]).order_by().iterator(): - for tag in fragment.tags.filter(category='theme').order_by().iterator(): - tags[tag.pk] = tags.get(tag.pk, 0) + 1 - - if self.id: - permanent_cache.set(cache_key, tags) - return tags + def parents(self): + books = [] + parent = self.parent + while parent is not None: + books.insert(0, parent) + parent = parent.parent + return books def pretty_title(self, html_links=False): - book = self - rel_info = book.related_info() - names = [(related_tag_name(tag), Tag.create_url('author', tag['slug'])) - for tag in rel_info['tags'].get('author', ())] - if 'parents' in rel_info: - books = [(name, Book.create_url(slug)) - for name, slug in rel_info['parents']] - names.extend(reversed(books)) - names.append((self.title, self.get_absolute_url())) + names = [(tag.name, tag.get_absolute_url()) + for tag in self.tags.filter(category='author')] + books = self.parents() + [self] + names.extend([(b.title, b.get_absolute_url()) for b in books]) if html_links: names = ['%s' % (tag[1], tag[0]) for tag in names] @@ -560,17 +453,8 @@ class Book(models.Model): also tagged with those tags. """ - # get relevant books and their tags objects = cls.tagged.with_all(tags) - parents = objects.exclude(children=None).only('slug') - # eliminate descendants - l_tags = Tag.objects.filter(category='book', - slug__in=[book.book_tag_slug() for book in parents.iterator()]) - descendants_keys = [book.pk for book in cls.tagged.with_any(l_tags).only('pk').iterator()] - if descendants_keys: - objects = objects.exclude(pk__in=descendants_keys) - - return objects + return objects.exclude(ancestor__in=objects) @classmethod def book_list(cls, filter=None): @@ -634,15 +518,19 @@ class Book(models.Model): return None, None def choose_fragment(self): - tag = self.book_tag() - fragments = Fragment.tagged.with_any([tag]) - if fragments.exists(): - return fragments.order_by('?')[0] + fragments = self.fragments.order_by() + fragments_count = fragments.count() + if not fragments_count and self.children.exists(): + fragments = Fragment.objects.filter(book__ancestor=self).order_by() + fragments_count = fragments.count() + if fragments_count: + return fragments[randint(0, fragments_count - 1)] elif self.parent: return self.parent.choose_fragment() else: return None + # add the file fields for format_ in Book.formats: field_name = "%s_file" % format_ diff --git a/apps/catalogue/models/listeners.py b/apps/catalogue/models/listeners.py index b7c5d55bf..031c4f13c 100644 --- a/apps/catalogue/models/listeners.py +++ b/apps/catalogue/models/listeners.py @@ -15,25 +15,6 @@ from newtagging.models import tags_updated permanent_cache = caches['permanent'] -def _tags_updated_handler(sender, affected_tags, **kwargs): - # reset tag global counter - # we want Tag.changed_at updated for API to know the tag was touched - for tag in affected_tags: - tasks.touch_tag(tag) - - # if book tags changed, reset book tag counter - if isinstance(sender, Book) and \ - Tag.objects.filter(pk__in=(tag.pk for tag in affected_tags)).\ - exclude(category__in=('book', 'theme', 'set')).count(): - sender.reset_tag_counter() - # if fragment theme changed, reset book theme counter - elif isinstance(sender, Fragment) and \ - Tag.objects.filter(pk__in=(tag.pk for tag in affected_tags)).\ - filter(category='theme').count(): - sender.book.reset_theme_counter() -tags_updated.connect(_tags_updated_handler) - - def _pre_delete_handler(sender, instance, **kwargs): """ refresh Book on BookMedia delete """ if sender == BookMedia: diff --git a/apps/catalogue/models/tag.py b/apps/catalogue/models/tag.py index b1e3d69f2..bff993248 100644 --- a/apps/catalogue/models/tag.py +++ b/apps/catalogue/models/tag.py @@ -19,7 +19,6 @@ TAG_CATEGORIES = ( ('genre', _('genre')), ('theme', _('theme')), ('set', _('set')), - ('book', _('book')), ('thing', _('thing')), # things shown on pictures ) @@ -37,8 +36,6 @@ class Tag(TagBase): description = models.TextField(_('description'), blank=True) user = models.ForeignKey(User, blank=True, null=True) - book_count = models.IntegerField(_('book count'), blank=True, null=True) - picture_count = models.IntegerField(_('picture count'), blank=True, null=True) gazeta_link = models.CharField(blank=True, max_length=240) culturepl_link = models.CharField(blank=True, max_length=240) wiki_link = models.CharField(blank=True, max_length=240) @@ -77,11 +74,6 @@ class Tag(TagBase): def get_absolute_url(self): return ('catalogue.views.tagged_object_list', [self.url_chunk]) - def clean(self): - if self.category == 'book' and (self.gazeta_link or self.wiki_link): - raise ValidationError(ugettext( - u"Book tags can't have attached links. Set them directly on the book instead of it's tag.")) - @classmethod @permalink def create_url(cls, category, slug): @@ -94,41 +86,6 @@ class Tag(TagBase): has_description.short_description = _('description') has_description.boolean = True - def get_count(self): - """Returns global book count for book tags, fragment count for themes.""" - from catalogue.models import Book, Fragment - - if self.category == 'book': - # never used - objects = Book.objects.none() - elif self.category == 'theme': - objects = Fragment.tagged.with_all((self,)) - else: - objects = Book.tagged.with_all((self,)).order_by() - if self.category != 'set': - # eliminate descendants - l_tags = Tag.objects.filter(slug__in=[book.book_tag_slug() for book in objects.iterator()]) - descendants_keys = [book.pk for book in Book.tagged.with_any(l_tags).iterator()] - if descendants_keys: - objects = objects.exclude(pk__in=descendants_keys) - return objects.count() - - # I shouldn't break the get_count() api - # just to include pictures. - def get_picture_count(self): - from picture.models import Picture, PictureArea - - if self.category == 'book': - # never used - objects = Picture.objects.none() - elif self.category == 'theme': - objects = PictureArea.tagged.with_all((self,)) - elif self.category == 'thing': - objects = Picture.tagged.with_all((self,)) - else: - objects = Picture.tagged.with_all((self,)).order_by() - return objects.count() - @staticmethod def get_tag_list(tags): if isinstance(tags, basestring): @@ -145,7 +102,7 @@ class Tag(TagBase): category = Tag.categories_rev[name] else: try: - real_tags.append(Tag.objects.exclude(category='book').get(slug=name)) + real_tags.append(Tag.objects.get(slug=name)) deprecated = True except Tag.MultipleObjectsReturned, e: ambiguous_slugs.append(name) diff --git a/apps/catalogue/tasks.py b/apps/catalogue/tasks.py index 663a5bce4..7d180687c 100644 --- a/apps/catalogue/tasks.py +++ b/apps/catalogue/tasks.py @@ -12,8 +12,6 @@ from wolnelektury.utils import localtime_to_utc # TODO: move to model? def touch_tag(tag): update_dict = { - 'book_count': tag.get_count(), - 'picture_count': tag.get_picture_count(), 'changed_at': localtime_to_utc(datetime.now()), } diff --git a/apps/catalogue/templates/catalogue/book_short.html b/apps/catalogue/templates/catalogue/book_short.html index 9c90bddc8..58859af38 100644 --- a/apps/catalogue/templates/catalogue/book_short.html +++ b/apps/catalogue/templates/catalogue/book_short.html @@ -27,16 +27,16 @@
- {% for tag in related.tags.author %} - {% related_tag_name tag %}{% if not forloop.last %}, - {% endif %}{% endfor %}{% for title, slug in related.parents %}, - {{ title }}{% endfor %} + {% for tag in tags.author %} + {{ tag.name }}{% if not forloop.last %}, + {% endif %}{% endfor %}{% for parent in parents %}, + {{ parent.title }}{% endfor %}
- {% if main_link %}{% endif %} - {{ book.title }} - {% if main_link %}{% endif %} -
+ {% if main_link %}{% endif %} + {{ book.title }} + {% if main_link %}{% endif %} +
@@ -49,29 +49,29 @@ {% block cover-area-extra %}{% endblock %}
- {% spaceless %} + {% spaceless %} {% trans "Epoch" %}:  - {% for tag in related.tags.epoch %} - {% related_tag_name tag %} - {% if not forloop.last %}, {% endif %} + {% for tag in tags.epoch %} + {{ tag.name }} + {% if not forloop.last %}, {% endif %} {% endfor %} {% trans "Kind" %}:  - {% for tag in related.tags.kind %} - {% related_tag_name tag %} - {% if not forloop.last %}, {% endif %} + {% for tag in tags.kind %} + {{ tag.name }} + {% if not forloop.last %}, {% endif %} {% endfor %} {% trans "Genre" %}:  - {% for tag in related.tags.genre %} - {% related_tag_name tag %} - {% if not forloop.last %}, {% endif %} + {% for tag in tags.genre %} + {{ tag.name }} + {% if not forloop.last %}, {% endif %} {% endfor %} @@ -102,25 +102,25 @@
  • {% trans "Download" %}
    - {% if book.pdf_file %} - PDF {% trans "to print" %} - {% endif %} - {% if book.epub_file %} - EPUB {% trans "for an e-book reader" %} - {% endif %} - {% if book.mobi_file %} - MOBI {% trans "for Kindle" %} - {% endif %} - {% if book.fb2_file %} - FB2 {% trans "FictionBook" %} - {% endif %} - {% if book.txt_file %} - TXT {% trans "for advanced usage" %} - {% endif %} + {% if book.pdf_file %} + PDF {% trans "to print" %} + {% endif %} + {% if book.epub_file %} + EPUB {% trans "for an e-book reader" %} + {% endif %} + {% if book.mobi_file %} + MOBI {% trans "for Kindle" %} + {% endif %} + {% if book.fb2_file %} + FB2 {% trans "FictionBook" %} + {% endif %} + {% if book.txt_file %} + TXT {% trans "for advanced usage" %} + {% endif %}
  • - {% if related.media.mp3 or related.media.ogg %} + {% if has_audio %} {% trans "Listen" %} {% endif %}
    • diff --git a/apps/catalogue/templates/catalogue/inline_tag_list.html b/apps/catalogue/templates/catalogue/inline_tag_list.html index cd48c3ea1..59b1acce9 100755 --- a/apps/catalogue/templates/catalogue/inline_tag_list.html +++ b/apps/catalogue/templates/catalogue/inline_tag_list.html @@ -6,11 +6,11 @@ diff --git a/apps/catalogue/templates/catalogue/search_multiple_hits.html b/apps/catalogue/templates/catalogue/search_multiple_hits.html index c9d5a35ce..667de9cc4 100644 --- a/apps/catalogue/templates/catalogue/search_multiple_hits.html +++ b/apps/catalogue/templates/catalogue/search_multiple_hits.html @@ -10,7 +10,7 @@ {% if did_you_mean %} {% trans "Did you mean" %} {{did_you_mean|lower}}? {% endif %} - +
    {% if tags.author %}
    diff --git a/apps/catalogue/templates/catalogue/tag_list.html b/apps/catalogue/templates/catalogue/tag_list.html index 87097b193..21430480c 100644 --- a/apps/catalogue/templates/catalogue/tag_list.html +++ b/apps/catalogue/templates/catalogue/tag_list.html @@ -6,11 +6,11 @@ diff --git a/apps/catalogue/templatetags/catalogue_tags.py b/apps/catalogue/templatetags/catalogue_tags.py index 1cc0bb75e..515c1fe97 100644 --- a/apps/catalogue/templatetags/catalogue_tags.py +++ b/apps/catalogue/templatetags/catalogue_tags.py @@ -15,7 +15,7 @@ from django.core.urlresolvers import reverse from django.contrib.auth.forms import UserCreationForm, AuthenticationForm from django.utils.translation import ugettext as _ -from catalogue.utils import related_tag_name as _related_tag_name +from catalogue.utils import split_tags from catalogue.models import Book, BookMedia, Fragment, Tag, Source from catalogue.constants import LICENSES @@ -324,23 +324,12 @@ def book_info(book): @register.inclusion_tag('catalogue/book_wide.html', takes_context=True) def book_wide(context, book): - book_themes = book.related_themes() - extra_info = book.extra_info - hide_about = extra_info.get('about', '').startswith('http://wiki.wolnepodreczniki.pl') - stage_note, stage_note_url = book.stage_note() - - return { - 'book': book, - 'main_link': reverse('book_text', args=[book.slug]) if book.html_file else None, - 'related': book.related_info(), - 'extra_info': extra_info, - 'hide_about': hide_about, - 'themes': book_themes, - 'request': context.get('request'), - 'show_lang': book.language_code() != settings.LANGUAGE_CODE, - 'stage_note': stage_note, - 'stage_note_url': stage_note_url, - } + ctx = book_short(context, book) + ctx['extra_info'] = book.extra_info + ctx['hide_about'] = ctx['extra_info'].get('about', '').startswith('http://wiki.wolnepodreczniki.pl') + ctx['themes'] = book.related_themes() + ctx['main_link'] = reverse('book_text', args=[book.slug]) if book.html_file else None + return ctx @register.inclusion_tag('catalogue/book_short.html', takes_context=True) @@ -349,8 +338,10 @@ def book_short(context, book): return { 'book': book, + 'has_audio': book.has_media('mp3'), 'main_link': book.get_absolute_url(), - 'related': book.related_info(), + 'parents': book.parents(), + 'tags': split_tags(book.tags.exclude(category__in=('set', 'theme'))), 'request': context.get('request'), 'show_lang': book.language_code() != settings.LANGUAGE_CODE, 'stage_note': stage_note, @@ -360,8 +351,8 @@ def book_short(context, book): @register.inclusion_tag('catalogue/book_mini_box.html') def book_mini(book, with_link=True): - author_str = ", ".join(related_tag_name(tag) - for tag in book.related_info()['tags'].get('author', ())) + author_str = ", ".join(tag.name + for tag in book.tags.filter(category='author')) return { 'book': book, 'author_str': author_str, @@ -378,14 +369,19 @@ def work_list(context, object_list): @register.inclusion_tag('catalogue/fragment_promo.html') def fragment_promo(arg=None): - if arg is None: - fragments = Fragment.objects.all().order_by('?') - fragment = fragments[0] if fragments.exists() else None - elif isinstance(arg, Book): + if isinstance(arg, Book): fragment = arg.choose_fragment() else: - fragments = Fragment.tagged.with_all(arg).order_by('?') - fragment = fragments[0] if fragments.exists() else None + if arg is None: + fragments = Fragment.objects.all() + else: + fragments = Fragment.tagged.with_all(arg) + fragments = fragments.order_by().only('id') + fragments_count = fragments.count() + if fragments_count: + fragment = fragments.order_by()[randint(0, fragments_count - 1)] + else: + fragment = None return { 'fragment': fragment, @@ -400,7 +396,7 @@ def related_books(book, limit=6, random=1, taken=0): if related is None: related = Book.tagged.related_to(book, Book.objects.exclude(common_slug=book.common_slug) - ).exclude(tag_relations__tag=book.book_tag())[:limit-random] + ).exclude(ancestor=book)[:limit-random] cache.set(cache_key, related, 1800) if random: random_books = Book.objects.exclude( @@ -437,7 +433,6 @@ def tag_url(category, slug): @register.simple_tag def download_audio(book, daisy=True): - related = book.related_info() links = [] if related['media'].get('mp3'): links.append("%s" % @@ -475,11 +470,6 @@ def license_icon(license_url): } -@register.simple_tag -def related_tag_name(tag, lang=None): - return _related_tag_name(tag, lang) - - @register.filter def class_name(obj): return obj.__class__.__name__ diff --git a/apps/catalogue/tests/book_import.py b/apps/catalogue/tests/book_import.py index 4b01a8f83..775fc2929 100644 --- a/apps/catalogue/tests/book_import.py +++ b/apps/catalogue/tests/book_import.py @@ -295,13 +295,15 @@ class TreeImportTest(WLTestCase): u"There should be only parent on common tag page." ) pies = models.Tag.objects.get(slug='pies') - self.assertEqual(self.parent.theme_counter, {pies.pk: 1}, + themes = self.parent.related_themes() + self.assertEqual(len(themes), 1, u"There should be child theme in parent theme counter." ) - epoch = models.Tag.objects.get(slug='x-epoch') - self.assertEqual(epoch.book_count, 1, - u"There should be only parent in common tag's counter." - ) + # TODO: book_count is deprecated, update here. + #~ epoch = models.Tag.objects.get(slug='x-epoch') + #~ self.assertEqual(epoch.book_count, 1, + #~ u"There should be only parent in common tag's counter." + #~ ) def test_child_republish(self): CHILD_TEXT = """ @@ -320,13 +322,14 @@ class TreeImportTest(WLTestCase): ) pies = models.Tag.objects.get(slug='pies') kot = models.Tag.objects.get(slug='kot') - self.assertEqual(self.parent.theme_counter, {pies.pk: 1, kot.pk: 1}, + self.assertEqual(len(self.parent.related_themes()), 2, u"There should be child themes in parent theme counter." ) - epoch = models.Tag.objects.get(slug='x-epoch') - self.assertEqual(epoch.book_count, 1, - u"There should only be parent in common tag's counter." - ) + # TODO: book_count is deprecated, update here. + #~ epoch = models.Tag.objects.get(slug='x-epoch') + #~ self.assertEqual(epoch.book_count, 1, + #~ u"There should only be parent in common tag's counter." + #~ ) def test_book_change_child(self): second_child_info = BookInfoStub( @@ -357,13 +360,14 @@ class TreeImportTest(WLTestCase): u"There should be parent and old child on common tag page." ) kot = models.Tag.objects.get(slug='kot') - self.assertEqual(self.parent.theme_counter, {kot.pk: 1}, + self.assertEqual(len(self.parent.related_themes()), 1, u"There should only be new child themes in parent theme counter." ) epoch = models.Tag.objects.get(slug='x-epoch') - self.assertEqual(epoch.book_count, 2, - u"There should be parent and old child in common tag's counter." - ) + # book_count deprecated, update test. + #~ self.assertEqual(epoch.book_count, 2, + #~ u"There should be parent and old child in common tag's counter." + #~ ) self.assertEqual( list(self.client.get('/katalog/lektura/parent/motyw/kot/' ).context['fragments']), diff --git a/apps/catalogue/tests/tags.py b/apps/catalogue/tests/tags.py index 503b98bbc..f10780c0a 100644 --- a/apps/catalogue/tests/tags.py +++ b/apps/catalogue/tests/tags.py @@ -115,28 +115,27 @@ class TagRelatedTagsTests(WLTestCase): 'missing `author` related tag') self.assertTrue('Epoch' in [tag.name for tag in cats['epoch']], 'missing `epoch` related tag') - self.assertTrue("ChildKind" in [tag.name for tag in cats['kind']], - "missing `kind` related tag") + self.assertFalse("kind" in cats, + "There should be no child-only related `kind` tags") self.assertTrue("Genre" in [tag.name for tag in cats['genre']], 'missing `genre` related tag') - self.assertTrue("ChildGenre" in [tag.name for tag in cats['genre']], - "missing child's related tag") + self.assertFalse("ChildGenre" in [tag.name for tag in cats['genre']], + "There should be no child-only related `genre` tags") self.assertTrue("GchildGenre" in [tag.name for tag in cats['genre']], "missing grandchild's related tag") self.assertTrue('Theme' in [tag.name for tag in cats['theme']], "missing related theme") - self.assertTrue('Child1Theme' in [tag.name for tag in cats['theme']], - "missing child's related theme") + self.assertFalse('Child1Theme' in [tag.name for tag in cats['theme']], + "There should be no child-only related `theme` tags") self.assertTrue('GChildTheme' in [tag.name for tag in cats['theme']], "missing grandchild's related theme") - def test_related_differ(self): """ related tags shouldn't include filtering tags """ response = self.client.get('/katalog/rodzaj/kind/') cats = response.context['categories'] - self.assertFalse('Kind' in [tag.name for tag in cats['kind']], + self.assertFalse('kind' in cats, 'filtering tag wrongly included in related') cats = self.client.get('/katalog/motyw/theme/').context['categories'] self.assertFalse('Theme' in [tag.name for tag in cats['theme']], @@ -194,7 +193,6 @@ class CleanTagRelationTests(WLTestCase): """ there should be no tag relations left after deleting tags """ models.Tag.objects.all().delete() - self.assertEqual(len(self.book.related_info()['tags']), 0) self.assertEqual(len(self.book.related_themes()), 0) self.assertEqual(models.Tag.intermediary_table_model.objects.all().count(), 0, "orphaned TagRelation objects left") @@ -225,10 +223,9 @@ class TestIdenticalTag(WLTestCase): """ there should be all related tags in relevant categories """ book = models.Book.from_text_and_meta(ContentFile(self.book_text), self.book_info) - related_info = book.related_info() related_themes = book.related_themes() for category in 'author', 'kind', 'genre', 'epoch': - self.assertTrue('tag' in [tag['slug'] for tag in related_info['tags'][category]], + self.assertTrue('tag' in [tag.slug for tag in book.tags.filter(category=category)], 'missing related tag for %s' % category) self.assertTrue('tag' in [tag.slug for tag in related_themes]) @@ -270,12 +267,11 @@ class BookTagsTests(WLTestCase): """ book should have own tags and whole tree's themes """ book = models.Book.objects.get(slug='parent') - related_info = book.related_info() related_themes = book.related_themes() - self.assertEqual([t['slug'] for t in related_info['tags']['author']], + self.assertEqual([t.slug for t in book.tags.filter(category='author')], ['common-man']) - self.assertEqual([t['slug'] for t in related_info['tags']['kind']], + self.assertEqual([t.slug for t in book.tags.filter(category='kind')], ['kind']) self.assertEqual([(tag.name, tag.count) for tag in related_themes], [('ChildTheme', 1), ('ParentTheme', 1), ('Theme', 2)]) @@ -283,8 +279,8 @@ class BookTagsTests(WLTestCase): def test_catalogue_tags(self): """ test main page tags and counts """ context = self.client.get('/katalog/').context - self.assertEqual([(tag.name, tag.book_count) for tag in context['categories']['author']], + self.assertEqual([(tag.name, tag.count) for tag in context['categories']['author']], [('Jim Lazy', 1), ('Common Man', 1)]) - self.assertEqual([(tag.name, tag.book_count) for tag in context['categories']['theme']], + self.assertEqual([(tag.name, tag.count) for tag in context['fragment_tags']], [('ChildTheme', 1), ('ParentTheme', 1), ('Theme', 2)]) diff --git a/apps/catalogue/utils.py b/apps/catalogue/utils.py index a0e834c60..df7e438e4 100644 --- a/apps/catalogue/utils.py +++ b/apps/catalogue/utils.py @@ -178,8 +178,8 @@ class SortedMultiQuerySet(MultiQuerySet): self.order_by = kwargs.pop('order_by', None) self.sortfn = kwargs.pop('sortfn', None) if self.order_by is not None: - self.sortfn = lambda a, b: cmp(getattr(a, self.order_by), - getattr(b, self.order_by)) + self.sortfn = lambda a, b: cmp((getattr(a, f) for f in self.order_by), + (getattr(b, f) for f in self.order_by)) super(SortedMultiQuerySet, self).__init__(*args, **kwargs) def __getitem__(self, item): @@ -349,10 +349,5 @@ This can sometimes occupy lots of memory, so trim it here a bit. or [] -def related_tag_name(tag_info, language=None): - return tag_info.get("name_%s" % (language or get_language()), - tag_info.get("name_%s" % settings.LANGUAGE_CODE, "")) - - def delete_from_cache_by_language(cache, key_template): cache.delete_many([key_template % lc for lc, ln in settings.LANGUAGES]) diff --git a/apps/catalogue/views.py b/apps/catalogue/views.py index 0026b0a08..76aaddf16 100644 --- a/apps/catalogue/views.py +++ b/apps/catalogue/views.py @@ -23,6 +23,7 @@ from django.views.decorators.vary import vary_on_headers from ajaxable.utils import AjaxableFormView from catalogue import models from catalogue import forms +from .helpers import get_related_tags, get_fragment_related_tags, tags_usage_for_books, tags_usage_for_works, tags_usage_for_fragments from catalogue.utils import split_tags, MultiQuerySet, SortedMultiQuerySet from catalogue.templatetags.catalogue_tags import tag_list, collection_list from pdcounter import models as pdcounter_models @@ -37,26 +38,27 @@ permanent_cache = get_cache('permanent') @vary_on_headers('X-Requested-With') def catalogue(request): - cache_key = 'catalogue.catalogue/' + get_language() - output = permanent_cache.get(cache_key) + #cache_key = 'catalogue.catalogue/' + get_language() + #output = permanent_cache.get(cache_key) + output = None if output is None: - tags = models.Tag.objects.exclude( - category__in=('set', 'book')).exclude(book_count=0, picture_count=0) - tags = list(tags) - for tag in tags: - tag.count = tag.book_count + tag.picture_count - categories = split_tags(tags) - fragment_tags = categories.get('theme', []) + common_categories = ('author',) + split_categories = ('epoch', 'genre', 'kind') + + categories = split_tags(tags_usage_for_works(common_categories)) + book_categories = split_tags(tags_usage_for_books(split_categories)) + picture_categories = split_tags( + models.Tag.objects.usage_for_model(Picture, counts=True).filter( + category__in=split_categories)) + # we want global usage for themes + fragment_tags = list(tags_usage_for_fragments(('theme',))) collections = models.Collection.objects.all() render_tag_list = lambda x: render_to_string( 'catalogue/tag_list.html', tag_list(x)) - has_pictures = lambda x: filter(lambda y: y.picture_count > 0, x) - has_books = lambda x: filter(lambda y: y.book_count > 0, x) - def render_split(tags): - with_books = has_books(tags) - with_pictures = has_pictures(tags) + + def render_split(with_books, with_pictures): ctx = {} if with_books: ctx['books'] = render_tag_list(with_books) @@ -64,17 +66,18 @@ def catalogue(request): ctx['pictures'] = render_tag_list(with_pictures) return render_to_string('catalogue/tag_list_split.html', ctx) - output = {'theme': {}} + output = {} output['theme'] = render_tag_list(fragment_tags) - for category, tags in categories.items(): - if category in ('author', 'theme'): - output[category] = render_tag_list(tags) - else: - output[category] = render_split(tags) + for category in common_categories: + output[category] = render_tag_list(categories.get(category, [])) + for category in split_categories: + output[category] = render_split( + book_categories.get(category, []), + picture_categories.get(category, [])) output['collections'] = render_to_string( 'catalogue/collection_list.html', collection_list(collections)) - permanent_cache.set(cache_key, output) + #permanent_cache.set(cache_key, output) if request.is_ajax(): return JsonResponse(output) else: @@ -142,7 +145,7 @@ def differentiate_tags(request, tags, ambiguous_slugs): beginning = '/'.join(tag.url_chunk for tag in tags) unparsed = '/'.join(ambiguous_slugs[1:]) options = [] - for tag in models.Tag.objects.exclude(category='book').filter(slug=ambiguous_slugs[0]): + for tag in models.Tag.objects.filter(slug=ambiguous_slugs[0]): options.append({ 'url_args': '/'.join((beginning, tag.url_chunk, unparsed)).strip('/'), 'tags': [tag] @@ -198,25 +201,14 @@ def tagged_object_list(request, tags=''): areas = PictureArea.tagged.with_all(fragment_tags) if shelf_tags: + # FIXME: book tags here books = models.Book.tagged.with_all(shelf_tags).order_by() l_tags = models.Tag.objects.filter(category='book', slug__in=[book.book_tag_slug() for book in books.iterator()]) fragments = models.Fragment.tagged.with_any(l_tags, fragments) - # newtagging goes crazy if we just try: - #related_tags = models.Tag.objects.usage_for_queryset(fragments, counts=True, - # extra={'where': ["catalogue_tag.category != 'book'"]}) - - related_tags = [] - - fragment_keys = [fragment.pk for fragment in fragments.iterator()] - if fragment_keys: - related_tags = models.Fragment.tags.usage(counts=True, - filters={'pk__in': fragment_keys}).exclude( - category='book') - related_tags = (tag for tag in related_tags if tag not in fragment_tags) - categories = split_tags(related_tags, categories) - + related_tags = get_fragment_related_tags(tags) + categories = split_tags(related_tags, categories) object_queries.insert(0, fragments) area_keys = [area.pk for area in areas.iterator()] @@ -232,41 +224,19 @@ def tagged_object_list(request, tags=''): objects = MultiQuerySet(*object_queries) else: if shelf_is_set: - books = models.Book.tagged.with_all(tags).order_by('sort_key_author') + books = models.Book.tagged.with_all(tags).order_by( + 'sort_key_author', 'title') else: - books = models.Book.tagged_top_level(tags).order_by('sort_key_author') + books = models.Book.tagged_top_level(tags).order_by( + 'sort_key_author', 'title') - pictures = Picture.tagged.with_all(tags).order_by('sort_key_author') + pictures = Picture.tagged.with_all(tags).order_by( + 'sort_key_author', 'title') - related_counts = {} - if books.count() > 0: - # get related tags from `tag_counter` and `theme_counter` - tags_pks = [tag.pk for tag in tags] - for book in books: - for tag_pk, value in itertools.chain(book.tag_counter.iteritems(), book.theme_counter.iteritems()): - if tag_pk in tags_pks: - continue - related_counts[tag_pk] = related_counts.get(tag_pk, 0) + value + categories = split_tags(get_related_tags(tags)) - if pictures.count() > 0: - tags_pks = [tag.pk for tag in tags] - for picture in pictures: - for tag_pk, value in itertools.chain(picture.tag_counter.iteritems(), picture.theme_counter.iteritems()): - if tag_pk in tags_pks: - continue - related_counts[tag_pk] = related_counts.get(tag_pk, 0) + value - - related_tags = models.Tag.objects.filter(pk__in=related_counts.keys()) - related_tags = [tag for tag in related_tags if tag not in tags] - - for tag in related_tags: - tag.count = related_counts[tag.pk] - - categories = split_tags(related_tags) - del related_tags - - - objects = SortedMultiQuerySet(pictures, books, order_by='sort_key_author') + objects = SortedMultiQuerySet(pictures, books, + order_by=('sort_key_author', 'title')) if not objects: @@ -278,7 +248,7 @@ def tagged_object_list(request, tags=''): 'object_list': objects, 'categories': categories, 'only_shelf': only_shelf, - 'only_author': only_author, + #~ 'only_author': only_author, 'only_my_shelf': only_my_shelf, 'formats_form': forms.DownloadFormatsForm(), 'tags': tags, @@ -289,10 +259,9 @@ def tagged_object_list(request, tags=''): def book_fragments(request, slug, theme_slug): book = get_object_or_404(models.Book, slug=slug) - - book_tag = book.book_tag() theme = get_object_or_404(models.Tag, slug=theme_slug, category='theme') - fragments = models.Fragment.tagged.with_all([book_tag, theme]) + fragments = models.Fragment.tagged.with_all([theme]).filter( + Q(book=book) | Q(book__ancestor=book)) return render_to_response('catalogue/book_fragments.html', locals(), context_instance=RequestContext(request)) @@ -353,7 +322,6 @@ def book_text(request, slug): if not book.has_html_file(): raise Http404 - related = book.related_info() return render_to_response('catalogue/book_text.html', locals(), context_instance=RequestContext(request)) @@ -443,9 +411,9 @@ def _tags_starting_with(prefix, user=None): books = models.Book.objects.filter(_word_starts_with('title', prefix)) tags = models.Tag.objects.filter(_word_starts_with('name', prefix)) if user and user.is_authenticated(): - tags = tags.filter(~Q(category='book') & (~Q(category='set') | Q(user=user))) + tags = tags.filter(~Q(category='set') | Q(user=user)) else: - tags = tags.filter(~Q(category='book') & ~Q(category='set')) + tags = tags.exclude(category='set') prefix_regexp = re.compile(_word_starts_with_regexp(prefix)) return list(books) + list(tags) + [app for app in _apps if prefix_regexp.search(app.lower)] + list(book_stubs) + list(authors) diff --git a/apps/funding/models.py b/apps/funding/models.py index 800a15a3c..a9c3d8724 100644 --- a/apps/funding/models.py +++ b/apps/funding/models.py @@ -13,7 +13,7 @@ from django.utils.timezone import utc from django.utils.translation import ugettext_lazy as _, override import getpaid from catalogue.models import Book -from catalogue.utils import get_random_hash, related_tag_name +from catalogue.utils import get_random_hash from polls.models import Poll from django.contrib.sites.models import Site from . import app_settings @@ -176,7 +176,7 @@ class Offer(models.Model): 'funding/email/published.txt', { 'offer': self, 'book': self.book, - 'author': ", ".join(related_tag_name(a) for a in self.book.related_info()['tags']['author']), + 'author': self.book.pretty_title(), 'current': self.current(), }) diff --git a/apps/lesmianator/models.py b/apps/lesmianator/models.py index 3482e0896..1f558e916 100644 --- a/apps/lesmianator/models.py +++ b/apps/lesmianator/models.py @@ -142,13 +142,7 @@ class Continuations(models.Model): @classmethod def for_set(cls, tag): - # book contains its descendants, we don't want them twice - books = Book.tagged.with_any((tag,)) - l_tags = Tag.objects.filter(category='book', slug__in=[book.book_tag_slug() for book in books.iterator()]) - descendants_keys = [book.pk for book in Book.tagged.with_any(l_tags).iterator()] - if descendants_keys: - books = books.exclude(pk__in=descendants_keys) - + books = Book.tagged_top_level([tag]) cont_tabs = (cls.get(b) for b in books.iterator()) return reduce(cls.join_conts, cont_tabs) diff --git a/apps/newtagging/models.py b/apps/newtagging/models.py index 3f666812c..7e9936f30 100644 --- a/apps/newtagging/models.py +++ b/apps/newtagging/models.py @@ -104,6 +104,7 @@ class TagManager(models.Manager): of field lookups to be applied to the given Model as the ``filters`` argument. """ + # TODO: Do we really need this filters stuff? if filters is None: filters = {} queryset = model._default_manager.filter() @@ -158,18 +159,15 @@ class TaggedItemManager(models.Manager): """ queryset, model = get_queryset_and_model(queryset_or_model) tags = self.tag_model.get_tag_list(tags) - tag_count = len(tags) - if not tag_count: + if not tags: # No existing tags were given return queryset.none() - elif tag_count == 1: - # Optimisation for single tag - fall through to the simpler - # query below. - return queryset.filter(tag_relations__tag=tags[0]) # TODO: presumes reverse generic relation - return queryset.filter(tag_relations__tag__in=tags - ).annotate(count=models.Count('pk')).filter(count=len(tags)) + # Multiple joins are WAY faster than having-count, at least on Postgres 9.1. + for tag in tags: + queryset = queryset.filter(tag_relations__tag=tag) + return queryset def get_union_by_model(self, queryset_or_model, tags): """ diff --git a/apps/opds/views.py b/apps/opds/views.py index 33ca9dff6..c529e0272 100644 --- a/apps/opds/views.py +++ b/apps/opds/views.py @@ -238,7 +238,7 @@ class ByCategoryFeed(Feed): return feed['title'] def items(self, feed): - return Tag.objects.filter(category=feed['category']).exclude(book_count=0) + return Tag.objects.filter(category=feed['category']).exclude(items=None) def item_title(self, item): return item.name @@ -264,13 +264,7 @@ class ByTagFeed(AcquisitionFeed): return get_object_or_404(Tag, category=category, slug=slug) def items(self, tag): - books = Book.tagged.with_any([tag]) - l_tags = Tag.objects.filter(category='book', slug__in=[book.book_tag_slug() for book in books.iterator()]) - descendants_keys = [book.pk for book in Book.tagged.with_any(l_tags)] - if descendants_keys: - books = books.exclude(pk__in=descendants_keys) - - return books + return Book.tagged_top_level([tag]) @factory_decorator(logged_in_or_basicauth()) @@ -289,7 +283,7 @@ class UserFeed(Feed): return u"Półki użytkownika %s" % user.username def items(self, user): - return Tag.objects.filter(category='set', user=user).exclude(book_count=0) + return Tag.objects.filter(category='set', user=user).exclude(items=None) def item_title(self, item): return item.name @@ -300,9 +294,6 @@ class UserFeed(Feed): def item_description(self): return u'' -# no class decorators in python 2.5 -#UserFeed = factory_decorator(logged_in_or_basicauth())(UserFeed) - @factory_decorator(logged_in_or_basicauth()) @piwik_track @@ -322,9 +313,6 @@ class UserSetFeed(AcquisitionFeed): def items(self, tag): return Book.tagged.with_any([tag]) -# no class decorators in python 2.5 -#UserSetFeed = factory_decorator(logged_in_or_basicauth())(UserSetFeed) - @piwik_track class SearchFeed(AcquisitionFeed): diff --git a/apps/picture/migrations/0002_remove_picture__related_info.py b/apps/picture/migrations/0002_remove_picture__related_info.py new file mode 100644 index 000000000..10542c85b --- /dev/null +++ b/apps/picture/migrations/0002_remove_picture__related_info.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('picture', '0001_initial'), + ] + + operations = [ + migrations.RemoveField( + model_name='picture', + name='_related_info', + ), + ] diff --git a/apps/picture/models.py b/apps/picture/models.py index 74d86313f..75ea4c37d 100644 --- a/apps/picture/models.py +++ b/apps/picture/models.py @@ -106,8 +106,6 @@ class Picture(models.Model): culturepl_link = models.CharField(blank=True, max_length=240) wiki_link = models.CharField(blank=True, max_length=240) - _related_info = jsonfield.JSONField(blank=True, null=True, editable=False) - width = models.IntegerField(null=True) height = models.IntegerField(null=True) @@ -333,7 +331,6 @@ class Picture(models.Model): if self.id is None: return - type(self).objects.filter(pk=self.pk).update(_related_info=None) for area in self.areas.all().iterator(): area.reset_short_html() @@ -350,7 +347,7 @@ class Picture(models.Model): def short_html(self): if self.id: cache_key = "Picture.short_html/%d/%s" % (self.id, get_language()) - short_html = get_cache('permanent').get(cache_key) + short_html = permanent_cache.get(cache_key) else: short_html = None @@ -365,12 +362,11 @@ class Picture(models.Model): {'picture': self, 'tags': tags})) if self.id: - get_cache('permanent').set(cache_key, short_html) + permanent_cache.set(cache_key, short_html) return mark_safe(short_html) def pretty_title(self, html_links=False): picture = self - # TODO Add translations (related_tag_info) names = [(tag.name, catalogue.models.Tag.create_url('author', tag.slug)) for tag in self.tags.filter(category='author')] @@ -382,89 +378,7 @@ class Picture(models.Model): names = [tag[0] for tag in names] return ', '.join(names) - def related_info(self): - """Keeps info about related objects (tags) in cache field.""" - if self._related_info is not None: - return self._related_info - else: - rel = {'tags': {}} - - tags = self.tags.filter(category__in=( - 'author', 'kind', 'genre', 'epoch')) - tags = split_tags(tags) - for category in tags: - cat = [] - for tag in tags[category]: - tag_info = {'slug': tag.slug, 'name': tag.name} - for lc, ln in settings.LANGUAGES: - tag_name = getattr(tag, "name_%s" % lc) - if tag_name: - tag_info["name_%s" % lc] = tag_name - cat.append(tag_info) - rel['tags'][category] = cat - - - if self.pk: - type(self).objects.filter(pk=self.pk).update(_related_info=rel) - return rel - # copied from book.py, figure out def related_themes(self): - # self.theme_counter hides a computation, so a line below actually makes sense - theme_counter = self.theme_counter - picture_themes = list(catalogue.models.Tag.objects.filter(pk__in=theme_counter.keys())) - for tag in picture_themes: - tag.count = theme_counter[tag.pk] - return picture_themes - - def reset_tag_counter(self): - if self.id is None: - return - - cache_key = "Picture.tag_counter/%d" % self.id - permanent_cache.delete(cache_key) - if self.parent: - self.parent.reset_tag_counter() - - @property - def tag_counter(self): - if self.id: - cache_key = "Picture.tag_counter/%d" % self.id - tags = permanent_cache.get(cache_key) - else: - tags = None - - if tags is None: - tags = {} - # do we need to do this? there are no children here. - for tag in self.tags.exclude(category__in=('book', 'theme', 'thing', 'set')).order_by().iterator(): - tags[tag.pk] = 1 - - if self.id: - permanent_cache.set(cache_key, tags) - return tags - - def reset_theme_counter(self): - if self.id is None: - return - - cache_key = "Picture.theme_counter/%d" % self.id - permanent_cache.delete(cache_key) - - @property - def theme_counter(self): - if self.id: - cache_key = "Picture.theme_counter/%d" % self.id - tags = permanent_cache.get(cache_key) - else: - tags = None - - if tags is None: - tags = {} - for area in PictureArea.objects.filter(picture=self).order_by().iterator(): - for tag in area.tags.filter(category__in=('theme', 'thing')).order_by().iterator(): - tags[tag.pk] = tags.get(tag.pk, 0) + 1 - - if self.id: - permanent_cache.set(cache_key, tags) - return tags + return catalogue.models.Tag.usage_for_queryset( + self.areas.all(), counts=True).filter(category__in=('theme', 'thing')) diff --git a/apps/picture/templates/picture/picture_wide.html b/apps/picture/templates/picture/picture_wide.html index 8525d32a9..28cd34f65 100644 --- a/apps/picture/templates/picture/picture_wide.html +++ b/apps/picture/templates/picture/picture_wide.html @@ -23,7 +23,7 @@

    {% trans "Motifs and themes" %}

    {% endif %} @@ -31,7 +31,7 @@

    {% trans "Objects" %}

    {% endif %} diff --git a/apps/picture/templatetags/picture_tags.py b/apps/picture/templatetags/picture_tags.py index 62826103b..1767ce228 100644 --- a/apps/picture/templatetags/picture_tags.py +++ b/apps/picture/templatetags/picture_tags.py @@ -17,7 +17,6 @@ def picture_short(context, picture): context.update({ 'picture': picture, 'main_link': picture.get_absolute_url(), - # 'related': picture.related_info(), 'request': context.get('request'), 'tags': split_tags(picture.tags), }) @@ -28,7 +27,6 @@ def picture_wide(context, picture): context.update({ 'picture': picture, 'main_link': picture.get_absolute_url(), - # 'related': picture.related_info(), 'request': context.get('request'), 'tags': split_tags(picture.tags), }) diff --git a/apps/picture/tests/picture_import.py b/apps/picture/tests/picture_import.py index 1b01d334c..7b1f64807 100644 --- a/apps/picture/tests/picture_import.py +++ b/apps/picture/tests/picture_import.py @@ -21,7 +21,7 @@ class PictureTest(WLTestCase): assert themes == set([(u'theme', u'nieporządek'), (u'thing', u'kosmos')]), \ 'Bad themes on Picture areas: %s' % themes - pic_themes = set([tag.name for tag in picture.tags if tag.category in ('theme', 'object')]) + pic_themes = set([tag.name for tag in picture.tags if tag.category in ('theme', 'thing')]) assert not pic_themes, 'Unwanted themes set on Pictures: %s' % pic_themes picture.delete() diff --git a/apps/reporting/utils.py b/apps/reporting/utils.py index 06a168074..8ecb9b045 100755 --- a/apps/reporting/utils.py +++ b/apps/reporting/utils.py @@ -113,7 +113,7 @@ def generated_file_view(file_name, mime_type, send_name=None, signals=None): else: name = send_name - response = HttpResponse(mimetype=mime_type) + response = HttpResponse(content_type=mime_type) response['Content-Disposition'] = 'attachment; filename=%s' % name with open(file_path) as f: for chunk in read_chunks(f): diff --git a/apps/search/index.py b/apps/search/index.py index 7fb60b508..7f8bf9f6a 100644 --- a/apps/search/index.py +++ b/apps/search/index.py @@ -852,7 +852,7 @@ class Search(SolrIndex): q |= self.index.Q(**{field: query + "*"}) else: q |= self.make_term_query(query, field=field) - qu = self.index.query(q).exclude(tag_category="book") + qu = self.index.query(q) return self.search_tags(qu, pdcounter=pdcounter) diff --git a/apps/search/templatetags/search_tags.py b/apps/search/templatetags/search_tags.py index a167f024d..65d94270e 100644 --- a/apps/search/templatetags/search_tags.py +++ b/apps/search/templatetags/search_tags.py @@ -10,6 +10,7 @@ from django import template # from django.db.models import Q # from django.utils.translation import ugettext as _ from catalogue.models import Book +from catalogue.templatetags.catalogue_tags import book_short import re # from catalogue.forms import SearchForm # from catalogue.utils import split_tags @@ -48,11 +49,6 @@ def book_searched(context, result): snip = snip.replace("\n", "
    ").replace('---', '—') hit['snippet'] = snip - return { - 'related': book.related_info(), - 'book': book, - 'main_link': book.get_absolute_url(), - 'request': context.get('request'), - 'hits': hits and zip(*hits)[1] or [], - 'main_link': book.get_absolute_url(), - } + ctx = book_short(context, book) + ctx['hits'] = hits and zip(*hits)[1] or [] + return ctx diff --git a/apps/social/templatetags/social_tags.py b/apps/social/templatetags/social_tags.py index ecf2c668c..bb1b4bca0 100755 --- a/apps/social/templatetags/social_tags.py +++ b/apps/social/templatetags/social_tags.py @@ -3,6 +3,7 @@ # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information. # from random import randint +from django.db.models import Q from django import template from catalogue.models import Book from social.models import Cite @@ -25,9 +26,7 @@ def choose_cite(context, ctx=None): if ctx is None: cites = Cite.objects.all() elif isinstance(ctx, Book): - cites = ctx.cite_set.all() - if not cites.exists(): - cites = cites_for_tags([ctx.book_tag()]) + cites = Cite.objects.filter(Q(book=ctx) | Q(book__ancestor=ctx)) else: cites = cites_for_tags(ctx) stickies = cites.filter(sticky=True) diff --git a/apps/social/utils.py b/apps/social/utils.py index 43bc02905..c6a93537b 100755 --- a/apps/social/utils.py +++ b/apps/social/utils.py @@ -37,11 +37,9 @@ def set_sets(user, work, sets): touch_tag(shelf) # delete empty tags - Tag.objects.filter(category='set', user=user, book_count=0).delete() + Tag.objects.filter(category='set', user=user, items=None).delete() def cites_for_tags(tags): """Returns a QuerySet with all Cites for books with given tags.""" - books = Book.tagged.with_all(tags).order_by().values_list('id', flat=True) - books = list(books) - return Cite.objects.filter(book__id__in=books) + return Cite.objects.filter(book__in=Book.tagged.with_all(tags)) -- 2.20.1