Dead links reporting.

[wolnelektury.git] / apps / catalogue / models / book.py

diff --git a/apps/catalogue/models/book.py b/apps/catalogue/models/book.py
index bab6f58..bfc9ec4 100644 (file)
--- a/apps/catalogue/models/book.py
+++ b/apps/catalogue/models/book.py
@@ -2,34 +2,36 @@
 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
 #
 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
 #

+from collections import OrderedDict
+from random import randint

 import re
 from django.conf import settings
 import re
 from django.conf import settings

-from django.core.cache import get_cache
-from django.db import models
+from django.db import connection, models, transaction

 from django.db.models import permalink
 import django.dispatch
 from django.db.models import permalink
 import django.dispatch

+from django.contrib.contenttypes.fields import GenericRelation

 from django.core.urlresolvers import reverse
 from django.core.urlresolvers import reverse

-from django.utils.datastructures import SortedDict

 from django.utils.translation import ugettext_lazy as _
 import jsonfield
 from fnpdjango.storage import BofhFileSystemStorage
 from django.utils.translation import ugettext_lazy as _
 import jsonfield
 from fnpdjango.storage import BofhFileSystemStorage

+from ssify import flush_ssi_includes
+from newtagging import managers

 from catalogue import constants
 from catalogue.fields import EbookField
 from catalogue.models import Tag, Fragment, BookMedia
 from catalogue import constants
 from catalogue.fields import EbookField
 from catalogue.models import Tag, Fragment, BookMedia

-from catalogue.utils import create_zip, split_tags, related_tag_name
+from catalogue.utils import create_zip

 from catalogue import app_settings
 from catalogue import tasks
 from catalogue import app_settings
 from catalogue import tasks

-from newtagging import managers

 
 bofh_storage = BofhFileSystemStorage()
 
 
 bofh_storage = BofhFileSystemStorage()
 

-permanent_cache = get_cache('permanent')
-

 
 def _cover_upload_to(i, n):
     return 'book/cover/%s.jpg' % i.slug
 
 def _cover_upload_to(i, n):
     return 'book/cover/%s.jpg' % i.slug

+

 def _cover_thumb_upload_to(i, n):
 def _cover_thumb_upload_to(i, n):

-    return 'book/cover_thumb/%s.jpg' % i.slug,
+    return 'book/cover_thumb/%s.jpg' % i.slug
+

 def _ebook_upload_to(upload_path):
     def _upload_to(i, n):
         return upload_path % i.slug
 def _ebook_upload_to(upload_path):
     def _upload_to(i, n):
         return upload_path % i.slug


@@ -38,7 +40,7 @@ def _ebook_upload_to(upload_path):
 
 class Book(models.Model):
     """Represents a book imported from WL-XML."""
 
 class Book(models.Model):
     """Represents a book imported from WL-XML."""

-    title         = models.CharField(_('title'), max_length=120)
+    title         = models.CharField(_('title'), max_length=32767)

     sort_key = models.CharField(_('sort key'), max_length=120, db_index=True, editable=False)
     sort_key_author = models.CharField(_('sort key by author'), max_length=120, db_index=True, editable=False, default=u'')
     slug = models.SlugField(_('slug'), max_length=120, db_index=True,
     sort_key = models.CharField(_('sort key'), max_length=120, db_index=True, editable=False)
     sort_key_author = models.CharField(_('sort key by author'), max_length=120, db_index=True, editable=False, default=u'')
     slug = models.SlugField(_('slug'), max_length=120, db_index=True,


@@ -60,7 +62,7 @@ class Book(models.Model):
             upload_to=_cover_upload_to,
             storage=bofh_storage, max_length=255)
     # Cleaner version of cover for thumbs
             upload_to=_cover_upload_to,
             storage=bofh_storage, max_length=255)
     # Cleaner version of cover for thumbs

-    cover_thumb = EbookField('cover_thumb', _('cover thumbnail'), 
+    cover_thumb = EbookField('cover_thumb', _('cover thumbnail'),

             null=True, blank=True,
             upload_to=_cover_thumb_upload_to,
             max_length=255)
             null=True, blank=True,
             upload_to=_cover_thumb_upload_to,
             max_length=255)


@@ -69,16 +71,19 @@ class Book(models.Model):
 
     parent = models.ForeignKey('self', blank=True, null=True,
         related_name='children')
 
     parent = models.ForeignKey('self', blank=True, null=True,
         related_name='children')

-
-    _related_info = jsonfield.JSONField(blank=True, null=True, editable=False)
+    ancestor = models.ManyToManyField('self', blank=True, null=True,
+        editable=False, related_name='descendant', symmetrical=False)

 
     objects  = models.Manager()
     tagged   = managers.ModelTaggedItemManager(Tag)
     tags     = managers.TagDescriptor(Tag)
 
     objects  = models.Manager()
     tagged   = managers.ModelTaggedItemManager(Tag)
     tags     = managers.TagDescriptor(Tag)

+    tag_relations = GenericRelation(Tag.intermediary_table_model)

 
     html_built = django.dispatch.Signal()
     published = django.dispatch.Signal()
 
 
     html_built = django.dispatch.Signal()
     published = django.dispatch.Signal()
 

+    short_html_url_name = 'catalogue_book_short'
+

     class AlreadyExists(Exception):
         pass
 
     class AlreadyExists(Exception):
         pass
 


@@ -91,16 +96,19 @@ class Book(models.Model):
     def __unicode__(self):
         return self.title
 
     def __unicode__(self):
         return self.title
 

-    def save(self, force_insert=False, force_update=False, reset_short_html=True, **kwargs):
+    def save(self, force_insert=False, force_update=False, **kwargs):

         from sortify import sortify
 
         from sortify import sortify
 

-        self.sort_key = sortify(self.title)
+        self.sort_key = sortify(self.title)[:120]

         self.title = unicode(self.title) # ???
 
         self.title = unicode(self.title) # ???
 

-        ret = super(Book, self).save(force_insert, force_update, **kwargs)
+        try:
+            author = self.tags.filter(category='author')[0].sort_key
+        except IndexError:
+            author = u''
+        self.sort_key_author = author

 
 

-        if reset_short_html:
-            self.reset_short_html()
+        ret = super(Book, self).save(force_insert, force_update, **kwargs)

 
         return ret
 
 
         return ret
 


@@ -123,18 +131,6 @@ class Book(models.Model):
     def language_name(self):
         return dict(settings.LANGUAGES).get(self.language_code(), "")
 
     def language_name(self):
         return dict(settings.LANGUAGES).get(self.language_code(), "")
 

-    def book_tag_slug(self):
-        return ('l-' + self.slug)[:120]
-
-    def book_tag(self):
-        slug = self.book_tag_slug()
-        book_tag, created = Tag.objects.get_or_create(slug=slug, category='book')
-        if created:
-            book_tag.name = self.title[:50]
-            book_tag.sort_key = self.title.lower()
-            book_tag.save()
-        return book_tag
-

     def has_media(self, type_):
         if type_ in Book.formats:
             return bool(getattr(self, "%s_file" % type_))
     def has_media(self, type_):
         if type_ in Book.formats:
             return bool(getattr(self, "%s_file" % type_))


@@ -159,23 +155,6 @@ class Book(models.Model):
     def get_daisy(self):
         return self.get_media("daisy")
 
     def get_daisy(self):
         return self.get_media("daisy")
 

-    def reset_short_html(self):
-        if self.id is None:
-            return
-
-        type(self).objects.filter(pk=self.pk).update(_related_info=None)
-        # Fragment.short_html relies on book's tags, so reset it here too
-        for fragm in self.fragments.all().iterator():
-            fragm.reset_short_html()
-
-        try:
-            author = self.tags.filter(category='author')[0].sort_key
-        except IndexError:
-            author = u''
-        type(self).objects.filter(pk=self.pk).update(sort_key_author=author)
-
-
-

     def has_description(self):
         return len(self.description) > 0
     has_description.short_description = _('description')
     def has_description(self):
         return len(self.description) > 0
     has_description.short_description = _('description')


@@ -231,9 +210,9 @@ class Book(models.Model):
         return create_zip(paths, "%s_%s" % (self.slug, format_))
 
     def search_index(self, book_info=None, index=None, index_tags=True, commit=True):
         return create_zip(paths, "%s_%s" % (self.slug, format_))
 
     def search_index(self, book_info=None, index=None, index_tags=True, commit=True):

-        import search

         if index is None:
         if index is None:

-            index = search.Index()
+            from search.index import Index
+            index = Index()

         try:
             index.index_book(self, book_info)
             if index_tags:
         try:
             index.index_book(self, book_info)
             if index_tags:


@@ -328,10 +307,11 @@ class Book(models.Model):
             child.parent = None
             child.parent_number = 0
             child.save()
             child.parent = None
             child.parent_number = 0
             child.save()

-            tasks.fix_tree_tags.delay(child)

             if old_cover:
                 notify_cover_changed.append(child)
 
             if old_cover:
                 notify_cover_changed.append(child)
 

+        cls.repopulate_ancestors()
+

         # No saves beyond this point.
 
         # Build cover.
         # No saves beyond this point.
 
         # Build cover.


@@ -355,46 +335,59 @@ class Book(models.Model):
         for child in notify_cover_changed:
             child.parent_cover_changed()
 
         for child in notify_cover_changed:
             child.parent_cover_changed()
 

-        cls.published.send(sender=book)
+        cls.published.send(sender=cls, instance=book)

         return book
 
         return book
 

-    def fix_tree_tags(self):
-        """Fixes the l-tags on the book's subtree.
-
-        Makes sure that:
-        * the book has its parents book-tags,
-        * its fragments have the book's and its parents book-tags,
-        * runs those for every child book too,
-        * touches all relevant tags,
-        * resets tag and theme counter on the book and its ancestry.
-        """
-        def fix_subtree(book, parent_tags):
-            affected_tags = set(book.tags)
-            book.tags = list(book.tags.exclude(category='book')) + parent_tags
-            sub_parent_tags = parent_tags + [book.book_tag()]
-            for frag in book.fragments.all():
-                affected_tags.update(frag.tags)
-                frag.tags = list(frag.tags.exclude(category='book')
-                                    ) + sub_parent_tags
-            for child in book.children.all():
-                affected_tags.update(fix_subtree(child, sub_parent_tags))
-            return affected_tags
-
-        parent_tags = []
-        parent = self.parent
-        while parent is not None:
-            parent_tags.append(parent.book_tag())
-            parent = parent.parent
-
-        affected_tags = fix_subtree(self, parent_tags)
-        for tag in affected_tags:
-            tasks.touch_tag(tag)
-
-        book = self
-        while book is not None:
-            book.reset_tag_counter()
-            book.reset_theme_counter()
-            book = book.parent
+    @classmethod
+    def repopulate_ancestors(cls):
+        """Fixes the ancestry cache."""
+        # TODO: table names
+        with transaction.atomic():
+            cursor = connection.cursor()
+            if connection.vendor == 'postgres':
+                cursor.execute("TRUNCATE catalogue_book_ancestor")
+                cursor.execute("""
+                    WITH RECURSIVE ancestry AS (
+                        SELECT book.id, book.parent_id
+                        FROM catalogue_book AS book
+                        WHERE book.parent_id IS NOT NULL
+                        UNION
+                        SELECT ancestor.id, book.parent_id
+                        FROM ancestry AS ancestor, catalogue_book AS book
+                        WHERE ancestor.parent_id = book.id
+                            AND book.parent_id IS NOT NULL
+                        )
+                    INSERT INTO catalogue_book_ancestor
+                        (from_book_id, to_book_id)
+                        SELECT id, parent_id
+                        FROM ancestry
+                        ORDER BY id;
+                    """)
+            else:
+                cursor.execute("DELETE FROM catalogue_book_ancestor")
+                for b in cls.objects.exclude(parent=None):
+                    parent = b.parent
+                    while parent is not None:
+                        b.ancestor.add(parent)
+                        parent = parent.parent
+
+    def flush_includes(self, languages=True):
+        if not languages:
+            return
+        if languages is True:
+            languages = [lc for (lc, _ln) in settings.LANGUAGES]
+        flush_ssi_includes([
+            template % (self.pk, lang)
+            for template in [
+                '/katalog/b/%d/mini.%s.html',
+                '/katalog/b/%d/mini_nolink.%s.html',
+                '/katalog/b/%d/short.%s.html',
+                '/katalog/b/%d/wide.%s.html',
+                '/api/include/book/%d.%s.json',
+                '/api/include/book/%d.%s.xml',
+                ]
+            for lang in languages
+            ])

 
     def cover_info(self, inherit=True):
         """Returns a dictionary to serve as fallback for BookInfo.
 
     def cover_info(self, inherit=True):
         """Returns a dictionary to serve as fallback for BookInfo.


@@ -415,6 +408,11 @@ class Book(models.Model):
             info = parent_info
         return info
 
             info = parent_info
         return info
 

+    def related_themes(self):
+        return Tag.objects.usage_for_queryset(
+            Fragment.objects.filter(models.Q(book=self) | models.Q(book__ancestor=self)),
+            counts=True).filter(category='theme')
+

     def parent_cover_changed(self):
         """Called when parent book's cover image is changed."""
         if not self.cover_info(inherit=False):
     def parent_cover_changed(self):
         """Called when parent book's cover image is changed."""
         if not self.cover_info(inherit=False):


@@ -431,116 +429,19 @@ class Book(models.Model):
         """Find other versions (i.e. in other languages) of the book."""
         return type(self).objects.filter(common_slug=self.common_slug).exclude(pk=self.pk)
 
         """Find other versions (i.e. in other languages) of the book."""
         return type(self).objects.filter(common_slug=self.common_slug).exclude(pk=self.pk)
 

-    def related_info(self):
-        """Keeps info about related objects (tags, media) in cache field."""
-        if self._related_info is not None:
-            return self._related_info
-        else:
-            rel = {'tags': {}, 'media': {}}
-
-            tags = self.tags.filter(category__in=(
-                    'author', 'kind', 'genre', 'epoch'))
-            tags = split_tags(tags)
-            for category in tags:
-                cat = []
-                for tag in tags[category]:
-                    tag_info = {'slug': tag.slug, 'name': tag.name}
-                    for lc, ln in settings.LANGUAGES:
-                        tag_name = getattr(tag, "name_%s" % lc)
-                        if tag_name:
-                            tag_info["name_%s" % lc] = tag_name
-                    cat.append(tag_info)
-                rel['tags'][category] = cat
-
-            for media_format in BookMedia.formats:
-                rel['media'][media_format] = self.has_media(media_format)
-
-            book = self
-            parents = []
-            while book.parent:
-                parents.append((book.parent.title, book.parent.slug))
-                book = book.parent
-            parents = parents[::-1]
-            if parents:
-                rel['parents'] = parents
-
-            if self.pk:
-                type(self).objects.filter(pk=self.pk).update(_related_info=rel)
-            return rel
-
-    def related_themes(self):
-        theme_counter = self.theme_counter
-        book_themes = list(Tag.objects.filter(pk__in=theme_counter.keys()))
-        for tag in book_themes:
-            tag.count = theme_counter[tag.pk]
-        return book_themes
-
-    def reset_tag_counter(self):
-        if self.id is None:
-            return
-
-        cache_key = "Book.tag_counter/%d" % self.id
-        permanent_cache.delete(cache_key)
-        if self.parent:
-            self.parent.reset_tag_counter()
-
-    @property
-    def tag_counter(self):
-        if self.id:
-            cache_key = "Book.tag_counter/%d" % self.id
-            tags = permanent_cache.get(cache_key)
-        else:
-            tags = None
-
-        if tags is None:
-            tags = {}
-            for child in self.children.all().order_by().iterator():
-                for tag_pk, value in child.tag_counter.iteritems():
-                    tags[tag_pk] = tags.get(tag_pk, 0) + value
-            for tag in self.tags.exclude(category__in=('book', 'theme', 'set')).order_by().iterator():
-                tags[tag.pk] = 1
-
-            if self.id:
-                permanent_cache.set(cache_key, tags)
-        return tags
-
-    def reset_theme_counter(self):
-        if self.id is None:
-            return
-
-        cache_key = "Book.theme_counter/%d" % self.id
-        permanent_cache.delete(cache_key)
-        if self.parent:
-            self.parent.reset_theme_counter()
-
-    @property
-    def theme_counter(self):
-        if self.id:
-            cache_key = "Book.theme_counter/%d" % self.id
-            tags = permanent_cache.get(cache_key)
-        else:
-            tags = None
-
-        if tags is None:
-            tags = {}
-            for fragment in Fragment.tagged.with_any([self.book_tag()]).order_by().iterator():
-                for tag in fragment.tags.filter(category='theme').order_by().iterator():
-                    tags[tag.pk] = tags.get(tag.pk, 0) + 1
-
-            if self.id:
-                permanent_cache.set(cache_key, tags)
-        return tags
+    def parents(self):
+        books = []
+        parent = self.parent
+        while parent is not None:
+            books.insert(0, parent)
+            parent = parent.parent
+        return books

 
     def pretty_title(self, html_links=False):
 
     def pretty_title(self, html_links=False):

-        book = self
-        rel_info = book.related_info()
-        names = [(related_tag_name(tag), Tag.create_url('author', tag['slug']))
-                    for tag in rel_info['tags'].get('author', ())]
-        if 'parents' in rel_info:
-            books = [(name, Book.create_url(slug))
-                        for name, slug in rel_info['parents']]
-            names.extend(reversed(books))
-        names.append((self.title, self.get_absolute_url()))
+        names = [(tag.name, tag.get_absolute_url())
+            for tag in self.tags.filter(category='author')]
+        books = self.parents() + [self]
+        names.extend([(b.title, b.get_absolute_url()) for b in books])

 
         if html_links:
             names = ['<a href="%s">%s</a>' % (tag[1], tag[0]) for tag in names]
 
         if html_links:
             names = ['<a href="%s">%s</a>' % (tag[1], tag[0]) for tag in names]


@@ -556,17 +457,8 @@ class Book(models.Model):
         also tagged with those tags.
 
         """
         also tagged with those tags.
 
         """

-        # get relevant books and their tags

         objects = cls.tagged.with_all(tags)
         objects = cls.tagged.with_all(tags)

-        parents = objects.exclude(children=None).only('slug')
-        # eliminate descendants
-        l_tags = Tag.objects.filter(category='book',
-            slug__in=[book.book_tag_slug() for book in parents.iterator()])
-        descendants_keys = [book.pk for book in cls.tagged.with_any(l_tags).only('pk').iterator()]
-        if descendants_keys:
-            objects = objects.exclude(pk__in=descendants_keys)
-
-        return objects
+        return objects.exclude(ancestor__in=objects)

 
     @classmethod
     def book_list(cls, filter=None):
 
     @classmethod
     def book_list(cls, filter=None):


@@ -593,7 +485,7 @@ class Book(models.Model):
                 books_by_parent.setdefault(book.parent_id, []).append(book)
 
         orphans = []
                 books_by_parent.setdefault(book.parent_id, []).append(book)
 
         orphans = []

-        books_by_author = SortedDict()
+        books_by_author = OrderedDict()

         for tag in Tag.objects.filter(category='author').iterator():
             books_by_author[tag] = []
 
         for tag in Tag.objects.filter(category='author').iterator():
             books_by_author[tag] = []
 


@@ -630,10 +522,13 @@ class Book(models.Model):
             return None, None
 
     def choose_fragment(self):
             return None, None
 
     def choose_fragment(self):

-        tag = self.book_tag()
-        fragments = Fragment.tagged.with_any([tag])
-        if fragments.exists():
-            return fragments.order_by('?')[0]
+        fragments = self.fragments.order_by()
+        fragments_count = fragments.count()
+        if not fragments_count and self.children.exists():
+            fragments = Fragment.objects.filter(book__ancestor=self).order_by()
+            fragments_count = fragments.count()
+        if fragments_count:
+            return fragments[randint(0, fragments_count - 1)]

         elif self.parent:
             return self.parent.choose_fragment()
         else:
         elif self.parent:
             return self.parent.choose_fragment()
         else:


@@ -643,9 +538,13 @@ class Book(models.Model):
 # add the file fields
 for format_ in Book.formats:
     field_name = "%s_file" % format_
 # add the file fields
 for format_ in Book.formats:
     field_name = "%s_file" % format_

-    upload_to = _ebook_upload_to('book/%s/%%s.%s' % (format_, format_))
+    # This weird globals() assignment makes Django migrations comfortable.
+    _upload_to = _ebook_upload_to('book/%s/%%s.%s' % (format_, format_))
+    _upload_to.__name__ = '_%s_upload_to' % format_
+    globals()[_upload_to.__name__] = _upload_to
+

     EbookField(format_, _("%s file" % format_.upper()),
     EbookField(format_, _("%s file" % format_.upper()),

-        upload_to=upload_to,
+        upload_to=_upload_to,

         storage=bofh_storage,
         max_length=255,
         blank=True,
         storage=bofh_storage,
         max_length=255,
         blank=True,