rework book box caching

[wolnelektury.git] / src / catalogue / models / book.py

diff --git a/src/catalogue/models/book.py b/src/catalogue/models/book.py
index 8dc39fe..7cd6386 100644 (file)
--- a/src/catalogue/models/book.py
+++ b/src/catalogue/models/book.py
@@ -3,6 +3,7 @@
 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
 #
 from collections import OrderedDict
 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
 #
 from collections import OrderedDict

+from datetime import date, timedelta

 from random import randint
 import os.path
 import re
 from random import randint
 import os.path
 import re


@@ -14,14 +15,17 @@ import django.dispatch
 from django.contrib.contenttypes.fields import GenericRelation
 from django.core.urlresolvers import reverse
 from django.utils.translation import ugettext_lazy as _, get_language
 from django.contrib.contenttypes.fields import GenericRelation
 from django.core.urlresolvers import reverse
 from django.utils.translation import ugettext_lazy as _, get_language

+from django.utils.deconstruct import deconstructible

 import jsonfield
 from fnpdjango.storage import BofhFileSystemStorage
 from ssify import flush_ssi_includes
 import jsonfield
 from fnpdjango.storage import BofhFileSystemStorage
 from ssify import flush_ssi_includes

+
+from librarian.html import transform_abstrakt

 from newtagging import managers
 from catalogue import constants
 from catalogue.fields import EbookField
 from catalogue.models import Tag, Fragment, BookMedia
 from newtagging import managers
 from catalogue import constants
 from catalogue.fields import EbookField
 from catalogue.models import Tag, Fragment, BookMedia

-from catalogue.utils import create_zip, gallery_url, gallery_path
+from catalogue.utils import create_zip, gallery_url, gallery_path, split_tags

 from catalogue.models.tag import prefetched_relations
 from catalogue import app_settings
 from catalogue import tasks
 from catalogue.models.tag import prefetched_relations
 from catalogue import app_settings
 from catalogue import tasks


@@ -30,18 +34,23 @@ from wolnelektury.utils import makedirs
 bofh_storage = BofhFileSystemStorage()
 
 
 bofh_storage = BofhFileSystemStorage()
 
 

-def _make_upload_to(path):
-    def _upload_to(i, n):
-        return path % i.slug
-    return _upload_to
+@deconstructible
+class UploadToPath(object):
+    def __init__(self, path):
+        self.path = path
+
+    def __call__(self, instance, filename):
+        return self.path % instance.slug

 
 
 
 

-_cover_upload_to = _make_upload_to('book/cover/%s.jpg')
-_cover_thumb_upload_to = _make_upload_to('book/cover_thumb/%s.jpg')
+_cover_upload_to = UploadToPath('book/cover/%s.jpg')
+_cover_thumb_upload_to = UploadToPath('book/cover_thumb/%s.jpg')
+_cover_api_thumb_upload_to = UploadToPath('book/cover_api_thumb/%s.jpg')
+_simple_cover_upload_to = UploadToPath('book/cover_simple/%s.jpg')

 
 
 def _ebook_upload_to(upload_path):
 
 
 def _ebook_upload_to(upload_path):

-    return _make_upload_to(upload_path)
+    return UploadToPath(upload_path)

 
 
 class Book(models.Model):
 
 
 class Book(models.Model):


@@ -54,13 +63,18 @@ class Book(models.Model):
     common_slug = models.SlugField(_('slug'), max_length=120, db_index=True)
     language = models.CharField(_('language code'), max_length=3, db_index=True, default=app_settings.DEFAULT_LANGUAGE)
     description = models.TextField(_('description'), blank=True)
     common_slug = models.SlugField(_('slug'), max_length=120, db_index=True)
     language = models.CharField(_('language code'), max_length=3, db_index=True, default=app_settings.DEFAULT_LANGUAGE)
     description = models.TextField(_('description'), blank=True)

+    abstract = models.TextField(_('abstract'), blank=True)

     created_at = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True)
     created_at = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True)

-    changed_at = models.DateTimeField(_('creation date'), auto_now=True, db_index=True)
+    changed_at = models.DateTimeField(_('change date'), auto_now=True, db_index=True)

     parent_number = models.IntegerField(_('parent number'), default=0)
     extra_info = jsonfield.JSONField(_('extra information'), default={})
     gazeta_link = models.CharField(blank=True, max_length=240)
     wiki_link = models.CharField(blank=True, max_length=240)
     print_on_demand = models.BooleanField(_('print on demand'), default=False)
     parent_number = models.IntegerField(_('parent number'), default=0)
     extra_info = jsonfield.JSONField(_('extra information'), default={})
     gazeta_link = models.CharField(blank=True, max_length=240)
     wiki_link = models.CharField(blank=True, max_length=240)
     print_on_demand = models.BooleanField(_('print on demand'), default=False)

+    recommended = models.BooleanField(_('recommended'), default=False)
+    audio_length = models.CharField(_('audio length'), blank=True, max_length=8)
+    preview = models.BooleanField(_('preview'), default=False)
+    preview_until = models.DateField(_('preview until'), blank=True, null=True)

 
     # files generated during publication
     cover = EbookField(
 
     # files generated during publication
     cover = EbookField(


@@ -74,12 +88,25 @@ class Book(models.Model):
         null=True, blank=True,
         upload_to=_cover_thumb_upload_to,
         max_length=255)
         null=True, blank=True,
         upload_to=_cover_thumb_upload_to,
         max_length=255)

+    cover_api_thumb = EbookField(
+        'cover_api_thumb', _('cover thumbnail for mobile app'),
+        null=True, blank=True,
+        upload_to=_cover_api_thumb_upload_to,
+        max_length=255)
+    simple_cover = EbookField(
+        'simple_cover', _('cover for mobile app'),
+        null=True, blank=True,
+        upload_to=_simple_cover_upload_to,
+        max_length=255)

     ebook_formats = constants.EBOOK_FORMATS
     formats = ebook_formats + ['html', 'xml']
 
     parent = models.ForeignKey('self', blank=True, null=True, related_name='children')
     ancestor = models.ManyToManyField('self', blank=True, editable=False, related_name='descendant', symmetrical=False)
 
     ebook_formats = constants.EBOOK_FORMATS
     formats = ebook_formats + ['html', 'xml']
 
     parent = models.ForeignKey('self', blank=True, null=True, related_name='children')
     ancestor = models.ManyToManyField('self', blank=True, editable=False, related_name='descendant', symmetrical=False)
 

+    cached_author = models.CharField(blank=True, max_length=240, db_index=True)
+    has_audience = models.BooleanField(default=False)
+

     objects = models.Manager()
     tagged = managers.ModelTaggedItemManager(Tag)
     tags = managers.TagDescriptor(Tag)
     objects = models.Manager()
     tagged = managers.ModelTaggedItemManager(Tag)
     tags = managers.TagDescriptor(Tag)


@@ -118,8 +145,11 @@ class Book(models.Model):
         else:
             return ', '.join(self.tags.filter(category=category).values_list('name', flat=True))
 
         else:
             return ', '.join(self.tags.filter(category=category).values_list('name', flat=True))
 

+    def tags_by_category(self):
+        return split_tags(self.tags.exclude(category__in=('set', 'theme')))
+

     def author_unicode(self):
     def author_unicode(self):

-        return self.tag_unicode('author')
+        return self.cached_author

 
     def translator(self):
         translators = self.extra_info.get('translators')
 
     def translator(self):
         translators = self.extra_info.get('translators')


@@ -132,6 +162,9 @@ class Book(models.Model):
             others = ''
         return ', '.join(u'\xa0'.join(reversed(translator.split(', ', 1))) for translator in translators) + others
 
             others = ''
         return ', '.join(u'\xa0'.join(reversed(translator.split(', ', 1))) for translator in translators) + others
 

+    def cover_source(self):
+        return self.extra_info.get('cover_source', self.parent.cover_source() if self.parent else '')
+

     def save(self, force_insert=False, force_update=False, **kwargs):
         from sortify import sortify
 
     def save(self, force_insert=False, force_update=False, **kwargs):
         from sortify import sortify
 


@@ -144,6 +177,9 @@ class Book(models.Model):
             author = u''
         self.sort_key_author = author
 
             author = u''
         self.sort_key_author = author
 

+        self.cached_author = self.tag_unicode('author')
+        self.has_audience = 'audience' in self.extra_info
+

         ret = super(Book, self).save(force_insert, force_update, **kwargs)
 
         return ret
         ret = super(Book, self).save(force_insert, force_update, **kwargs)
 
         return ret


@@ -176,12 +212,41 @@ class Book(models.Model):
     def is_foreign(self):
         return self.language_code() != settings.LANGUAGE_CODE
 
     def is_foreign(self):
         return self.language_code() != settings.LANGUAGE_CODE
 

+    def set_audio_length(self):
+        length = self.get_audio_length()
+        if length > 0:
+            self.audio_length = self.format_audio_length(length)
+            self.save()
+
+    @staticmethod
+    def format_audio_length(seconds):
+        if seconds < 60*60:
+            minutes = seconds // 60
+            seconds = seconds % 60
+            return '%d:%02d' % (minutes, seconds)
+        else:
+            hours = seconds // 3600
+            minutes = seconds % 3600 // 60
+            seconds = seconds % 60
+            return '%d:%02d:%02d' % (hours, minutes, seconds)
+
+    def get_audio_length(self):
+        from mutagen.mp3 import MP3
+        total = 0
+        for media in self.get_mp3() or ():
+            audio = MP3(media.file.path)
+            total += audio.info.length
+        return int(total)
+

     def has_media(self, type_):
         if type_ in Book.formats:
             return bool(getattr(self, "%s_file" % type_))
         else:
             return self.media.filter(type=type_).exists()
 
     def has_media(self, type_):
         if type_ in Book.formats:
             return bool(getattr(self, "%s_file" % type_))
         else:
             return self.media.filter(type=type_).exists()
 

+    def has_audio(self):
+        return self.has_media('mp3')
+

     def get_media(self, type_):
         if self.has_media(type_):
             if type_ in Book.formats:
     def get_media(self, type_):
         if self.has_media(type_):
             if type_ in Book.formats:


@@ -203,27 +268,84 @@ class Book(models.Model):
     def get_daisy(self):
         return self.get_media("daisy")
 
     def get_daisy(self):
         return self.get_media("daisy")
 

+    def media_url(self, format_):
+        media = self.get_media(format_)
+        if media:
+            if self.preview:
+                return reverse('embargo_link', kwargs={'slug': self.slug, 'format_': format_})
+            else:
+                return media.url
+        else:
+            return None
+
+    def html_url(self):
+        return self.media_url('html')
+
+    def pdf_url(self):
+        return self.media_url('pdf')
+
+    def epub_url(self):
+        return self.media_url('epub')
+
+    def mobi_url(self):
+        return self.media_url('mobi')
+
+    def txt_url(self):
+        return self.media_url('txt')
+
+    def fb2_url(self):
+        return self.media_url('fb2')
+
+    def xml_url(self):
+        return self.media_url('xml')
+

     def has_description(self):
         return len(self.description) > 0
     has_description.short_description = _('description')
     has_description.boolean = True
 
     def has_description(self):
         return len(self.description) > 0
     has_description.short_description = _('description')
     has_description.boolean = True
 

-    # ugly ugly ugly

     def has_mp3_file(self):
     def has_mp3_file(self):

-        return bool(self.has_media("mp3"))
+        return self.has_media("mp3")

     has_mp3_file.short_description = 'MP3'
     has_mp3_file.boolean = True
 
     def has_ogg_file(self):
     has_mp3_file.short_description = 'MP3'
     has_mp3_file.boolean = True
 
     def has_ogg_file(self):

-        return bool(self.has_media("ogg"))
+        return self.has_media("ogg")

     has_ogg_file.short_description = 'OGG'
     has_ogg_file.boolean = True
 
     def has_daisy_file(self):
     has_ogg_file.short_description = 'OGG'
     has_ogg_file.boolean = True
 
     def has_daisy_file(self):

-        return bool(self.has_media("daisy"))
+        return self.has_media("daisy")

     has_daisy_file.short_description = 'DAISY'
     has_daisy_file.boolean = True
 
     has_daisy_file.short_description = 'DAISY'
     has_daisy_file.boolean = True
 

+    def get_audiobooks(self):
+        ogg_files = {}
+        for m in self.media.filter(type='ogg').order_by().iterator():
+            ogg_files[m.name] = m
+
+        audiobooks = []
+        projects = set()
+        for mp3 in self.media.filter(type='mp3').iterator():
+            # ogg files are always from the same project
+            meta = mp3.extra_info
+            project = meta.get('project')
+            if not project:
+                # temporary fallback
+                project = u'CzytamySłuchając'
+
+            projects.add((project, meta.get('funded_by', '')))
+
+            media = {'mp3': mp3}
+
+            ogg = ogg_files.get(mp3.name)
+            if ogg:
+                media['ogg'] = ogg
+            audiobooks.append(media)
+
+        projects = sorted(projects)
+        return audiobooks, projects
+

     def wldocument(self, parse_dublincore=True, inherit=True):
         from catalogue.import_utils import ORMDocProvider
         from librarian.parser import WLDocument
     def wldocument(self, parse_dublincore=True, inherit=True):
         from catalogue.import_utils import ORMDocProvider
         from librarian.parser import WLDocument


@@ -248,7 +370,7 @@ class Book(models.Model):
                 format_)
 
         field_name = "%s_file" % format_
                 format_)
 
         field_name = "%s_file" % format_

-        books = Book.objects.filter(parent=None).exclude(**{field_name: ""})
+        books = Book.objects.filter(parent=None).exclude(**{field_name: ""}).exclude(preview=True)

         paths = [(pretty_file_name(b), getattr(b, field_name).path) for b in books.iterator()]
         return create_zip(paths, app_settings.FORMAT_ZIPS[format_])
 
         paths = [(pretty_file_name(b), getattr(b, field_name).path) for b in books.iterator()]
         return create_zip(paths, app_settings.FORMAT_ZIPS[format_])
 


@@ -271,6 +393,7 @@ class Book(models.Model):
             index.index.rollback()
             raise e
 
             index.index.rollback()
             raise e
 

+    # will make problems in conjunction with paid previews

     def download_pictures(self, remote_gallery_url):
         gallery_path = self.gallery_path()
         # delete previous files, so we don't include old files in ebooks
     def download_pictures(self, remote_gallery_url):
         gallery_path = self.gallery_path()
         # delete previous files, so we don't include old files in ebooks


@@ -286,6 +409,13 @@ class Book(models.Model):
                 ilustr_path = os.path.join(gallery_path, ilustr_src)
                 urllib.urlretrieve('%s/%s' % (remote_gallery_url, ilustr_src), ilustr_path)
 
                 ilustr_path = os.path.join(gallery_path, ilustr_src)
                 urllib.urlretrieve('%s/%s' % (remote_gallery_url, ilustr_src), ilustr_path)
 

+    def load_abstract(self):
+        abstract = self.wldocument(parse_dublincore=False).edoc.getroot().find('.//abstrakt')
+        if abstract is not None:
+            self.abstract = transform_abstrakt(abstract)
+        else:
+            self.abstract = ''
+

     @classmethod
     def from_xml_file(cls, xml_file, **kwargs):
         from django.core.files import File
     @classmethod
     def from_xml_file(cls, xml_file, **kwargs):
         from django.core.files import File


@@ -304,7 +434,7 @@ class Book(models.Model):
 
     @classmethod
     def from_text_and_meta(cls, raw_file, book_info, overwrite=False, dont_build=None, search_index=True,
 
     @classmethod
     def from_text_and_meta(cls, raw_file, book_info, overwrite=False, dont_build=None, search_index=True,

-                           search_index_tags=True, remote_gallery_url=None):
+                           search_index_tags=True, remote_gallery_url=None, days=0):

         if dont_build is None:
             dont_build = set()
         dont_build = set.union(set(dont_build), set(app_settings.DONT_BUILD))
         if dont_build is None:
             dont_build = set()
         dont_build = set.union(set(dont_build), set(app_settings.DONT_BUILD))


@@ -327,6 +457,9 @@ class Book(models.Model):
         if created:
             book_shelves = []
             old_cover = None
         if created:
             book_shelves = []
             old_cover = None

+            book.preview = bool(days)
+            if book.preview:
+                book.preview_until = date.today() + timedelta(days)

         else:
             if not overwrite:
                 raise Book.AlreadyExists(_('Book %s already exists') % book_slug)
         else:
             if not overwrite:
                 raise Book.AlreadyExists(_('Book %s already exists') % book_slug)


@@ -336,6 +469,8 @@ class Book(models.Model):
 
         # Save XML file
         book.xml_file.save('%s.xml' % book.slug, raw_file, save=False)
 
         # Save XML file
         book.xml_file.save('%s.xml' % book.slug, raw_file, save=False)

+        if book.preview:
+            book.xml_file.set_readable(False)

 
         book.language = book_info.language
         book.title = book_info.title
 
         book.language = book_info.language
         book.title = book_info.title


@@ -344,10 +479,16 @@ class Book(models.Model):
         else:
             book.common_slug = book.slug
         book.extra_info = book_info.to_dict()
         else:
             book.common_slug = book.slug
         book.extra_info = book_info.to_dict()

+        book.load_abstract()

         book.save()
 
         meta_tags = Tag.tags_from_info(book_info)
 
         book.save()
 
         meta_tags = Tag.tags_from_info(book_info)
 

+        for tag in meta_tags:
+            if not tag.for_books:
+                tag.for_books = True
+                tag.save()
+

         book.tags = set(meta_tags + book_shelves)
 
         cover_changed = old_cover != book.cover_info()
         book.tags = set(meta_tags + book_shelves)
 
         cover_changed = old_cover != book.cover_info()


@@ -381,6 +522,8 @@ class Book(models.Model):
         if 'cover' not in dont_build:
             book.cover.build_delay()
             book.cover_thumb.build_delay()
         if 'cover' not in dont_build:
             book.cover.build_delay()
             book.cover_thumb.build_delay()

+            book.cover_api_thumb.build_delay()
+            book.simple_cover.build_delay()

 
         # Build HTML and ebooks.
         book.html_file.build_delay()
 
         # Build HTML and ebooks.
         book.html_file.build_delay()


@@ -399,6 +542,7 @@ class Book(models.Model):
             child.parent_cover_changed()
 
         book.save()  # update sort_key_author
             child.parent_cover_changed()
 
         book.save()  # update sort_key_author

+        book.update_popularity()

         cls.published.send(sender=cls, instance=book)
         return book
 
         cls.published.send(sender=cls, instance=book)
         return book
 


@@ -483,6 +627,8 @@ class Book(models.Model):
             if 'cover' not in app_settings.DONT_BUILD:
                 self.cover.build_delay()
                 self.cover_thumb.build_delay()
             if 'cover' not in app_settings.DONT_BUILD:
                 self.cover.build_delay()
                 self.cover_thumb.build_delay()

+                self.cover_api_thumb.build_delay()
+                self.simple_cover.build_delay()

             for format_ in constants.EBOOK_FORMATS_WITH_COVERS:
                 if format_ not in app_settings.DONT_BUILD:
                     getattr(self, '%s_file' % format_).build_delay()
             for format_ in constants.EBOOK_FORMATS_WITH_COVERS:
                 if format_ not in app_settings.DONT_BUILD:
                     getattr(self, '%s_file' % format_).build_delay()


@@ -605,6 +751,13 @@ class Book(models.Model):
         else:
             return None
 
         else:
             return None
 

+    def fragment_data(self):
+        fragment = self.choose_fragment()
+        if fragment:
+            return {'title': fragment.book.pretty_title(), 'html': fragment.get_short_text()}
+        else:
+            return None
+

     def update_popularity(self):
         count = self.tags.filter(category='set').values('user').order_by('user').distinct().count()
         try:
     def update_popularity(self):
         count = self.tags.filter(category='set').values('user').order_by('user').distinct().count()
         try:


@@ -635,9 +788,10 @@ def add_file_fields():
             default=''
         ).contribute_to_class(Book, field_name)
 
             default=''
         ).contribute_to_class(Book, field_name)
 

+

 add_file_fields()
 
 
 class BookPopularity(models.Model):
     book = models.OneToOneField(Book, related_name='popularity')
 add_file_fields()
 
 
 class BookPopularity(models.Model):
     book = models.OneToOneField(Book, related_name='popularity')

-    count = models.IntegerField(default=0)
+    count = models.IntegerField(default=0, db_index=True)