fix

[wolnelektury.git] / src / catalogue / fields.py

diff --git a/src/catalogue/fields.py b/src/catalogue/fields.py
index 92e8de4..49fd46e 100644 (file)
--- a/src/catalogue/fields.py
+++ b/src/catalogue/fields.py
@@ -1,21 +1,25 @@
-# -*- coding: utf-8 -*-

 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
 #
 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
 #

+import os

 from django.conf import settings
 from django.core.files import File
 from django.core.files.storage import FileSystemStorage
 from django.db import models
 from django.db.models.fields.files import FieldFile
 from catalogue import app_settings
 from django.conf import settings
 from django.core.files import File
 from django.core.files.storage import FileSystemStorage
 from django.db import models
 from django.db.models.fields.files import FieldFile
 from catalogue import app_settings

-from catalogue.constants import LANGUAGES_3TO2
-from catalogue.utils import remove_zip, truncate_html_words, gallery_path, gallery_url
+from catalogue.constants import LANGUAGES_3TO2, EBOOK_FORMATS_WITH_CHILDREN, EBOOK_FORMATS_WITHOUT_CHILDREN
+from catalogue.utils import absolute_url, remove_zip, truncate_html_words, gallery_path, gallery_url

 from celery.task import Task, task
 from celery.utils.log import get_task_logger
 from waiter.utils import clear_cache
 
 task_logger = get_task_logger(__name__)
 
 from celery.task import Task, task
 from celery.utils.log import get_task_logger
 from waiter.utils import clear_cache
 
 task_logger = get_task_logger(__name__)
 

+ETAG_SCHEDULED_SUFFIX = '-scheduled'
+EBOOK_BUILD_PRIORITY = 0
+EBOOK_REBUILD_PRIORITY = 9
+

 
 class EbookFieldFile(FieldFile):
     """Represents contents of an ebook file field."""
 
 class EbookFieldFile(FieldFile):
     """Represents contents of an ebook file field."""


@@ -24,9 +28,15 @@ class EbookFieldFile(FieldFile):
         """Build the ebook immediately."""
         return self.field.builder.build(self)
 
         """Build the ebook immediately."""
         return self.field.builder.build(self)
 

-    def build_delay(self):
+    def build_delay(self, priority=EBOOK_BUILD_PRIORITY):

         """Builds the ebook in a delayed task."""
         """Builds the ebook in a delayed task."""

-        return self.field.builder.delay(self.instance, self.field.attname)
+        self.update_etag(
+            "".join([self.field.get_current_etag(), ETAG_SCHEDULED_SUFFIX])
+        )
+        return self.field.builder.apply_async(
+            [self.instance, self.field.attname],
+            priority=priority
+        )

 
     def get_url(self):
         return self.instance.media_url(self.field.attname.split('_')[0])
 
     def get_url(self):
         return self.instance.media_url(self.field.attname.split('_')[0])


@@ -36,10 +46,16 @@ class EbookFieldFile(FieldFile):
         permissions = 0o644 if readable else 0o600
         os.chmod(self.path, permissions)
 
         permissions = 0o644 if readable else 0o600
         os.chmod(self.path, permissions)
 

+    def update_etag(self, etag):
+        setattr(self.instance, self.field.etag_field_name, etag)
+        if self.instance.pk:
+            self.instance.save(update_fields=[self.field.etag_field_name])
+

 
 class EbookField(models.FileField):
     """Represents an ebook file field, attachable to a model."""
     attr_class = EbookFieldFile
 
 class EbookField(models.FileField):
     """Represents an ebook file field, attachable to a model."""
     attr_class = EbookFieldFile

+    registry = []

 
     def __init__(self, format_name, *args, **kwargs):
         super(EbookField, self).__init__(*args, **kwargs)
 
     def __init__(self, format_name, *args, **kwargs):
         super(EbookField, self).__init__(*args, **kwargs)


@@ -58,16 +74,58 @@ class EbookField(models.FileField):
     def contribute_to_class(self, cls, name):
         super(EbookField, self).contribute_to_class(cls, name)
 
     def contribute_to_class(self, cls, name):
         super(EbookField, self).contribute_to_class(cls, name)
 

+        self.etag_field_name = f'{name}_etag'
+

         def has(model_instance):
             return bool(getattr(model_instance, self.attname, None))
         has.__doc__ = None
         has.__name__ = str("has_%s" % self.attname)
         has.short_description = self.name
         has.boolean = True
         def has(model_instance):
             return bool(getattr(model_instance, self.attname, None))
         has.__doc__ = None
         has.__name__ = str("has_%s" % self.attname)
         has.short_description = self.name
         has.boolean = True

+
+        self.registry.append(self)
+

         setattr(cls, 'has_%s' % self.attname, has)
 
         setattr(cls, 'has_%s' % self.attname, has)
 

+    def get_current_etag(self):
+        import pkg_resources
+        librarian_version = pkg_resources.get_distribution("librarian").version
+        return librarian_version
+
+    def schedule_stale(self, queryset=None):
+        """Schedule building this format for all the books where etag is stale."""
+        # If there is not ETag field, bail. That's true for xml file field.
+        if not hasattr(self.model, f'{self.attname}_etag'):
+            return
+
+        etag = self.get_current_etag()
+        if queryset is None:
+            queryset = self.model.objects.all()
+
+        if self.format_name in EBOOK_FORMATS_WITHOUT_CHILDREN + ['html']:
+            queryset = queryset.filter(children=None)
+
+        queryset = queryset.exclude(**{
+            f'{self.etag_field_name}__in': [
+                etag, f'{etag}{ETAG_SCHEDULED_SUFFIX}'
+            ]
+        })
+        for obj in queryset:
+            fieldfile = getattr(obj, self.attname)
+            priority = EBOOK_REBUILD_PRIORITY if fieldfile else EBOOK_BUILD_PRIORITY
+            fieldfile.build_delay(priority=priority)
+
+    @classmethod
+    def schedule_all_stale(cls):
+        """Schedules all stale ebooks of all formats to rebuild."""
+        for field in cls.registry:
+            field.schedule_stale()
+
+

 
 class BuildEbook(Task):
 
 class BuildEbook(Task):

+    librarian2_api = False
+

     formats = {}
 
     @classmethod
     formats = {}
 
     @classmethod


@@ -94,9 +152,14 @@ class BuildEbook(Task):
 
     def run(self, obj, field_name):
         """Just run `build` on FieldFile, can't pass it directly to Celery."""
 
     def run(self, obj, field_name):
         """Just run `build` on FieldFile, can't pass it directly to Celery."""

-        task_logger.info("%s -> %s" % (obj.slug, field_name))
+        fieldfile = getattr(obj, field_name)
+
+        # Get etag value before actually building the file.
+        etag = fieldfile.field.get_current_etag()
+        task_logger.info("%s -> %s@%s" % (obj.slug, field_name, etag))

         ret = self.build(getattr(obj, field_name))
         ret = self.build(getattr(obj, field_name))

-        obj.flush_includes()
+        fieldfile.update_etag(etag)
+        obj.clear_cache()

         return ret
 
     def set_file_permissions(self, fieldfile):
         return ret
 
     def set_file_permissions(self, fieldfile):


@@ -105,16 +168,13 @@ class BuildEbook(Task):
 
     def build(self, fieldfile):
         book = fieldfile.instance
 
     def build(self, fieldfile):
         book = fieldfile.instance

-        out = self.transform(book.wldocument(), fieldfile)
-        fieldfile.save(None, File(open(out.get_filename())), save=False)
+        out = self.transform(
+            book.wldocument2() if self.librarian2_api else book.wldocument(),
+            fieldfile)
+        fieldfile.save(None, File(open(out.get_filename(), 'rb')), save=False)

         self.set_file_permissions(fieldfile)
         if book.pk is not None:
         self.set_file_permissions(fieldfile)
         if book.pk is not None:

-            books = type(book).objects.filter(pk=book.pk)
-            books.update(**{
-                fieldfile.field.attname: fieldfile
-            })
-            for book in books:
-                book.save()  # just to trigger post-save
+            book.save(update_fields=[fieldfile.field.attname])

         if fieldfile.field.format_name in app_settings.FORMAT_ZIPS:
             remove_zip(app_settings.FORMAT_ZIPS[fieldfile.field.format_name])
 # Don't decorate BuildEbook, because we want to subclass it.
         if fieldfile.field.format_name in app_settings.FORMAT_ZIPS:
             remove_zip(app_settings.FORMAT_ZIPS[fieldfile.field.format_name])
 # Don't decorate BuildEbook, because we want to subclass it.


@@ -134,8 +194,9 @@ class BuildTxt(BuildEbook):
 class BuildPdf(BuildEbook):
     @staticmethod
     def transform(wldoc, fieldfile):
 class BuildPdf(BuildEbook):
     @staticmethod
     def transform(wldoc, fieldfile):

-        return wldoc.as_pdf(morefloats=settings.LIBRARIAN_PDF_MOREFLOATS, cover=True,
-                            ilustr_path=gallery_path(wldoc.book_info.url.slug))
+        return wldoc.as_pdf(
+            morefloats=settings.LIBRARIAN_PDF_MOREFLOATS, cover=True,
+            base_url=absolute_url(gallery_url(wldoc.book_info.url.slug)), customizations=['notoc'])

 
     def build(self, fieldfile):
         BuildEbook.build(self, fieldfile)
 
     def build(self, fieldfile):
         BuildEbook.build(self, fieldfile)


@@ -145,17 +206,29 @@ class BuildPdf(BuildEbook):
 @BuildEbook.register('epub')
 @task(ignore_result=True)
 class BuildEpub(BuildEbook):
 @BuildEbook.register('epub')
 @task(ignore_result=True)
 class BuildEpub(BuildEbook):

+    librarian2_api = True
+

     @staticmethod
     def transform(wldoc, fieldfile):
     @staticmethod
     def transform(wldoc, fieldfile):

-        return wldoc.as_epub(cover=True, ilustr_path=gallery_path(wldoc.book_info.url.slug))
+        from librarian.builders import EpubBuilder
+        return EpubBuilder(
+                base_url='file://' + os.path.abspath(gallery_path(wldoc.meta.url.slug)) + '/',
+                fundraising=settings.EPUB_FUNDRAISING
+            ).build(wldoc)

 
 
 @BuildEbook.register('mobi')
 @task(ignore_result=True)
 class BuildMobi(BuildEbook):
 
 
 @BuildEbook.register('mobi')
 @task(ignore_result=True)
 class BuildMobi(BuildEbook):

+    librarian2_api = True
+

     @staticmethod
     def transform(wldoc, fieldfile):
     @staticmethod
     def transform(wldoc, fieldfile):

-        return wldoc.as_mobi(cover=True, ilustr_path=gallery_path(wldoc.book_info.url.slug))
+        from librarian.builders import MobiBuilder
+        return MobiBuilder(
+                base_url='file://' + os.path.abspath(gallery_path(wldoc.meta.url.slug)) + '/',
+                fundraising=settings.EPUB_FUNDRAISING
+            ).build(wldoc)

 
 
 @BuildEbook.register('html')
 
 
 @BuildEbook.register('html')


@@ -163,7 +236,7 @@ class BuildMobi(BuildEbook):
 class BuildHtml(BuildEbook):
     def build(self, fieldfile):
         from django.core.files.base import ContentFile
 class BuildHtml(BuildEbook):
     def build(self, fieldfile):
         from django.core.files.base import ContentFile

-        from fnpdjango.utils.text.slughifi import slughifi
+        from slugify import slugify

         from sortify import sortify
         from librarian import html
         from catalogue.models import Fragment, Tag
         from sortify import sortify
         from librarian import html
         from catalogue.models import Fragment, Tag


@@ -184,7 +257,7 @@ class BuildHtml(BuildEbook):
             if lang not in [ln[0] for ln in settings.LANGUAGES]:
                 lang = None
 
             if lang not in [ln[0] for ln in settings.LANGUAGES]:
                 lang = None
 

-            fieldfile.save(None, ContentFile(html_output.get_string()), save=False)
+            fieldfile.save(None, ContentFile(html_output.get_bytes()), save=False)

             self.set_file_permissions(fieldfile)
             type(book).objects.filter(pk=book.pk).update(**{
                 fieldfile.field.attname: fieldfile
             self.set_file_permissions(fieldfile)
             type(book).objects.filter(pk=book.pk).update(**{
                 fieldfile.field.attname: fieldfile


@@ -204,8 +277,9 @@ class BuildHtml(BuildEbook):
                     if lang == settings.LANGUAGE_CODE:
                         # Allow creating themes if book in default language.
                         tag, created = Tag.objects.get_or_create(
                     if lang == settings.LANGUAGE_CODE:
                         # Allow creating themes if book in default language.
                         tag, created = Tag.objects.get_or_create(

-                                            slug=slughifi(theme_name),
-                                            category='theme')
+                            slug=slugify(theme_name),
+                            category='theme'
+                        )

                         if created:
                             tag.name = theme_name
                             setattr(tag, "name_%s" % lang, theme_name)
                         if created:
                             tag.name = theme_name
                             setattr(tag, "name_%s" % lang, theme_name)


@@ -216,7 +290,10 @@ class BuildHtml(BuildEbook):
                     elif lang is not None:
                         # Don't create unknown themes in non-default languages.
                         try:
                     elif lang is not None:
                         # Don't create unknown themes in non-default languages.
                         try:

-                            tag = Tag.objects.get(category='theme', **{"name_%s" % lang: theme_name})
+                            tag = Tag.objects.get(
+                                category='theme',
+                                **{"name_%s" % lang: theme_name}
+                            )

                         except Tag.DoesNotExist:
                             pass
                         else:
                         except Tag.DoesNotExist:
                             pass
                         else:


@@ -228,7 +305,12 @@ class BuildHtml(BuildEbook):
                 short_text = truncate_html_words(text, 15)
                 if text == short_text:
                     short_text = ''
                 short_text = truncate_html_words(text, 15)
                 if text == short_text:
                     short_text = ''

-                new_fragment = Fragment.objects.create(anchor=fragment.id, book=book, text=text, short_text=short_text)
+                new_fragment = Fragment.objects.create(
+                    anchor=fragment.id,
+                    book=book,
+                    text=text,
+                    short_text=short_text
+                )

 
                 new_fragment.save()
                 new_fragment.tags = set(meta_tags + themes)
 
                 new_fragment.save()
                 new_fragment.tags = set(meta_tags + themes)


@@ -246,10 +328,13 @@ class BuildHtml(BuildEbook):
         from librarian import DCNS
         url_elem = wldoc.edoc.getroot().find('.//' + DCNS('identifier.url'))
         if url_elem is None:
         from librarian import DCNS
         url_elem = wldoc.edoc.getroot().find('.//' + DCNS('identifier.url'))
         if url_elem is None:

-            gallery = ''
+            gal_url = ''
+            gal_path = ''

         else:
         else:

-            gallery = gallery_url(slug=url_elem.text.rsplit('/', 1)[1])
-        return wldoc.as_html(options={'gallery': "'%s'" % gallery})
+            slug = url_elem.text.rstrip('/').rsplit('/', 1)[1]
+            gal_url = gallery_url(slug=slug)
+            gal_path = gallery_path(slug=slug)
+        return wldoc.as_html(gallery_path=gal_path, gallery_url=gal_url, base_url=absolute_url(gal_url))

 
 
 class BuildCover(BuildEbook):
 
 
 class BuildCover(BuildEbook):


@@ -257,6 +342,15 @@ class BuildCover(BuildEbook):
         pass
 
 
         pass
 
 

+@BuildEbook.register('cover_clean')
+@task(ignore_result=True)
+class BuildCoverClean(BuildCover):
+    @classmethod
+    def transform(cls, wldoc, fieldfile):
+        from librarian.cover import WLCover
+        return WLCover(wldoc.book_info, width=240).output_file()
+
+

 @BuildEbook.register('cover_thumb')
 @task(ignore_result=True)
 class BuildCoverThumb(BuildCover):
 @BuildEbook.register('cover_thumb')
 @task(ignore_result=True)
 class BuildCoverThumb(BuildCover):


@@ -284,6 +378,15 @@ class BuildSimpleCover(BuildCover):
         return WLNoBoxCover(wldoc.book_info, height=1000).output_file()
 
 
         return WLNoBoxCover(wldoc.book_info, height=1000).output_file()
 
 

+@BuildEbook.register('cover_ebookpoint')
+@task(ignore_result=True)
+class BuildCoverEbookpoint(BuildCover):
+    @classmethod
+    def transform(cls, wldoc, fieldfile):
+        from librarian.cover import EbookpointCover
+        return EbookpointCover(wldoc.book_info).output_file()
+
+

 # not used, but needed for migrations
 class OverwritingFieldFile(FieldFile):
     """
 # not used, but needed for migrations
 class OverwritingFieldFile(FieldFile):
     """