procent

[wolnelektury.git] / src / catalogue / fields.py

diff --git a/src/catalogue/fields.py b/src/catalogue/fields.py
index 3d10cd8..94bdb60 100644 (file)
--- a/src/catalogue/fields.py
+++ b/src/catalogue/fields.py
@@ -1,47 +1,81 @@
-# This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
-# Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
+# This file is part of Wolne Lektury, licensed under GNU Affero GPLv3 or later.
+# Copyright © Fundacja Wolne Lektury. See NOTICE for more information.

 #
 #

+import io

 import os
 import os

+import pkg_resources
+import random
+import time
+from urllib.request import urlopen
+from django.apps import apps

 from django.conf import settings
 from django.core.files import File
 from django.conf import settings
 from django.core.files import File

-from django.core.files.storage import FileSystemStorage

 from django.db import models
 from django.db.models.fields.files import FieldFile
 from django.db import models
 from django.db.models.fields.files import FieldFile

-from catalogue import app_settings
-from catalogue.constants import LANGUAGES_3TO2, EBOOK_FORMATS_WITH_CHILDREN, EBOOK_FORMATS_WITHOUT_CHILDREN
+from django.utils.deconstruct import deconstructible
+from librarian.cover import make_cover
+from catalogue.constants import LANGUAGES_3TO2

 from catalogue.utils import absolute_url, remove_zip, truncate_html_words, gallery_path, gallery_url
 from catalogue.utils import absolute_url, remove_zip, truncate_html_words, gallery_path, gallery_url

-#from celery import Task, shared_task
-from celery.task import Task, task
-from celery.utils.log import get_task_logger

 from waiter.utils import clear_cache
 
 from waiter.utils import clear_cache
 

-task_logger = get_task_logger(__name__)
-

 ETAG_SCHEDULED_SUFFIX = '-scheduled'
 EBOOK_BUILD_PRIORITY = 0
 EBOOK_REBUILD_PRIORITY = 9
 
 
 ETAG_SCHEDULED_SUFFIX = '-scheduled'
 EBOOK_BUILD_PRIORITY = 0
 EBOOK_REBUILD_PRIORITY = 9
 
 

+@deconstructible
+class UploadToPath(object):
+    def __init__(self, path):
+        self.path = path
+
+    def __call__(self, instance, filename):
+        return self.path % instance.slug
+
+    def __eq__(self, other):
+        return isinstance(other, type(self)) and other.path == self.path
+
+
+def get_make_cover(book):
+    extra = book.get_extra_info_json()
+    cover_logo = extra.get('logo_mono', extra.get('logo'))
+    if cover_logo:
+        while True:
+            try:
+                cover_logo = io.BytesIO(urlopen(cover_logo, timeout=3).read())
+            except:
+                time.sleep(2)
+            else:
+                break
+    
+    def mc(*args, **kwargs):
+        if cover_logo:
+            kwargs['cover_logo'] = cover_logo
+        return make_cover(*args, **kwargs)
+    return mc
+    
+

 class EbookFieldFile(FieldFile):
     """Represents contents of an ebook file field."""
 
     def build(self):
         """Build the ebook immediately."""
 class EbookFieldFile(FieldFile):
     """Represents contents of an ebook file field."""
 
     def build(self):
         """Build the ebook immediately."""

-        return self.field.builder.build(self)
+        etag = self.field.get_current_etag()
+        self.field.build(self)
+        self.update_etag(etag)
+        self.instance.clear_cache()

 
     def build_delay(self, priority=EBOOK_BUILD_PRIORITY):
         """Builds the ebook in a delayed task."""
 
     def build_delay(self, priority=EBOOK_BUILD_PRIORITY):
         """Builds the ebook in a delayed task."""

+        from .tasks import build_field
+

         self.update_etag(
             "".join([self.field.get_current_etag(), ETAG_SCHEDULED_SUFFIX])
         )
         self.update_etag(
             "".join([self.field.get_current_etag(), ETAG_SCHEDULED_SUFFIX])
         )

-        return self.field.builder.apply_async(
-            [self.instance, self.field.attname],
+        return build_field.apply_async(
+            [self.instance.pk, self.field.attname],

             priority=priority
         )
 
             priority=priority
         )
 

-    def get_url(self):
-        return self.instance.media_url(self.field.attname.split('_')[0])
-

     def set_readable(self, readable):
         import os
         permissions = 0o644 if readable else 0o600
     def set_readable(self, readable):
         import os
         permissions = 0o644 if readable else 0o600


@@ -56,26 +90,55 @@ class EbookFieldFile(FieldFile):
 class EbookField(models.FileField):
     """Represents an ebook file field, attachable to a model."""
     attr_class = EbookFieldFile
 class EbookField(models.FileField):
     """Represents an ebook file field, attachable to a model."""
     attr_class = EbookFieldFile

-    registry = []
+    ext = None
+    for_parents = True
+    librarian2_api = False
+    ZIP = None
+
+    def __init__(self, verbose_name=None, with_etag=True, etag_field_name=None, **kwargs):
+        kwargs.setdefault('verbose_name', verbose_name)
+        self.with_etag = with_etag
+        self.etag_field_name = etag_field_name
+        kwargs.setdefault('max_length', 255)
+        kwargs.setdefault('blank', True)
+        kwargs.setdefault('default', '')
+        kwargs.setdefault('upload_to', self.get_upload_to(self.ext))

 
 

-    def __init__(self, format_name, *args, **kwargs):
-        super(EbookField, self).__init__(*args, **kwargs)
-        self.format_name = format_name
+        super().__init__(**kwargs)

 
     def deconstruct(self):
 
     def deconstruct(self):

-        name, path, args, kwargs = super(EbookField, self).deconstruct()
-        args.insert(0, self.format_name)
+        name, path, args, kwargs = super().deconstruct()
+        if kwargs.get('max_length') == 255:
+            del kwargs['max_length']
+        if kwargs.get('blank') is True:
+            del kwargs['blank']
+        if kwargs.get('default') == '':
+            del kwargs['default']
+        if self.get_upload_to(self.ext) == kwargs.get('upload_to'):
+            del kwargs['upload_to']
+        # with_etag creates a second field, which then deconstructs to manage
+        # its own migrations. So for migrations, etag_field_name is explicitly
+        # set to avoid double creation of the etag field.
+        if self.with_etag:
+            kwargs['etag_field_name'] = self.etag_field_name
+        else:
+            kwargs['with_etag'] = self.with_etag
+

         return name, path, args, kwargs
 
         return name, path, args, kwargs
 

-    @property
-    def builder(self):
-        """Finds a celery task suitable for the format of the field."""
-        return BuildEbook.for_format(self.format_name)
+    @classmethod
+    def get_upload_to(cls, directory):
+        directory = getattr(cls, 'directory', cls.ext)
+        upload_template = f'book/{directory}/%s.{cls.ext}'
+        return UploadToPath(upload_template)

 
     def contribute_to_class(self, cls, name):
         super(EbookField, self).contribute_to_class(cls, name)
 
 
     def contribute_to_class(self, cls, name):
         super(EbookField, self).contribute_to_class(cls, name)
 

-        self.etag_field_name = f'{name}_etag'
+        if self.with_etag and not self.etag_field_name:
+            self.etag_field_name = f'{name}_etag'
+            self.etag_field = models.CharField(max_length=255, editable=False, default='', db_index=True)
+            self.etag_field.contribute_to_class(cls, f'{name}_etag')

 
         def has(model_instance):
             return bool(getattr(model_instance, self.attname, None))
 
         def has(model_instance):
             return bool(getattr(model_instance, self.attname, None))


@@ -84,84 +147,58 @@ class EbookField(models.FileField):
         has.short_description = self.name
         has.boolean = True
 
         has.short_description = self.name
         has.boolean = True
 

-        self.registry.append(self)
-

         setattr(cls, 'has_%s' % self.attname, has)
 
     def get_current_etag(self):
         setattr(cls, 'has_%s' % self.attname, has)
 
     def get_current_etag(self):

-        import pkg_resources
+        MediaInsertSet = apps.get_model('annoy', 'MediaInsertSet')

         librarian_version = pkg_resources.get_distribution("librarian").version
         librarian_version = pkg_resources.get_distribution("librarian").version

-        return librarian_version
-
-    def schedule_stale(self, queryset=None):
-        """Schedule building this format for all the books where etag is stale."""
+        etag = librarian_version
+        mis = MediaInsertSet.get_for_format(self.ext)
+        if mis is not None:
+            etag += '_' + mis.etag
+        return etag
+
+    def find_stale(self, limit):
+        """Find some books where this format is stale."""

         # If there is not ETag field, bail. That's true for xml file field.
         # If there is not ETag field, bail. That's true for xml file field.

-        if not hasattr(self.model, f'{self.attname}_etag'):
-            return
+        if not self.with_etag:
+            return []

 
         etag = self.get_current_etag()
 
         etag = self.get_current_etag()

-        if queryset is None:
-            queryset = self.model.objects.all()

 
 

-        if self.format_name in EBOOK_FORMATS_WITHOUT_CHILDREN + ['html']:
+        queryset = self.model.objects.all()
+        if not self.for_parents:

             queryset = queryset.filter(children=None)
 
         queryset = queryset.exclude(**{
             f'{self.etag_field_name}__in': [
                 etag, f'{etag}{ETAG_SCHEDULED_SUFFIX}'
             queryset = queryset.filter(children=None)
 
         queryset = queryset.exclude(**{
             f'{self.etag_field_name}__in': [
                 etag, f'{etag}{ETAG_SCHEDULED_SUFFIX}'

-            ]
+           ]

         })
         })

-        for obj in queryset:
-            fieldfile = getattr(obj, self.attname)
-            priority = EBOOK_REBUILD_PRIORITY if fieldfile else EBOOK_BUILD_PRIORITY
-            fieldfile.build_delay(priority=priority)
-
-    @classmethod
-    def schedule_all_stale(cls):
-        """Schedules all stale ebooks of all formats to rebuild."""
-        for field in cls.registry:
-            field.schedule_stale()
-
-

 
 

-class BuildEbook(Task):
-    librarian2_api = False
-
-    formats = {}
-
-    @classmethod
-    def register(cls, format_name):
-        """A decorator for registering subclasses for particular formats."""
-        def wrapper(builder):
-            cls.formats[format_name] = builder
-            return builder
-        return wrapper
+        queryset = queryset.order_by('?')[:limit]
+        return queryset

 
     @classmethod
 
     @classmethod

-    def for_format(cls, format_name):
-        """Returns a celery task suitable for specified format."""
-        return cls.formats.get(format_name, BuildEbookTask)
+    def find_all_stale(cls, model, limit):
+        """Schedules all stale ebooks of all formats to rebuild."""
+        found = []
+        for field in model._meta.fields:
+            if isinstance(field, cls):
+                for instance in field.find_stale(limit):
+                    found.append((
+                        field.name,
+                        instance
+                    ))
+        random.shuffle(found)
+        found = found[:limit]
+        return found

 
     @staticmethod
 
     @staticmethod

-    def transform(wldoc, fieldfile):
+    def transform(wldoc, book):

         """Transforms an librarian.WLDocument into an librarian.OutputFile.
         """Transforms an librarian.WLDocument into an librarian.OutputFile.

-
-        By default, it just calls relevant wldoc.as_??? method.
-

         """
         """

-        return getattr(wldoc, "as_%s" % fieldfile.field.format_name)()
-
-    def run(self, obj, field_name):
-        """Just run `build` on FieldFile, can't pass it directly to Celery."""
-        fieldfile = getattr(obj, field_name)
-
-        # Get etag value before actually building the file.
-        etag = fieldfile.field.get_current_etag()
-        task_logger.info("%s -> %s@%s" % (obj.slug, field_name, etag))
-        ret = self.build(getattr(obj, field_name))
-        fieldfile.update_etag(etag)
-        obj.clear_cache()
-        return ret
+        raise NotImplemented()

 
     def set_file_permissions(self, fieldfile):
         if fieldfile.instance.preview:
 
     def set_file_permissions(self, fieldfile):
         if fieldfile.instance.preview:


@@ -171,70 +208,95 @@ class BuildEbook(Task):
         book = fieldfile.instance
         out = self.transform(
             book.wldocument2() if self.librarian2_api else book.wldocument(),
         book = fieldfile.instance
         out = self.transform(
             book.wldocument2() if self.librarian2_api else book.wldocument(),

-            fieldfile)
-        fieldfile.save(None, File(open(out.get_filename(), 'rb')), save=False)
+            book,
+        )
+        with open(out.get_filename(), 'rb') as f:
+            fieldfile.save(None, File(f), save=False)

         self.set_file_permissions(fieldfile)
         if book.pk is not None:
         self.set_file_permissions(fieldfile)
         if book.pk is not None:

-            book.save(update_fields=[fieldfile.field.attname])
-        if fieldfile.field.format_name in app_settings.FORMAT_ZIPS:
-            remove_zip(app_settings.FORMAT_ZIPS[fieldfile.field.format_name])
-# Don't decorate BuildEbook, because we want to subclass it.
-BuildEbookTask = task(BuildEbook, ignore_result=True)
+            book.save(update_fields=[self.attname])
+        if self.ZIP:
+            remove_zip(self.ZIP)
+
+
+class XmlField(EbookField):
+    ext = 'xml'
+
+    def build(self, fieldfile):
+        pass
+

 
 

+class TxtField(EbookField):
+    ext = 'txt'
+    for_parents = False

 
 

-@BuildEbook.register('txt')
-@task(ignore_result=True)
-class BuildTxt(BuildEbook):

     @staticmethod
     @staticmethod

-    def transform(wldoc, fieldfile):
+    def transform(wldoc, book):

         return wldoc.as_text()
 
 
         return wldoc.as_text()
 
 

-@BuildEbook.register('pdf')
-@task(ignore_result=True)
-class BuildPdf(BuildEbook):
+class Fb2Field(EbookField):
+    ext = 'fb2'
+    for_parents = False
+    ZIP = 'wolnelektury_pl_fb2'
+
+    @staticmethod
+    def transform(wldoc, book):
+        return wldoc.as_fb2()
+
+
+class PdfField(EbookField):
+    ext = 'pdf'
+    ZIP = 'wolnelektury_pl_pdf'
+

     @staticmethod
     @staticmethod

-    def transform(wldoc, fieldfile):
+    def transform(wldoc, book):

         return wldoc.as_pdf(
         return wldoc.as_pdf(

-            morefloats=settings.LIBRARIAN_PDF_MOREFLOATS, cover=True,
+            morefloats=settings.LIBRARIAN_PDF_MOREFLOATS,
+            cover=get_make_cover(book),

             base_url=absolute_url(gallery_url(wldoc.book_info.url.slug)), customizations=['notoc'])
 
     def build(self, fieldfile):
             base_url=absolute_url(gallery_url(wldoc.book_info.url.slug)), customizations=['notoc'])
 
     def build(self, fieldfile):

-        BuildEbook.build(self, fieldfile)
+        super().build(fieldfile)

         clear_cache(fieldfile.instance.slug)
 
 
         clear_cache(fieldfile.instance.slug)
 
 

-@BuildEbook.register('epub')
-@task(ignore_result=True)
-class BuildEpub(BuildEbook):
+class EpubField(EbookField):
+    ext = 'epub'

     librarian2_api = True
     librarian2_api = True

+    ZIP = 'wolnelektury_pl_epub'

 
     @staticmethod
 
     @staticmethod

-    def transform(wldoc, fieldfile):
+    def transform(wldoc, book):

         from librarian.builders import EpubBuilder
         from librarian.builders import EpubBuilder

+        MediaInsertSet = apps.get_model('annoy', 'MediaInsertSet')

         return EpubBuilder(
                 base_url='file://' + os.path.abspath(gallery_path(wldoc.meta.url.slug)) + '/',
         return EpubBuilder(
                 base_url='file://' + os.path.abspath(gallery_path(wldoc.meta.url.slug)) + '/',

-                fundraising=settings.EPUB_FUNDRAISING
+                fundraising=MediaInsertSet.get_texts_for('epub'),
+                cover=get_make_cover(book),

             ).build(wldoc)
 
 
             ).build(wldoc)
 
 

-@BuildEbook.register('mobi')
-@task(ignore_result=True)
-class BuildMobi(BuildEbook):
+class MobiField(EbookField):
+    ext = 'mobi'

     librarian2_api = True
     librarian2_api = True

+    ZIP = 'wolnelektury_pl_mobi'

 
     @staticmethod
 
     @staticmethod

-    def transform(wldoc, fieldfile):
+    def transform(wldoc, book):

         from librarian.builders import MobiBuilder
         from librarian.builders import MobiBuilder

+        MediaInsertSet = apps.get_model('annoy', 'MediaInsertSet')

         return MobiBuilder(
                 base_url='file://' + os.path.abspath(gallery_path(wldoc.meta.url.slug)) + '/',
         return MobiBuilder(
                 base_url='file://' + os.path.abspath(gallery_path(wldoc.meta.url.slug)) + '/',

-                fundraising=settings.EPUB_FUNDRAISING
+                fundraising=MediaInsertSet.get_texts_for('mobi'),
+                cover=get_make_cover(book),

             ).build(wldoc)
 
 
             ).build(wldoc)
 
 

-@BuildEbook.register('html')
-@task(ignore_result=True)
-class BuildHtml(BuildEbook):
+class HtmlField(EbookField):
+    ext = 'html'
+    for_parents = False
+

     def build(self, fieldfile):
         from django.core.files.base import ContentFile
         from slugify import slugify
     def build(self, fieldfile):
         from django.core.files.base import ContentFile
         from slugify import slugify


@@ -244,7 +306,7 @@ class BuildHtml(BuildEbook):
 
         book = fieldfile.instance
 
 
         book = fieldfile.instance
 

-        html_output = self.transform(book.wldocument(parse_dublincore=False), fieldfile)
+        html_output = self.transform(book.wldocument(parse_dublincore=False), book)

 
         # Delete old fragments, create from scratch if necessary.
         book.fragments.all().delete()
 
         # Delete old fragments, create from scratch if necessary.
         book.fragments.all().delete()


@@ -324,7 +386,7 @@ class BuildHtml(BuildEbook):
         return False
 
     @staticmethod
         return False
 
     @staticmethod

-    def transform(wldoc, fieldfile):
+    def transform(wldoc, book):

         # ugly, but we can't use wldoc.book_info here
         from librarian import DCNS
         url_elem = wldoc.edoc.getroot().find('.//' + DCNS('identifier.url'))
         # ugly, but we can't use wldoc.book_info here
         from librarian import DCNS
         url_elem = wldoc.edoc.getroot().find('.//' + DCNS('identifier.url'))


@@ -338,79 +400,57 @@ class BuildHtml(BuildEbook):
         return wldoc.as_html(gallery_path=gal_path, gallery_url=gal_url, base_url=absolute_url(gal_url))
 
 
         return wldoc.as_html(gallery_path=gal_path, gallery_url=gal_url, base_url=absolute_url(gal_url))
 
 

-class BuildCover(BuildEbook):
+class CoverField(EbookField):
+    ext = 'jpg'
+    directory = 'cover'
+
+    @staticmethod
+    def transform(wldoc, book):
+        return get_make_cover(book)(wldoc.book_info, width=360).output_file()
+

     def set_file_permissions(self, fieldfile):
         pass
 
 
     def set_file_permissions(self, fieldfile):
         pass
 
 

-@BuildEbook.register('cover_clean')
-@task(ignore_result=True)
-class BuildCoverClean(BuildCover):
-    @classmethod
-    def transform(cls, wldoc, fieldfile):
-        if wldoc.book_info.cover_box_position == 'none':
-            from librarian.cover import WLCover
-            return WLCover(wldoc.book_info, width=240).output_file()
-        from librarian.covers.marquise import MarquiseCover
-        return MarquiseCover(wldoc.book_info, width=240).output_file()
+class CoverCleanField(CoverField):
+    directory = 'cover_clean'

 
 

+    @staticmethod
+    def transform(wldoc, book):
+        return get_make_cover(book)(wldoc.book_info, width=360).output_file()

 
 

-@BuildEbook.register('cover_thumb')
-@task(ignore_result=True)
-class BuildCoverThumb(BuildCover):
-    @classmethod
-    def transform(cls, wldoc, fieldfile):
+
+class CoverThumbField(CoverField):
+    directory = 'cover_thumb'
+
+    @staticmethod
+    def transform(wldoc, book):

         from librarian.cover import WLCover
         return WLCover(wldoc.book_info, height=193).output_file()
 
 
         from librarian.cover import WLCover
         return WLCover(wldoc.book_info, height=193).output_file()
 
 

-@BuildEbook.register('cover_api_thumb')
-@task(ignore_result=True)
-class BuildCoverApiThumb(BuildCover):
-    @classmethod
-    def transform(cls, wldoc, fieldfile):
+class CoverApiThumbField(CoverField):
+    directory = 'cover_api_thumb'
+
+    @staticmethod
+    def transform(wldoc, book):

         from librarian.cover import WLNoBoxCover
         return WLNoBoxCover(wldoc.book_info, height=500).output_file()
 
 
         from librarian.cover import WLNoBoxCover
         return WLNoBoxCover(wldoc.book_info, height=500).output_file()
 
 

-@BuildEbook.register('simple_cover')
-@task(ignore_result=True)
-class BuildSimpleCover(BuildCover):
-    @classmethod
-    def transform(cls, wldoc, fieldfile):
+class SimpleCoverField(CoverField):
+    directory = 'cover_simple'
+
+    @staticmethod
+    def transform(wldoc, book):

         from librarian.cover import WLNoBoxCover
         return WLNoBoxCover(wldoc.book_info, height=1000).output_file()
 
 
         from librarian.cover import WLNoBoxCover
         return WLNoBoxCover(wldoc.book_info, height=1000).output_file()
 
 

-@BuildEbook.register('cover_ebookpoint')
-@task(ignore_result=True)
-class BuildCoverEbookpoint(BuildCover):
-    @classmethod
-    def transform(cls, wldoc, fieldfile):
+class CoverEbookpointField(CoverField):
+    directory = 'cover_ebookpoint'
+
+    @staticmethod
+    def transform(wldoc, book):

         from librarian.cover import EbookpointCover
         return EbookpointCover(wldoc.book_info).output_file()
         from librarian.cover import EbookpointCover
         return EbookpointCover(wldoc.book_info).output_file()

-
-
-# not used, but needed for migrations
-class OverwritingFieldFile(FieldFile):
-    """
-        Deletes the old file before saving the new one.
-    """
-
-    def save(self, name, content, *args, **kwargs):
-        leave = kwargs.pop('leave', None)
-        # delete if there's a file already and there's a new one coming
-        if not leave and self and (not hasattr(content, 'path') or content.path != self.path):
-            self.delete(save=False)
-        return super(OverwritingFieldFile, self).save(name, content, *args, **kwargs)
-
-
-class OverwritingFileField(models.FileField):
-    attr_class = OverwritingFieldFile
-
-
-class OverwriteStorage(FileSystemStorage):
-
-    def get_available_name(self, name, max_length=None):
-        self.delete(name)
-        return name