procent

[wolnelektury.git] / src / catalogue / fields.py

diff --git a/src/catalogue/fields.py b/src/catalogue/fields.py
index 1efc769..94bdb60 100644 (file)
--- a/src/catalogue/fields.py
+++ b/src/catalogue/fields.py
@@ -1,14 +1,20 @@
-# This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
-# Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
+# This file is part of Wolne Lektury, licensed under GNU Affero GPLv3 or later.
+# Copyright © Fundacja Wolne Lektury. See NOTICE for more information.

 #
 #

+import io

 import os
 import os

+import pkg_resources
+import random
+import time
+from urllib.request import urlopen
+from django.apps import apps

 from django.conf import settings
 from django.core.files import File
 from django.db import models
 from django.db.models.fields.files import FieldFile
 from django.utils.deconstruct import deconstructible
 from django.conf import settings
 from django.core.files import File
 from django.db import models
 from django.db.models.fields.files import FieldFile
 from django.utils.deconstruct import deconstructible

-from django.utils.translation import gettext_lazy as _
-from catalogue.constants import LANGUAGES_3TO2, EBOOK_FORMATS_WITH_CHILDREN, EBOOK_FORMATS_WITHOUT_CHILDREN
+from librarian.cover import make_cover
+from catalogue.constants import LANGUAGES_3TO2

 from catalogue.utils import absolute_url, remove_zip, truncate_html_words, gallery_path, gallery_url
 from waiter.utils import clear_cache
 
 from catalogue.utils import absolute_url, remove_zip, truncate_html_words, gallery_path, gallery_url
 from waiter.utils import clear_cache
 


@@ -29,6 +35,25 @@ class UploadToPath(object):
         return isinstance(other, type(self)) and other.path == self.path
 
 
         return isinstance(other, type(self)) and other.path == self.path
 
 

+def get_make_cover(book):
+    extra = book.get_extra_info_json()
+    cover_logo = extra.get('logo_mono', extra.get('logo'))
+    if cover_logo:
+        while True:
+            try:
+                cover_logo = io.BytesIO(urlopen(cover_logo, timeout=3).read())
+            except:
+                time.sleep(2)
+            else:
+                break
+    
+    def mc(*args, **kwargs):
+        if cover_logo:
+            kwargs['cover_logo'] = cover_logo
+        return make_cover(*args, **kwargs)
+    return mc
+    
+

 class EbookFieldFile(FieldFile):
     """Represents contents of an ebook file field."""
 
 class EbookFieldFile(FieldFile):
     """Represents contents of an ebook file field."""
 


@@ -66,6 +91,7 @@ class EbookField(models.FileField):
     """Represents an ebook file field, attachable to a model."""
     attr_class = EbookFieldFile
     ext = None
     """Represents an ebook file field, attachable to a model."""
     attr_class = EbookFieldFile
     ext = None

+    for_parents = True

     librarian2_api = False
     ZIP = None
 
     librarian2_api = False
     ZIP = None
 


@@ -124,42 +150,52 @@ class EbookField(models.FileField):
         setattr(cls, 'has_%s' % self.attname, has)
 
     def get_current_etag(self):
         setattr(cls, 'has_%s' % self.attname, has)
 
     def get_current_etag(self):

-        import pkg_resources
+        MediaInsertSet = apps.get_model('annoy', 'MediaInsertSet')

         librarian_version = pkg_resources.get_distribution("librarian").version
         librarian_version = pkg_resources.get_distribution("librarian").version

-        return librarian_version
-
-    def schedule_stale(self, queryset=None):
-        """Schedule building this format for all the books where etag is stale."""
+        etag = librarian_version
+        mis = MediaInsertSet.get_for_format(self.ext)
+        if mis is not None:
+            etag += '_' + mis.etag
+        return etag
+
+    def find_stale(self, limit):
+        """Find some books where this format is stale."""

         # If there is not ETag field, bail. That's true for xml file field.
         if not self.with_etag:
         # If there is not ETag field, bail. That's true for xml file field.
         if not self.with_etag:

-            return
+            return []

 
         etag = self.get_current_etag()
 
         etag = self.get_current_etag()

-        if queryset is None:
-            queryset = self.model.objects.all()

 
 

-        if self.format_name in EBOOK_FORMATS_WITHOUT_CHILDREN + ['html']:
+        queryset = self.model.objects.all()
+        if not self.for_parents:

             queryset = queryset.filter(children=None)
 
         queryset = queryset.exclude(**{
             f'{self.etag_field_name}__in': [
                 etag, f'{etag}{ETAG_SCHEDULED_SUFFIX}'
             queryset = queryset.filter(children=None)
 
         queryset = queryset.exclude(**{
             f'{self.etag_field_name}__in': [
                 etag, f'{etag}{ETAG_SCHEDULED_SUFFIX}'

-            ]
+           ]

         })
         })

-        for obj in queryset:
-            fieldfile = getattr(obj, self.attname)
-            priority = EBOOK_REBUILD_PRIORITY if fieldfile else EBOOK_BUILD_PRIORITY
-            fieldfile.build_delay(priority=priority)
+
+        queryset = queryset.order_by('?')[:limit]
+        return queryset

 
     @classmethod
 
     @classmethod

-    def schedule_all_stale(cls, model):
+    def find_all_stale(cls, model, limit):

         """Schedules all stale ebooks of all formats to rebuild."""
         """Schedules all stale ebooks of all formats to rebuild."""

+        found = []

         for field in model._meta.fields:
             if isinstance(field, cls):
         for field in model._meta.fields:
             if isinstance(field, cls):

-                field.schedule_stale()
+                for instance in field.find_stale(limit):
+                    found.append((
+                        field.name,
+                        instance
+                    ))
+        random.shuffle(found)
+        found = found[:limit]
+        return found

 
     @staticmethod
 
     @staticmethod

-    def transform(wldoc):
+    def transform(wldoc, book):

         """Transforms an librarian.WLDocument into an librarian.OutputFile.
         """
         raise NotImplemented()
         """Transforms an librarian.WLDocument into an librarian.OutputFile.
         """
         raise NotImplemented()


@@ -172,8 +208,10 @@ class EbookField(models.FileField):
         book = fieldfile.instance
         out = self.transform(
             book.wldocument2() if self.librarian2_api else book.wldocument(),
         book = fieldfile.instance
         out = self.transform(
             book.wldocument2() if self.librarian2_api else book.wldocument(),

+            book,

         )
         )

-        fieldfile.save(None, File(open(out.get_filename(), 'rb')), save=False)
+        with open(out.get_filename(), 'rb') as f:
+            fieldfile.save(None, File(f), save=False)

         self.set_file_permissions(fieldfile)
         if book.pk is not None:
             book.save(update_fields=[self.attname])
         self.set_file_permissions(fieldfile)
         if book.pk is not None:
             book.save(update_fields=[self.attname])


@@ -190,18 +228,20 @@ class XmlField(EbookField):
 
 class TxtField(EbookField):
     ext = 'txt'
 
 class TxtField(EbookField):
     ext = 'txt'

+    for_parents = False

 
     @staticmethod
 
     @staticmethod

-    def transform(wldoc):
+    def transform(wldoc, book):

         return wldoc.as_text()
 
 
 class Fb2Field(EbookField):
     ext = 'fb2'
         return wldoc.as_text()
 
 
 class Fb2Field(EbookField):
     ext = 'fb2'

+    for_parents = False

     ZIP = 'wolnelektury_pl_fb2'
 
     @staticmethod
     ZIP = 'wolnelektury_pl_fb2'
 
     @staticmethod

-    def transform(wldoc):
+    def transform(wldoc, book):

         return wldoc.as_fb2()
 
 
         return wldoc.as_fb2()
 
 


@@ -210,9 +250,10 @@ class PdfField(EbookField):
     ZIP = 'wolnelektury_pl_pdf'
 
     @staticmethod
     ZIP = 'wolnelektury_pl_pdf'
 
     @staticmethod

-    def transform(wldoc):
+    def transform(wldoc, book):

         return wldoc.as_pdf(
         return wldoc.as_pdf(

-            morefloats=settings.LIBRARIAN_PDF_MOREFLOATS, cover=True,
+            morefloats=settings.LIBRARIAN_PDF_MOREFLOATS,
+            cover=get_make_cover(book),

             base_url=absolute_url(gallery_url(wldoc.book_info.url.slug)), customizations=['notoc'])
 
     def build(self, fieldfile):
             base_url=absolute_url(gallery_url(wldoc.book_info.url.slug)), customizations=['notoc'])
 
     def build(self, fieldfile):


@@ -226,11 +267,13 @@ class EpubField(EbookField):
     ZIP = 'wolnelektury_pl_epub'
 
     @staticmethod
     ZIP = 'wolnelektury_pl_epub'
 
     @staticmethod

-    def transform(wldoc):
+    def transform(wldoc, book):

         from librarian.builders import EpubBuilder
         from librarian.builders import EpubBuilder

+        MediaInsertSet = apps.get_model('annoy', 'MediaInsertSet')

         return EpubBuilder(
                 base_url='file://' + os.path.abspath(gallery_path(wldoc.meta.url.slug)) + '/',
         return EpubBuilder(
                 base_url='file://' + os.path.abspath(gallery_path(wldoc.meta.url.slug)) + '/',

-                fundraising=settings.EPUB_FUNDRAISING
+                fundraising=MediaInsertSet.get_texts_for('epub'),
+                cover=get_make_cover(book),

             ).build(wldoc)
 
 
             ).build(wldoc)
 
 


@@ -240,16 +283,19 @@ class MobiField(EbookField):
     ZIP = 'wolnelektury_pl_mobi'
 
     @staticmethod
     ZIP = 'wolnelektury_pl_mobi'
 
     @staticmethod

-    def transform(wldoc):
+    def transform(wldoc, book):

         from librarian.builders import MobiBuilder
         from librarian.builders import MobiBuilder

+        MediaInsertSet = apps.get_model('annoy', 'MediaInsertSet')

         return MobiBuilder(
                 base_url='file://' + os.path.abspath(gallery_path(wldoc.meta.url.slug)) + '/',
         return MobiBuilder(
                 base_url='file://' + os.path.abspath(gallery_path(wldoc.meta.url.slug)) + '/',

-                fundraising=settings.EPUB_FUNDRAISING
+                fundraising=MediaInsertSet.get_texts_for('mobi'),
+                cover=get_make_cover(book),

             ).build(wldoc)
 
 
 class HtmlField(EbookField):
     ext = 'html'
             ).build(wldoc)
 
 
 class HtmlField(EbookField):
     ext = 'html'

+    for_parents = False

 
     def build(self, fieldfile):
         from django.core.files.base import ContentFile
 
     def build(self, fieldfile):
         from django.core.files.base import ContentFile


@@ -260,7 +306,7 @@ class HtmlField(EbookField):
 
         book = fieldfile.instance
 
 
         book = fieldfile.instance
 

-        html_output = self.transform(book.wldocument(parse_dublincore=False))
+        html_output = self.transform(book.wldocument(parse_dublincore=False), book)

 
         # Delete old fragments, create from scratch if necessary.
         book.fragments.all().delete()
 
         # Delete old fragments, create from scratch if necessary.
         book.fragments.all().delete()


@@ -340,7 +386,7 @@ class HtmlField(EbookField):
         return False
 
     @staticmethod
         return False
 
     @staticmethod

-    def transform(wldoc):
+    def transform(wldoc, book):

         # ugly, but we can't use wldoc.book_info here
         from librarian import DCNS
         url_elem = wldoc.edoc.getroot().find('.//' + DCNS('identifier.url'))
         # ugly, but we can't use wldoc.book_info here
         from librarian import DCNS
         url_elem = wldoc.edoc.getroot().find('.//' + DCNS('identifier.url'))


@@ -358,6 +404,10 @@ class CoverField(EbookField):
     ext = 'jpg'
     directory = 'cover'
 
     ext = 'jpg'
     directory = 'cover'
 

+    @staticmethod
+    def transform(wldoc, book):
+        return get_make_cover(book)(wldoc.book_info, width=360).output_file()
+

     def set_file_permissions(self, fieldfile):
         pass
 
     def set_file_permissions(self, fieldfile):
         pass
 


@@ -366,19 +416,15 @@ class CoverCleanField(CoverField):
     directory = 'cover_clean'
 
     @staticmethod
     directory = 'cover_clean'
 
     @staticmethod

-    def transform(wldoc):
-        if wldoc.book_info.cover_box_position == 'none':
-            from librarian.cover import WLCover
-            return WLCover(wldoc.book_info, width=240).output_file()
-        from librarian.covers.marquise import MarquiseCover
-        return MarquiseCover(wldoc.book_info, width=240).output_file()
+    def transform(wldoc, book):
+        return get_make_cover(book)(wldoc.book_info, width=360).output_file()

 
 
 class CoverThumbField(CoverField):
     directory = 'cover_thumb'
 
     @staticmethod
 
 
 class CoverThumbField(CoverField):
     directory = 'cover_thumb'
 
     @staticmethod

-    def transform(wldoc):
+    def transform(wldoc, book):

         from librarian.cover import WLCover
         return WLCover(wldoc.book_info, height=193).output_file()
 
         from librarian.cover import WLCover
         return WLCover(wldoc.book_info, height=193).output_file()
 


@@ -387,7 +433,7 @@ class CoverApiThumbField(CoverField):
     directory = 'cover_api_thumb'
 
     @staticmethod
     directory = 'cover_api_thumb'
 
     @staticmethod

-    def transform(wldoc):
+    def transform(wldoc, book):

         from librarian.cover import WLNoBoxCover
         return WLNoBoxCover(wldoc.book_info, height=500).output_file()
 
         from librarian.cover import WLNoBoxCover
         return WLNoBoxCover(wldoc.book_info, height=500).output_file()
 


@@ -396,7 +442,7 @@ class SimpleCoverField(CoverField):
     directory = 'cover_simple'
 
     @staticmethod
     directory = 'cover_simple'
 
     @staticmethod

-    def transform(wldoc):
+    def transform(wldoc, book):

         from librarian.cover import WLNoBoxCover
         return WLNoBoxCover(wldoc.book_info, height=1000).output_file()
 
         from librarian.cover import WLNoBoxCover
         return WLNoBoxCover(wldoc.book_info, height=1000).output_file()
 


@@ -405,6 +451,6 @@ class CoverEbookpointField(CoverField):
     directory = 'cover_ebookpoint'
 
     @staticmethod
     directory = 'cover_ebookpoint'
 
     @staticmethod

-    def transform(wldoc):
+    def transform(wldoc, book):

         from librarian.cover import EbookpointCover
         return EbookpointCover(wldoc.book_info).output_file()
         from librarian.cover import EbookpointCover
         return EbookpointCover(wldoc.book_info).output_file()