X-Git-Url: https://git.mdrn.pl/wolnelektury.git/blobdiff_plain/d57cd71379f89196addaa90d1c14a17a359f35d3..1959361810afbb5f1de88b7d717da32979c7235d:/src/catalogue/fields.py diff --git a/src/catalogue/fields.py b/src/catalogue/fields.py index 581bf38a1..c4dec7e3f 100644 --- a/src/catalogue/fields.py +++ b/src/catalogue/fields.py @@ -1,46 +1,81 @@ -# This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later. -# Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information. +# This file is part of Wolne Lektury, licensed under GNU Affero GPLv3 or later. +# Copyright © Fundacja Wolne Lektury. See NOTICE for more information. # +import io import os +import pkg_resources +import random +import time +from urllib.request import urlopen +from django.apps import apps from django.conf import settings from django.core.files import File -from django.core.files.storage import FileSystemStorage from django.db import models from django.db.models.fields.files import FieldFile -from catalogue import app_settings -from catalogue.constants import LANGUAGES_3TO2, EBOOK_FORMATS_WITH_CHILDREN, EBOOK_FORMATS_WITHOUT_CHILDREN +from django.utils.deconstruct import deconstructible +from librarian.cover import make_cover +from catalogue.constants import LANGUAGES_3TO2 from catalogue.utils import absolute_url, remove_zip, truncate_html_words, gallery_path, gallery_url -from celery.task import Task, task -from celery.utils.log import get_task_logger from waiter.utils import clear_cache -task_logger = get_task_logger(__name__) - ETAG_SCHEDULED_SUFFIX = '-scheduled' EBOOK_BUILD_PRIORITY = 0 EBOOK_REBUILD_PRIORITY = 9 +@deconstructible +class UploadToPath(object): + def __init__(self, path): + self.path = path + + def __call__(self, instance, filename): + return self.path % instance.slug + + def __eq__(self, other): + return isinstance(other, type(self)) and other.path == self.path + + +def get_make_cover(book): + extra = book.get_extra_info_json() + cover_logo = extra.get('logo_mono', extra.get('logo')) + if cover_logo: + while True: + try: + cover_logo = io.BytesIO(urlopen(cover_logo, timeout=3).read()) + except: + time.sleep(2) + else: + break + + def mc(*args, **kwargs): + if cover_logo: + kwargs['cover_logo'] = cover_logo + return make_cover(*args, **kwargs) + return mc + + class EbookFieldFile(FieldFile): """Represents contents of an ebook file field.""" def build(self): """Build the ebook immediately.""" - return self.field.builder.build(self) + etag = self.field.get_current_etag() + self.field.build(self) + self.update_etag(etag) + self.instance.clear_cache() def build_delay(self, priority=EBOOK_BUILD_PRIORITY): """Builds the ebook in a delayed task.""" + from .tasks import build_field + self.update_etag( "".join([self.field.get_current_etag(), ETAG_SCHEDULED_SUFFIX]) ) - return self.field.builder.apply_async( - [self.instance, self.field.attname], + return build_field.apply_async( + [self.instance.pk, self.field.attname], priority=priority ) - def get_url(self): - return self.instance.media_url(self.field.attname.split('_')[0]) - def set_readable(self, readable): import os permissions = 0o644 if readable else 0o600 @@ -55,26 +90,55 @@ class EbookFieldFile(FieldFile): class EbookField(models.FileField): """Represents an ebook file field, attachable to a model.""" attr_class = EbookFieldFile - registry = [] + ext = None + for_parents = True + librarian2_api = False + ZIP = None - def __init__(self, format_name, *args, **kwargs): - super(EbookField, self).__init__(*args, **kwargs) - self.format_name = format_name + def __init__(self, verbose_name=None, with_etag=True, etag_field_name=None, **kwargs): + kwargs.setdefault('verbose_name', verbose_name) + self.with_etag = with_etag + self.etag_field_name = etag_field_name + kwargs.setdefault('max_length', 255) + kwargs.setdefault('blank', True) + kwargs.setdefault('default', '') + kwargs.setdefault('upload_to', self.get_upload_to(self.ext)) + + super().__init__(**kwargs) def deconstruct(self): - name, path, args, kwargs = super(EbookField, self).deconstruct() - args.insert(0, self.format_name) + name, path, args, kwargs = super().deconstruct() + if kwargs.get('max_length') == 255: + del kwargs['max_length'] + if kwargs.get('blank') is True: + del kwargs['blank'] + if kwargs.get('default') == '': + del kwargs['default'] + if self.get_upload_to(self.ext) == kwargs.get('upload_to'): + del kwargs['upload_to'] + # with_etag creates a second field, which then deconstructs to manage + # its own migrations. So for migrations, etag_field_name is explicitly + # set to avoid double creation of the etag field. + if self.with_etag: + kwargs['etag_field_name'] = self.etag_field_name + else: + kwargs['with_etag'] = self.with_etag + return name, path, args, kwargs - @property - def builder(self): - """Finds a celery task suitable for the format of the field.""" - return BuildEbook.for_format(self.format_name) + @classmethod + def get_upload_to(cls, directory): + directory = getattr(cls, 'directory', cls.ext) + upload_template = f'book/{directory}/%s.{cls.ext}' + return UploadToPath(upload_template) def contribute_to_class(self, cls, name): super(EbookField, self).contribute_to_class(cls, name) - self.etag_field_name = f'{name}_etag' + if self.with_etag and not self.etag_field_name: + self.etag_field_name = f'{name}_etag' + self.etag_field = models.CharField(max_length=255, editable=False, default='', db_index=True) + self.etag_field.contribute_to_class(cls, f'{name}_etag') def has(model_instance): return bool(getattr(model_instance, self.attname, None)) @@ -83,84 +147,58 @@ class EbookField(models.FileField): has.short_description = self.name has.boolean = True - self.registry.append(self) - setattr(cls, 'has_%s' % self.attname, has) def get_current_etag(self): - import pkg_resources + MediaInsertSet = apps.get_model('annoy', 'MediaInsertSet') librarian_version = pkg_resources.get_distribution("librarian").version - return librarian_version - - def schedule_stale(self, queryset=None): - """Schedule building this format for all the books where etag is stale.""" + etag = librarian_version + mis = MediaInsertSet.get_for_format(self.ext) + if mis is not None: + etag += '_' + mis.etag + return etag + + def find_stale(self, limit): + """Find some books where this format is stale.""" # If there is not ETag field, bail. That's true for xml file field. - if not hasattr(self.model, f'{self.attname}_etag'): - return + if not self.with_etag: + return [] etag = self.get_current_etag() - if queryset is None: - queryset = self.model.objects.all() - if self.format_name in EBOOK_FORMATS_WITHOUT_CHILDREN + ['html']: + queryset = self.model.objects.all() + if not self.for_parents: queryset = queryset.filter(children=None) queryset = queryset.exclude(**{ f'{self.etag_field_name}__in': [ etag, f'{etag}{ETAG_SCHEDULED_SUFFIX}' - ] + ] }) - for obj in queryset: - fieldfile = getattr(obj, self.attname) - priority = EBOOK_REBUILD_PRIORITY if fieldfile else EBOOK_BUILD_PRIORITY - fieldfile.build_delay(priority=priority) - - @classmethod - def schedule_all_stale(cls): - """Schedules all stale ebooks of all formats to rebuild.""" - for field in cls.registry: - field.schedule_stale() - - -class BuildEbook(Task): - librarian2_api = False - - formats = {} - - @classmethod - def register(cls, format_name): - """A decorator for registering subclasses for particular formats.""" - def wrapper(builder): - cls.formats[format_name] = builder - return builder - return wrapper + queryset = queryset.order_by('?')[:limit] + return queryset @classmethod - def for_format(cls, format_name): - """Returns a celery task suitable for specified format.""" - return cls.formats.get(format_name, BuildEbookTask) + def find_all_stale(cls, model, limit): + """Schedules all stale ebooks of all formats to rebuild.""" + found = [] + for field in model._meta.fields: + if isinstance(field, cls): + for instance in field.find_stale(limit): + found.append(( + field.name, + instance + )) + random.shuffle(found) + found = found[:limit] + return found @staticmethod - def transform(wldoc, fieldfile): + def transform(wldoc, book): """Transforms an librarian.WLDocument into an librarian.OutputFile. - - By default, it just calls relevant wldoc.as_??? method. - """ - return getattr(wldoc, "as_%s" % fieldfile.field.format_name)() - - def run(self, obj, field_name): - """Just run `build` on FieldFile, can't pass it directly to Celery.""" - fieldfile = getattr(obj, field_name) - - # Get etag value before actually building the file. - etag = fieldfile.field.get_current_etag() - task_logger.info("%s -> %s@%s" % (obj.slug, field_name, etag)) - ret = self.build(getattr(obj, field_name)) - fieldfile.update_etag(etag) - obj.clear_cache() - return ret + raise NotImplemented() def set_file_permissions(self, fieldfile): if fieldfile.instance.preview: @@ -170,70 +208,98 @@ class BuildEbook(Task): book = fieldfile.instance out = self.transform( book.wldocument2() if self.librarian2_api else book.wldocument(), - fieldfile) - fieldfile.save(None, File(open(out.get_filename(), 'rb')), save=False) + book, + ) + with open(out.get_filename(), 'rb') as f: + fieldfile.save(None, File(f), save=False) self.set_file_permissions(fieldfile) if book.pk is not None: - book.save(update_fields=[fieldfile.field.attname]) - if fieldfile.field.format_name in app_settings.FORMAT_ZIPS: - remove_zip(app_settings.FORMAT_ZIPS[fieldfile.field.format_name]) -# Don't decorate BuildEbook, because we want to subclass it. -BuildEbookTask = task(BuildEbook, ignore_result=True) + book.save(update_fields=[self.attname]) + if self.ZIP: + remove_zip(self.ZIP) + + +class XmlField(EbookField): + ext = 'xml' + + def build(self, fieldfile): + pass -@BuildEbook.register('txt') -@task(ignore_result=True) -class BuildTxt(BuildEbook): +class TxtField(EbookField): + ext = 'txt' + for_parents = False + @staticmethod - def transform(wldoc, fieldfile): + def transform(wldoc, book): return wldoc.as_text() -@BuildEbook.register('pdf') -@task(ignore_result=True) -class BuildPdf(BuildEbook): +class Fb2Field(EbookField): + ext = 'fb2' + for_parents = False + ZIP = 'wolnelektury_pl_fb2' + @staticmethod - def transform(wldoc, fieldfile): + def transform(wldoc, book): + return wldoc.as_fb2() + + +class PdfField(EbookField): + ext = 'pdf' + ZIP = 'wolnelektury_pl_pdf' + + @staticmethod + def transform(wldoc, book): + MediaInsertSet = apps.get_model('annoy', 'MediaInsertSet') return wldoc.as_pdf( - morefloats=settings.LIBRARIAN_PDF_MOREFLOATS, cover=True, - base_url=absolute_url(gallery_url(wldoc.book_info.url.slug)), customizations=['notoc']) + morefloats=settings.LIBRARIAN_PDF_MOREFLOATS, + cover=get_make_cover(book), + base_url=absolute_url(gallery_url(wldoc.book_info.url.slug)), customizations=['notoc'], + fundraising=MediaInsertSet.get_texts_for('pdf'), + ) def build(self, fieldfile): - BuildEbook.build(self, fieldfile) + super().build(fieldfile) clear_cache(fieldfile.instance.slug) -@BuildEbook.register('epub') -@task(ignore_result=True) -class BuildEpub(BuildEbook): +class EpubField(EbookField): + ext = 'epub' librarian2_api = True + ZIP = 'wolnelektury_pl_epub' @staticmethod - def transform(wldoc, fieldfile): + def transform(wldoc, book): from librarian.builders import EpubBuilder + MediaInsertSet = apps.get_model('annoy', 'MediaInsertSet') return EpubBuilder( base_url='file://' + os.path.abspath(gallery_path(wldoc.meta.url.slug)) + '/', - fundraising=settings.EPUB_FUNDRAISING + fundraising=MediaInsertSet.get_texts_for('epub'), + cover=get_make_cover(book), ).build(wldoc) -@BuildEbook.register('mobi') -@task(ignore_result=True) -class BuildMobi(BuildEbook): +class MobiField(EbookField): + ext = 'mobi' librarian2_api = True + ZIP = 'wolnelektury_pl_mobi' @staticmethod - def transform(wldoc, fieldfile): + def transform(wldoc, book): from librarian.builders import MobiBuilder + MediaInsertSet = apps.get_model('annoy', 'MediaInsertSet') return MobiBuilder( base_url='file://' + os.path.abspath(gallery_path(wldoc.meta.url.slug)) + '/', - fundraising=settings.EPUB_FUNDRAISING + fundraising=MediaInsertSet.get_texts_for('mobi'), + cover=get_make_cover(book), ).build(wldoc) -@BuildEbook.register('html') -@task(ignore_result=True) -class BuildHtml(BuildEbook): +class HtmlField(EbookField): + ext = 'html' + for_parents = False + def build(self, fieldfile): from django.core.files.base import ContentFile from slugify import slugify @@ -243,7 +309,7 @@ class BuildHtml(BuildEbook): book = fieldfile.instance - html_output = self.transform(book.wldocument(parse_dublincore=False), fieldfile) + html_output = self.transform(book.wldocument(parse_dublincore=False), book) # Delete old fragments, create from scratch if necessary. book.fragments.all().delete() @@ -323,7 +389,7 @@ class BuildHtml(BuildEbook): return False @staticmethod - def transform(wldoc, fieldfile): + def transform(wldoc, book): # ugly, but we can't use wldoc.book_info here from librarian import DCNS url_elem = wldoc.edoc.getroot().find('.//' + DCNS('identifier.url')) @@ -337,67 +403,57 @@ class BuildHtml(BuildEbook): return wldoc.as_html(gallery_path=gal_path, gallery_url=gal_url, base_url=absolute_url(gal_url)) -class BuildCover(BuildEbook): +class CoverField(EbookField): + ext = 'jpg' + directory = 'cover' + + @staticmethod + def transform(wldoc, book): + return get_make_cover(book)(wldoc.book_info, width=360).output_file() + def set_file_permissions(self, fieldfile): pass -@BuildEbook.register('cover_thumb') -@task(ignore_result=True) -class BuildCoverThumb(BuildCover): - @classmethod - def transform(cls, wldoc, fieldfile): +class CoverCleanField(CoverField): + directory = 'cover_clean' + + @staticmethod + def transform(wldoc, book): + return get_make_cover(book)(wldoc.book_info, width=360).output_file() + + +class CoverThumbField(CoverField): + directory = 'cover_thumb' + + @staticmethod + def transform(wldoc, book): from librarian.cover import WLCover return WLCover(wldoc.book_info, height=193).output_file() -@BuildEbook.register('cover_api_thumb') -@task(ignore_result=True) -class BuildCoverApiThumb(BuildCover): - @classmethod - def transform(cls, wldoc, fieldfile): +class CoverApiThumbField(CoverField): + directory = 'cover_api_thumb' + + @staticmethod + def transform(wldoc, book): from librarian.cover import WLNoBoxCover return WLNoBoxCover(wldoc.book_info, height=500).output_file() -@BuildEbook.register('simple_cover') -@task(ignore_result=True) -class BuildSimpleCover(BuildCover): - @classmethod - def transform(cls, wldoc, fieldfile): +class SimpleCoverField(CoverField): + directory = 'cover_simple' + + @staticmethod + def transform(wldoc, book): from librarian.cover import WLNoBoxCover return WLNoBoxCover(wldoc.book_info, height=1000).output_file() -@BuildEbook.register('cover_ebookpoint') -@task(ignore_result=True) -class BuildCoverEbookpoint(BuildCover): - @classmethod - def transform(cls, wldoc, fieldfile): +class CoverEbookpointField(CoverField): + directory = 'cover_ebookpoint' + + @staticmethod + def transform(wldoc, book): from librarian.cover import EbookpointCover return EbookpointCover(wldoc.book_info).output_file() - - -# not used, but needed for migrations -class OverwritingFieldFile(FieldFile): - """ - Deletes the old file before saving the new one. - """ - - def save(self, name, content, *args, **kwargs): - leave = kwargs.pop('leave', None) - # delete if there's a file already and there's a new one coming - if not leave and self and (not hasattr(content, 'path') or content.path != self.path): - self.delete(save=False) - return super(OverwritingFieldFile, self).save(name, content, *args, **kwargs) - - -class OverwritingFileField(models.FileField): - attr_class = OverwritingFieldFile - - -class OverwriteStorage(FileSystemStorage): - - def get_available_name(self, name, max_length=None): - self.delete(name) - return name