From 4157358510703a54cde8f3b0f9814f2cd1c9f40a Mon Sep 17 00:00:00 2001 From: Radek Czajka Date: Fri, 30 Sep 2022 11:53:55 +0200 Subject: [PATCH 1/1] Housekeeping: reorganize format fields, simplify the building tasks. --- src/catalogue/__init__.py | 28 -- src/catalogue/fields.py | 287 +++++++++--------- .../commands/schedule_stale_ebooks.py | 3 +- src/catalogue/migrations/0001_initial.py | 20 +- .../migrations/0008_auto_20151221_1225.py | 2 +- .../migrations/0014_auto_20170627_1442.py | 4 +- .../migrations/0020_book_cover_api_thumb.py | 3 +- .../migrations/0021_auto_20171222_1404.py | 5 +- .../migrations/0028_book_cover_ebookpoint.py | 3 +- .../migrations/0034_auto_20220310_1251.py | 3 +- .../migrations/0038_auto_20220930_1050.py | 80 +++++ src/catalogue/models/book.py | 122 ++------ src/catalogue/models/bookmedia.py | 5 +- src/catalogue/tasks.py | 12 +- src/catalogue/urls.py | 10 +- src/catalogue/views.py | 10 +- 16 files changed, 294 insertions(+), 303 deletions(-) create mode 100644 src/catalogue/migrations/0038_auto_20220930_1050.py diff --git a/src/catalogue/__init__.py b/src/catalogue/__init__.py index da120b574..e87986255 100644 --- a/src/catalogue/__init__.py +++ b/src/catalogue/__init__.py @@ -15,40 +15,12 @@ class Settings(AppSettings): DEFAULT_LANGUAGE = 'pol' # PDF needs TeXML + XeLaTeX, MOBI needs Calibre. DONT_BUILD = {'pdf', 'mobi'} - FORMAT_ZIPS = { - 'epub': 'wolnelektury_pl_epub', - 'pdf': 'wolnelektury_pl_pdf', - 'mobi': 'wolnelektury_pl_mobi', - 'fb2': 'wolnelektury_pl_fb2', - } REDAKCJA_URL = "http://redakcja.wolnelektury.pl" GOOD_LICENSES = {r'CC BY \d\.\d', r'CC BY-SA \d\.\d'} RELATED_RANDOM_PICTURE_CHANCE = .5 GET_MP3_LENGTH = 'catalogue.utils.get_mp3_length' - def _more_DONT_BUILD(self, value): - for format_ in ['cover', 'pdf', 'epub', 'mobi', 'fb2', 'txt']: - attname = 'NO_BUILD_%s' % format_.upper() - if hasattr(settings, attname): - logging.warn("%s is deprecated, use CATALOGUE_DONT_BUILD instead", attname) - if getattr(settings, attname): - value.add(format_) - else: - value.remove(format_) - return value - - def _more_FORMAT_ZIPS(self, value): - for format_ in ['epub', 'pdf', 'mobi', 'fb2']: - attname = 'ALL_%s_ZIP' % format_.upper() - if hasattr(settings, attname): - logging.warn( - "%s is deprecated, use CATALOGUE_FORMAT_ZIPS[%s] instead", - attname, format_ - ) - value[format_] = getattr(settings, attname) - return value - def _more_GET_MP3_LENGTH(self, value): return import_string(value) diff --git a/src/catalogue/fields.py b/src/catalogue/fields.py index 3d10cd8b8..38cac775b 100644 --- a/src/catalogue/fields.py +++ b/src/catalogue/fields.py @@ -4,44 +4,53 @@ import os from django.conf import settings from django.core.files import File -from django.core.files.storage import FileSystemStorage from django.db import models from django.db.models.fields.files import FieldFile -from catalogue import app_settings +from django.utils.deconstruct import deconstructible +from django.utils.translation import gettext_lazy as _ from catalogue.constants import LANGUAGES_3TO2, EBOOK_FORMATS_WITH_CHILDREN, EBOOK_FORMATS_WITHOUT_CHILDREN from catalogue.utils import absolute_url, remove_zip, truncate_html_words, gallery_path, gallery_url -#from celery import Task, shared_task -from celery.task import Task, task -from celery.utils.log import get_task_logger from waiter.utils import clear_cache -task_logger = get_task_logger(__name__) - ETAG_SCHEDULED_SUFFIX = '-scheduled' EBOOK_BUILD_PRIORITY = 0 EBOOK_REBUILD_PRIORITY = 9 +@deconstructible +class UploadToPath(object): + def __init__(self, path): + self.path = path + + def __call__(self, instance, filename): + return self.path % instance.slug + + def __eq__(self, other): + return isinstance(other, type(self)) and other.path == self.path + + class EbookFieldFile(FieldFile): """Represents contents of an ebook file field.""" def build(self): """Build the ebook immediately.""" - return self.field.builder.build(self) + etag = self.field.get_current_etag() + self.field.build(self) + self.update_etag(etag) + self.instance.clear_cache() def build_delay(self, priority=EBOOK_BUILD_PRIORITY): """Builds the ebook in a delayed task.""" + from .tasks import build_field + self.update_etag( "".join([self.field.get_current_etag(), ETAG_SCHEDULED_SUFFIX]) ) - return self.field.builder.apply_async( - [self.instance, self.field.attname], + return build_field.apply_async( + [self.instance.pk, self.field.attname], priority=priority ) - def get_url(self): - return self.instance.media_url(self.field.attname.split('_')[0]) - def set_readable(self, readable): import os permissions = 0o644 if readable else 0o600 @@ -56,26 +65,59 @@ class EbookFieldFile(FieldFile): class EbookField(models.FileField): """Represents an ebook file field, attachable to a model.""" attr_class = EbookFieldFile - registry = [] + ext = None + librarian2_api = False + ZIP = None - def __init__(self, format_name, *args, **kwargs): - super(EbookField, self).__init__(*args, **kwargs) - self.format_name = format_name + def __init__(self, verbose_name_=None, with_etag=True, **kwargs): + # This is just for compatibility with older migrations, + # where first argument was for ebook format. + # Can be scrapped if old migrations are updated/removed. + verbose_name = verbose_name_ or _("%s file") % self.ext + kwargs.setdefault('verbose_name', verbose_name_ ) + + self.with_etag = with_etag + kwargs.setdefault('max_length', 255) + kwargs.setdefault('blank', True) + kwargs.setdefault('default', '') + kwargs.setdefault('upload_to', self.get_upload_to(self.ext)) + + super().__init__(**kwargs) def deconstruct(self): - name, path, args, kwargs = super(EbookField, self).deconstruct() - args.insert(0, self.format_name) + name, path, args, kwargs = super().deconstruct() + if kwargs.get('max_length') == 255: + del kwargs['max_length'] + if kwargs.get('blank') is True: + del kwargs['blank'] + if kwargs.get('default') == '': + del kwargs['default'] + if self.get_upload_to(self.ext) == kwargs.get('upload_to'): + del kwargs['upload_to'] + if not self.with_etag: + kwargs['with_etag'] = self.with_etag + # Compatibility + verbose_name = kwargs.get('verbose_name') + if verbose_name: + del kwargs['verbose_name'] + if verbose_name != _("%s file") % self.ext: + args = [verbose_name] + args return name, path, args, kwargs - @property - def builder(self): - """Finds a celery task suitable for the format of the field.""" - return BuildEbook.for_format(self.format_name) + + @classmethod + def get_upload_to(cls, directory): + directory = getattr(cls, 'directory', cls.ext) + upload_template = f'book/{directory}/%s.{cls.ext}' + return UploadToPath(upload_template) def contribute_to_class(self, cls, name): super(EbookField, self).contribute_to_class(cls, name) self.etag_field_name = f'{name}_etag' + if self.with_etag: + self.etag_field = models.CharField(max_length=255, editable=False, default='', db_index=True) + self.etag_field.contribute_to_class(cls, f'{name}_etag') def has(model_instance): return bool(getattr(model_instance, self.attname, None)) @@ -84,8 +126,6 @@ class EbookField(models.FileField): has.short_description = self.name has.boolean = True - self.registry.append(self) - setattr(cls, 'has_%s' % self.attname, has) def get_current_etag(self): @@ -96,7 +136,7 @@ class EbookField(models.FileField): def schedule_stale(self, queryset=None): """Schedule building this format for all the books where etag is stale.""" # If there is not ETag field, bail. That's true for xml file field. - if not hasattr(self.model, f'{self.attname}_etag'): + if not self.with_etag: return etag = self.get_current_etag() @@ -117,51 +157,17 @@ class EbookField(models.FileField): fieldfile.build_delay(priority=priority) @classmethod - def schedule_all_stale(cls): + def schedule_all_stale(cls, model): """Schedules all stale ebooks of all formats to rebuild.""" - for field in cls.registry: - field.schedule_stale() - - - -class BuildEbook(Task): - librarian2_api = False - - formats = {} - - @classmethod - def register(cls, format_name): - """A decorator for registering subclasses for particular formats.""" - def wrapper(builder): - cls.formats[format_name] = builder - return builder - return wrapper - - @classmethod - def for_format(cls, format_name): - """Returns a celery task suitable for specified format.""" - return cls.formats.get(format_name, BuildEbookTask) + for field in model._meta.fields: + if isinstance(field, cls): + field.schedule_stale() @staticmethod - def transform(wldoc, fieldfile): + def transform(wldoc): """Transforms an librarian.WLDocument into an librarian.OutputFile. - - By default, it just calls relevant wldoc.as_??? method. - """ - return getattr(wldoc, "as_%s" % fieldfile.field.format_name)() - - def run(self, obj, field_name): - """Just run `build` on FieldFile, can't pass it directly to Celery.""" - fieldfile = getattr(obj, field_name) - - # Get etag value before actually building the file. - etag = fieldfile.field.get_current_etag() - task_logger.info("%s -> %s@%s" % (obj.slug, field_name, etag)) - ret = self.build(getattr(obj, field_name)) - fieldfile.update_etag(etag) - obj.clear_cache() - return ret + raise NotImplemented() def set_file_permissions(self, fieldfile): if fieldfile.instance.preview: @@ -171,30 +177,45 @@ class BuildEbook(Task): book = fieldfile.instance out = self.transform( book.wldocument2() if self.librarian2_api else book.wldocument(), - fieldfile) + ) fieldfile.save(None, File(open(out.get_filename(), 'rb')), save=False) self.set_file_permissions(fieldfile) if book.pk is not None: - book.save(update_fields=[fieldfile.field.attname]) - if fieldfile.field.format_name in app_settings.FORMAT_ZIPS: - remove_zip(app_settings.FORMAT_ZIPS[fieldfile.field.format_name]) -# Don't decorate BuildEbook, because we want to subclass it. -BuildEbookTask = task(BuildEbook, ignore_result=True) + book.save(update_fields=[self.attname]) + if self.ZIP: + remove_zip(self.ZIP) + + +class XmlField(EbookField): + ext = 'xml' + + def build(self, fieldfile): + pass -@BuildEbook.register('txt') -@task(ignore_result=True) -class BuildTxt(BuildEbook): +class TxtField(EbookField): + ext = 'txt' + @staticmethod - def transform(wldoc, fieldfile): + def transform(wldoc): return wldoc.as_text() -@BuildEbook.register('pdf') -@task(ignore_result=True) -class BuildPdf(BuildEbook): +class Fb2Field(EbookField): + ext = 'fb2' + ZIP = 'wolnelektury_pl_fb2' + + @staticmethod + def transform(wldoc): + return wldoc.as_fb2() + + +class PdfField(EbookField): + ext = 'pdf' + ZIP = 'wolnelektury_pl_pdf' + @staticmethod - def transform(wldoc, fieldfile): + def transform(wldoc): return wldoc.as_pdf( morefloats=settings.LIBRARIAN_PDF_MOREFLOATS, cover=True, base_url=absolute_url(gallery_url(wldoc.book_info.url.slug)), customizations=['notoc']) @@ -204,13 +225,13 @@ class BuildPdf(BuildEbook): clear_cache(fieldfile.instance.slug) -@BuildEbook.register('epub') -@task(ignore_result=True) -class BuildEpub(BuildEbook): +class EpubField(EbookField): + ext = 'epub' librarian2_api = True + ZIP = 'wolnelektury_pl_epub' @staticmethod - def transform(wldoc, fieldfile): + def transform(wldoc): from librarian.builders import EpubBuilder return EpubBuilder( base_url='file://' + os.path.abspath(gallery_path(wldoc.meta.url.slug)) + '/', @@ -218,13 +239,13 @@ class BuildEpub(BuildEbook): ).build(wldoc) -@BuildEbook.register('mobi') -@task(ignore_result=True) -class BuildMobi(BuildEbook): +class MobiField(EbookField): + ext = 'mobi' librarian2_api = True + ZIP = 'wolnelektury_pl_mobi' @staticmethod - def transform(wldoc, fieldfile): + def transform(wldoc): from librarian.builders import MobiBuilder return MobiBuilder( base_url='file://' + os.path.abspath(gallery_path(wldoc.meta.url.slug)) + '/', @@ -232,9 +253,9 @@ class BuildMobi(BuildEbook): ).build(wldoc) -@BuildEbook.register('html') -@task(ignore_result=True) -class BuildHtml(BuildEbook): +class HtmlField(EbookField): + ext = 'html' + def build(self, fieldfile): from django.core.files.base import ContentFile from slugify import slugify @@ -244,7 +265,7 @@ class BuildHtml(BuildEbook): book = fieldfile.instance - html_output = self.transform(book.wldocument(parse_dublincore=False), fieldfile) + html_output = self.transform(book.wldocument(parse_dublincore=False)) # Delete old fragments, create from scratch if necessary. book.fragments.all().delete() @@ -324,7 +345,7 @@ class BuildHtml(BuildEbook): return False @staticmethod - def transform(wldoc, fieldfile): + def transform(wldoc): # ugly, but we can't use wldoc.book_info here from librarian import DCNS url_elem = wldoc.edoc.getroot().find('.//' + DCNS('identifier.url')) @@ -338,16 +359,19 @@ class BuildHtml(BuildEbook): return wldoc.as_html(gallery_path=gal_path, gallery_url=gal_url, base_url=absolute_url(gal_url)) -class BuildCover(BuildEbook): +class CoverField(EbookField): + ext = 'jpg' + directory = 'cover' + def set_file_permissions(self, fieldfile): pass -@BuildEbook.register('cover_clean') -@task(ignore_result=True) -class BuildCoverClean(BuildCover): - @classmethod - def transform(cls, wldoc, fieldfile): +class CoverCleanField(CoverField): + directory = 'cover_clean' + + @staticmethod + def transform(wldoc): if wldoc.book_info.cover_box_position == 'none': from librarian.cover import WLCover return WLCover(wldoc.book_info, width=240).output_file() @@ -355,62 +379,37 @@ class BuildCoverClean(BuildCover): return MarquiseCover(wldoc.book_info, width=240).output_file() -@BuildEbook.register('cover_thumb') -@task(ignore_result=True) -class BuildCoverThumb(BuildCover): - @classmethod - def transform(cls, wldoc, fieldfile): +class CoverThumbField(CoverField): + directory = 'cover_thumb' + + @staticmethod + def transform(wldoc): from librarian.cover import WLCover return WLCover(wldoc.book_info, height=193).output_file() -@BuildEbook.register('cover_api_thumb') -@task(ignore_result=True) -class BuildCoverApiThumb(BuildCover): - @classmethod - def transform(cls, wldoc, fieldfile): +class CoverApiThumbField(CoverField): + directory = 'cover_api_thumb' + + @staticmethod + def transform(wldoc): from librarian.cover import WLNoBoxCover return WLNoBoxCover(wldoc.book_info, height=500).output_file() -@BuildEbook.register('simple_cover') -@task(ignore_result=True) -class BuildSimpleCover(BuildCover): - @classmethod - def transform(cls, wldoc, fieldfile): +class SimpleCoverField(CoverField): + directory = 'cover_simple' + + @staticmethod + def transform(wldoc): from librarian.cover import WLNoBoxCover return WLNoBoxCover(wldoc.book_info, height=1000).output_file() -@BuildEbook.register('cover_ebookpoint') -@task(ignore_result=True) -class BuildCoverEbookpoint(BuildCover): - @classmethod - def transform(cls, wldoc, fieldfile): +class CoverEbookpointField(CoverField): + directory = 'cover_ebookpoint' + + @staticmethod + def transform(wldoc): from librarian.cover import EbookpointCover return EbookpointCover(wldoc.book_info).output_file() - - -# not used, but needed for migrations -class OverwritingFieldFile(FieldFile): - """ - Deletes the old file before saving the new one. - """ - - def save(self, name, content, *args, **kwargs): - leave = kwargs.pop('leave', None) - # delete if there's a file already and there's a new one coming - if not leave and self and (not hasattr(content, 'path') or content.path != self.path): - self.delete(save=False) - return super(OverwritingFieldFile, self).save(name, content, *args, **kwargs) - - -class OverwritingFileField(models.FileField): - attr_class = OverwritingFieldFile - - -class OverwriteStorage(FileSystemStorage): - - def get_available_name(self, name, max_length=None): - self.delete(name) - return name diff --git a/src/catalogue/management/commands/schedule_stale_ebooks.py b/src/catalogue/management/commands/schedule_stale_ebooks.py index 91836fbf3..4420a815e 100644 --- a/src/catalogue/management/commands/schedule_stale_ebooks.py +++ b/src/catalogue/management/commands/schedule_stale_ebooks.py @@ -5,10 +5,11 @@ from django.conf import settings from django.core.management.base import BaseCommand from catalogue.fields import EbookField +from catalogue.models import Book class Command(BaseCommand): help = 'Schedule regenerating stale ebook files.' def handle(self, **options): - EbookField.schedule_all_stale() + EbookField.schedule_all_stale(Book) diff --git a/src/catalogue/migrations/0001_initial.py b/src/catalogue/migrations/0001_initial.py index ac8f9d738..e62b23358 100644 --- a/src/catalogue/migrations/0001_initial.py +++ b/src/catalogue/migrations/0001_initial.py @@ -35,16 +35,16 @@ class Migration(migrations.Migration): ('extra_info', models.TextField(default='{}', verbose_name='Additional information')), ('gazeta_link', models.CharField(max_length=240, blank=True)), ('wiki_link', models.CharField(max_length=240, blank=True)), - ('cover', catalogue.fields.EbookField('cover', upload_to=catalogue.models.book._cover_upload_to, storage=fnpdjango.storage.BofhFileSystemStorage(), max_length=255, blank=True, null=True, verbose_name='cover')), - ('cover_thumb', catalogue.fields.EbookField('cover_thumb', max_length=255, upload_to=catalogue.models.book._cover_thumb_upload_to, null=True, verbose_name='cover thumbnail', blank=True)), + ('cover', catalogue.fields.EbookField('cover', storage=fnpdjango.storage.BofhFileSystemStorage(), max_length=255, blank=True, null=True, verbose_name='cover')), + ('cover_thumb', catalogue.fields.EbookField('cover_thumb', max_length=255, null=True, verbose_name='cover thumbnail', blank=True)), ('_related_info', models.TextField(null=True, editable=False, blank=True)), - ('txt_file', catalogue.fields.EbookField('txt', default='', storage=fnpdjango.storage.BofhFileSystemStorage(), upload_to=catalogue.models.book._txt_upload_to, max_length=255, blank=True, verbose_name='TXT file')), - ('fb2_file', catalogue.fields.EbookField('fb2', default='', storage=fnpdjango.storage.BofhFileSystemStorage(), upload_to=catalogue.models.book._fb2_upload_to, max_length=255, blank=True, verbose_name='FB2 file')), - ('pdf_file', catalogue.fields.EbookField('pdf', default='', storage=fnpdjango.storage.BofhFileSystemStorage(), upload_to=catalogue.models.book._pdf_upload_to, max_length=255, blank=True, verbose_name='PDF file')), - ('epub_file', catalogue.fields.EbookField('epub', default='', storage=fnpdjango.storage.BofhFileSystemStorage(), upload_to=catalogue.models.book._epub_upload_to, max_length=255, blank=True, verbose_name='EPUB file')), - ('mobi_file', catalogue.fields.EbookField('mobi', default='', storage=fnpdjango.storage.BofhFileSystemStorage(), upload_to=catalogue.models.book._mobi_upload_to, max_length=255, blank=True, verbose_name='MOBI file')), - ('html_file', catalogue.fields.EbookField('html', default='', storage=fnpdjango.storage.BofhFileSystemStorage(), upload_to=catalogue.models.book._html_upload_to, max_length=255, blank=True, verbose_name='HTML file')), - ('xml_file', catalogue.fields.EbookField('xml', default='', storage=fnpdjango.storage.BofhFileSystemStorage(), upload_to=catalogue.models.book._xml_upload_to, max_length=255, blank=True, verbose_name='XML file')), + ('txt_file', catalogue.fields.EbookField('txt', default='', storage=fnpdjango.storage.BofhFileSystemStorage(), max_length=255, blank=True, verbose_name='TXT file')), + ('fb2_file', catalogue.fields.EbookField('fb2', default='', storage=fnpdjango.storage.BofhFileSystemStorage(), max_length=255, blank=True, verbose_name='FB2 file')), + ('pdf_file', catalogue.fields.EbookField('pdf', default='', storage=fnpdjango.storage.BofhFileSystemStorage(), max_length=255, blank=True, verbose_name='PDF file')), + ('epub_file', catalogue.fields.EbookField('epub', default='', storage=fnpdjango.storage.BofhFileSystemStorage(), max_length=255, blank=True, verbose_name='EPUB file')), + ('mobi_file', catalogue.fields.EbookField('mobi', default='', storage=fnpdjango.storage.BofhFileSystemStorage(), max_length=255, blank=True, verbose_name='MOBI file')), + ('html_file', catalogue.fields.EbookField('html', default='', storage=fnpdjango.storage.BofhFileSystemStorage(), max_length=255, blank=True, verbose_name='HTML file')), + ('xml_file', catalogue.fields.EbookField('xml', default='', storage=fnpdjango.storage.BofhFileSystemStorage(), max_length=255, blank=True, verbose_name='XML file')), ('parent', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='children', blank=True, to='catalogue.Book', null=True)), ], options={ @@ -60,7 +60,7 @@ class Migration(migrations.Migration): ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), ('type', models.CharField(db_index=True, max_length=20, verbose_name='type', choices=[('mp3', 'MP3 file'), ('ogg', 'Ogg Vorbis file'), ('daisy', 'DAISY file')])), ('name', models.CharField(max_length=512, verbose_name='name')), - ('file', catalogue.fields.OverwritingFileField(upload_to=catalogue.models.bookmedia._file_upload_to, max_length=600, verbose_name='XML file')), + ('file', models.FileField(upload_to=catalogue.models.bookmedia._file_upload_to, max_length=600, verbose_name='XML file')), ('uploaded_at', models.DateTimeField(auto_now_add=True, verbose_name='creation date', db_index=True)), ('extra_info', models.TextField(default='{}', verbose_name='Additional information', editable=False)), ('source_sha1', models.CharField(max_length=40, null=True, editable=False, blank=True)), diff --git a/src/catalogue/migrations/0008_auto_20151221_1225.py b/src/catalogue/migrations/0008_auto_20151221_1225.py index 04e79c38e..cdaaa0eb4 100644 --- a/src/catalogue/migrations/0008_auto_20151221_1225.py +++ b/src/catalogue/migrations/0008_auto_20151221_1225.py @@ -69,7 +69,7 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='bookmedia', name='file', - field=catalogue.fields.OverwritingFileField(upload_to=catalogue.models.bookmedia._file_upload_to, max_length=600, verbose_name='file'), + field=models.FileField(upload_to=catalogue.models.bookmedia._file_upload_to, max_length=600, verbose_name='file'), ), migrations.AlterField( model_name='collection', diff --git a/src/catalogue/migrations/0014_auto_20170627_1442.py b/src/catalogue/migrations/0014_auto_20170627_1442.py index 0affd0a19..cdfac230d 100644 --- a/src/catalogue/migrations/0014_auto_20170627_1442.py +++ b/src/catalogue/migrations/0014_auto_20170627_1442.py @@ -2,7 +2,7 @@ # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information. # from django.db import migrations, models -import catalogue.fields +import fnpdjango.storage import catalogue.models.bookmedia @@ -16,6 +16,6 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='bookmedia', name='file', - field=models.FileField(storage=catalogue.fields.OverwriteStorage(), upload_to=catalogue.models.bookmedia._file_upload_to, max_length=600, verbose_name='file'), + field=models.FileField(storage=fnpdjango.storage.BofhFileSystemStorage(), upload_to=catalogue.models.bookmedia._file_upload_to, max_length=600, verbose_name='file'), ), ] diff --git a/src/catalogue/migrations/0020_book_cover_api_thumb.py b/src/catalogue/migrations/0020_book_cover_api_thumb.py index 2d73807c7..0dba17502 100644 --- a/src/catalogue/migrations/0020_book_cover_api_thumb.py +++ b/src/catalogue/migrations/0020_book_cover_api_thumb.py @@ -3,7 +3,6 @@ # from django.db import migrations, models import catalogue.fields -import catalogue.models.book class Migration(migrations.Migration): @@ -16,6 +15,6 @@ class Migration(migrations.Migration): migrations.AddField( model_name='book', name='cover_api_thumb', - field=catalogue.fields.EbookField('cover_api_thumb', max_length=255, upload_to=catalogue.models.book.UploadToPath('book/cover_api_thumb/%s.jpg'), null=True, verbose_name='cover thumbnail for API', blank=True), + field=catalogue.fields.EbookField('cover_api_thumb', max_length=255, upload_to=catalogue.fields.UploadToPath('book/cover_api_thumb/%s.jpg'), null=True, verbose_name='cover thumbnail for API', blank=True), ), ] diff --git a/src/catalogue/migrations/0021_auto_20171222_1404.py b/src/catalogue/migrations/0021_auto_20171222_1404.py index 8f4e4f23a..e4a8762e2 100644 --- a/src/catalogue/migrations/0021_auto_20171222_1404.py +++ b/src/catalogue/migrations/0021_auto_20171222_1404.py @@ -3,7 +3,6 @@ # from django.db import migrations, models import catalogue.fields -import catalogue.models.book class Migration(migrations.Migration): @@ -16,11 +15,11 @@ class Migration(migrations.Migration): migrations.AddField( model_name='book', name='simple_cover', - field=catalogue.fields.EbookField('simple_cover', max_length=255, upload_to=catalogue.models.book.UploadToPath('book/cover_simple/%s.jpg'), null=True, verbose_name='cover for mobile app', blank=True), + field=catalogue.fields.EbookField('simple_cover', max_length=255, upload_to=catalogue.fields.UploadToPath('book/cover_simple/%s.jpg'), null=True, verbose_name='cover for mobile app', blank=True), ), migrations.AlterField( model_name='book', name='cover_api_thumb', - field=catalogue.fields.EbookField('cover_api_thumb', max_length=255, upload_to=catalogue.models.book.UploadToPath('book/cover_api_thumb/%s.jpg'), null=True, verbose_name='cover thumbnail for mobile app', blank=True), + field=catalogue.fields.EbookField('cover_api_thumb', max_length=255, upload_to=catalogue.fields.UploadToPath('book/cover_api_thumb/%s.jpg'), null=True, verbose_name='cover thumbnail for mobile app', blank=True), ), ] diff --git a/src/catalogue/migrations/0028_book_cover_ebookpoint.py b/src/catalogue/migrations/0028_book_cover_ebookpoint.py index 6c6e74cc7..b9ba1b064 100644 --- a/src/catalogue/migrations/0028_book_cover_ebookpoint.py +++ b/src/catalogue/migrations/0028_book_cover_ebookpoint.py @@ -1,7 +1,6 @@ # Generated by Django 2.2.10 on 2020-04-03 09:40 import catalogue.fields -import catalogue.models.book from django.db import migrations @@ -15,6 +14,6 @@ class Migration(migrations.Migration): migrations.AddField( model_name='book', name='cover_ebookpoint', - field=catalogue.fields.EbookField('cover_ebookpoint', blank=True, max_length=255, null=True, upload_to=catalogue.models.book.UploadToPath('book/cover_ebookpoint/%s.jpg'), verbose_name='cover for Ebookpoint'), + field=catalogue.fields.EbookField('cover_ebookpoint', blank=True, max_length=255, null=True, upload_to=catalogue.fields.UploadToPath('book/cover_ebookpoint/%s.jpg'), verbose_name='cover for Ebookpoint'), ), ] diff --git a/src/catalogue/migrations/0034_auto_20220310_1251.py b/src/catalogue/migrations/0034_auto_20220310_1251.py index 758106ab7..9360229a1 100644 --- a/src/catalogue/migrations/0034_auto_20220310_1251.py +++ b/src/catalogue/migrations/0034_auto_20220310_1251.py @@ -1,7 +1,6 @@ # Generated by Django 2.2.25 on 2022-03-10 11:51 import catalogue.fields -import catalogue.models.book from django.db import migrations, models @@ -15,7 +14,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='book', name='cover_clean', - field=catalogue.fields.EbookField('cover_clean', blank=True, max_length=255, null=True, upload_to=catalogue.models.book.UploadToPath('book/cover_clean/%s.jpg'), verbose_name='clean cover'), + field=catalogue.fields.EbookField('cover_clean', blank=True, max_length=255, null=True, upload_to=catalogue.fields.UploadToPath('book/cover_clean/%s.jpg'), verbose_name='clean cover'), ), migrations.AddField( model_name='book', diff --git a/src/catalogue/migrations/0038_auto_20220930_1050.py b/src/catalogue/migrations/0038_auto_20220930_1050.py new file mode 100644 index 000000000..4963e6690 --- /dev/null +++ b/src/catalogue/migrations/0038_auto_20220930_1050.py @@ -0,0 +1,80 @@ +# Generated by Django 2.2.28 on 2022-09-30 08:50 + +import catalogue.fields +from django.db import migrations +import fnpdjango.storage + + +class Migration(migrations.Migration): + + dependencies = [ + ('catalogue', '0037_auto_20220928_0124'), + ] + + operations = [ + migrations.AlterField( + model_name='book', + name='cover', + field=catalogue.fields.CoverField('cover', storage=fnpdjango.storage.BofhFileSystemStorage()), + ), + migrations.AlterField( + model_name='book', + name='cover_api_thumb', + field=catalogue.fields.CoverApiThumbField('cover thumbnail for mobile app'), + ), + migrations.AlterField( + model_name='book', + name='cover_clean', + field=catalogue.fields.CoverCleanField('clean cover'), + ), + migrations.AlterField( + model_name='book', + name='cover_ebookpoint', + field=catalogue.fields.CoverEbookpointField('cover for Ebookpoint'), + ), + migrations.AlterField( + model_name='book', + name='cover_thumb', + field=catalogue.fields.CoverThumbField('cover thumbnail'), + ), + migrations.AlterField( + model_name='book', + name='epub_file', + field=catalogue.fields.EpubField(storage=fnpdjango.storage.BofhFileSystemStorage()), + ), + migrations.AlterField( + model_name='book', + name='fb2_file', + field=catalogue.fields.Fb2Field(storage=fnpdjango.storage.BofhFileSystemStorage()), + ), + migrations.AlterField( + model_name='book', + name='html_file', + field=catalogue.fields.HtmlField(storage=fnpdjango.storage.BofhFileSystemStorage()), + ), + migrations.AlterField( + model_name='book', + name='mobi_file', + field=catalogue.fields.MobiField(storage=fnpdjango.storage.BofhFileSystemStorage()), + ), + migrations.AlterField( + model_name='book', + name='pdf_file', + field=catalogue.fields.PdfField(storage=fnpdjango.storage.BofhFileSystemStorage()), + ), + migrations.AlterField( + model_name='book', + name='simple_cover', + field=catalogue.fields.SimpleCoverField('cover for mobile app'), + ), + migrations.AlterField( + model_name='book', + name='txt_file', + field=catalogue.fields.TxtField(storage=fnpdjango.storage.BofhFileSystemStorage()), + ), + migrations.AlterField( + model_name='book', + name='xml_file', + field=catalogue.fields.XmlField(storage=fnpdjango.storage.BofhFileSystemStorage(), with_etag=False), + ), + ] diff --git a/src/catalogue/models/book.py b/src/catalogue/models/book.py index 7d923ea3c..fc07fc5f7 100644 --- a/src/catalogue/models/book.py +++ b/src/catalogue/models/book.py @@ -16,45 +16,22 @@ from django.contrib.contenttypes.fields import GenericRelation from django.template.loader import render_to_string from django.urls import reverse from django.utils.translation import ugettext_lazy as _, get_language -from django.utils.deconstruct import deconstructible from fnpdjango.storage import BofhFileSystemStorage from lxml import html from librarian.cover import WLCover from librarian.html import transform_abstrakt from newtagging import managers from catalogue import constants -from catalogue.fields import EbookField +from catalogue import fields from catalogue.models import Tag, Fragment, BookMedia from catalogue.utils import create_zip, gallery_url, gallery_path, split_tags, get_random_hash from catalogue.models.tag import prefetched_relations from catalogue import app_settings -from catalogue import tasks from wolnelektury.utils import makedirs, cached_render, clear_cached_renders bofh_storage = BofhFileSystemStorage() -@deconstructible -class UploadToPath(object): - def __init__(self, path): - self.path = path - - def __call__(self, instance, filename): - return self.path % instance.slug - - -_cover_upload_to = UploadToPath('book/cover/%s.jpg') -_cover_clean_upload_to = UploadToPath('book/cover_clean/%s.jpg') -_cover_thumb_upload_to = UploadToPath('book/cover_thumb/%s.jpg') -_cover_api_thumb_upload_to = UploadToPath('book/cover_api_thumb/%s.jpg') -_simple_cover_upload_to = UploadToPath('book/cover_simple/%s.jpg') -_cover_ebookpoint_upload_to = UploadToPath('book/cover_ebookpoint/%s.jpg') - - -def _ebook_upload_to(upload_path): - return UploadToPath(upload_path) - - class Book(models.Model): """Represents a book imported from WL-XML.""" title = models.CharField(_('title'), max_length=32767) @@ -82,44 +59,24 @@ class Book(models.Model): findable = models.BooleanField(_('findable'), default=True, db_index=True) # files generated during publication - cover = EbookField( - 'cover', _('cover'), - null=True, blank=True, - upload_to=_cover_upload_to, - storage=bofh_storage, max_length=255) - cover_etag = models.CharField(max_length=255, editable=False, default='', db_index=True) + xml_file = fields.XmlField(storage=bofh_storage, with_etag=False) + html_file = fields.HtmlField(storage=bofh_storage) + fb2_file = fields.Fb2Field(storage=bofh_storage) + txt_file = fields.TxtField(storage=bofh_storage) + epub_file = fields.EpubField(storage=bofh_storage) + mobi_file = fields.MobiField(storage=bofh_storage) + pdf_file = fields.PdfField(storage=bofh_storage) + + cover = fields.CoverField(_('cover'), storage=bofh_storage) # Cleaner version of cover for thumbs - cover_clean = EbookField( - 'cover_clean', _('clean cover'), - null=True, blank=True, - upload_to=_cover_clean_upload_to, - max_length=255 - ) - cover_clean_etag = models.CharField(max_length=255, editable=False, default='', db_index=True) - cover_thumb = EbookField( - 'cover_thumb', _('cover thumbnail'), - null=True, blank=True, - upload_to=_cover_thumb_upload_to, - max_length=255) - cover_thumb_etag = models.CharField(max_length=255, editable=False, default='', db_index=True) - cover_api_thumb = EbookField( - 'cover_api_thumb', _('cover thumbnail for mobile app'), - null=True, blank=True, - upload_to=_cover_api_thumb_upload_to, - max_length=255) - cover_api_thumb_etag = models.CharField(max_length=255, editable=False, default='', db_index=True) - simple_cover = EbookField( - 'simple_cover', _('cover for mobile app'), - null=True, blank=True, - upload_to=_simple_cover_upload_to, - max_length=255) - simple_cover_etag = models.CharField(max_length=255, editable=False, default='', db_index=True) - cover_ebookpoint = EbookField( - 'cover_ebookpoint', _('cover for Ebookpoint'), - null=True, blank=True, - upload_to=_cover_ebookpoint_upload_to, - max_length=255) - cover_ebookpoint_etag = models.CharField(max_length=255, editable=False, default='', db_index=True) + cover_clean = fields.CoverCleanField(_('clean cover')) + cover_thumb = fields.CoverThumbField(_('cover thumbnail')) + cover_api_thumb = fields.CoverApiThumbField( + _('cover thumbnail for mobile app')) + simple_cover = fields.SimpleCoverField(_('cover for mobile app')) + cover_ebookpoint = fields.CoverEbookpointField( + _('cover for Ebookpoint')) + ebook_formats = constants.EBOOK_FORMATS formats = ebook_formats + ['html', 'xml'] @@ -295,7 +252,7 @@ class Book(models.Model): if self.parent.html_file: return self.parent - + return self.parent.get_prev_text() def get_next_text(self, inside=True): @@ -329,7 +286,7 @@ class Book(models.Model): def get_children(self): return self.children.all().order_by('parent_number') - + @property def name(self): return self.title @@ -463,7 +420,7 @@ class Book(models.Model): @property def media_daisy(self): return self.get_media('daisy') - + @property def media_audio_epub(self): return self.get_media('audio.epub') @@ -536,9 +493,10 @@ class Book(models.Model): format_) field_name = "%s_file" % format_ + field = getattr(Book, field_name) books = Book.objects.filter(parent=None).exclude(**{field_name: ""}).exclude(preview=True).exclude(findable=False) paths = [(pretty_file_name(b), getattr(b, field_name).path) for b in books.iterator()] - return create_zip(paths, app_settings.FORMAT_ZIPS[format_]) + return create_zip(paths, field.ZIP) def zip_audiobooks(self, format_): bm = BookMedia.objects.filter(book=self, type=format_) @@ -610,7 +568,7 @@ class Book(models.Model): a.attrib['href'] = html_link + a.attrib['href'] self.toc = html.tostring(toc, encoding='unicode') # div#toc - + @classmethod def from_xml_file(cls, xml_file, **kwargs): from django.core.files import File @@ -630,6 +588,8 @@ class Book(models.Model): @classmethod def from_text_and_meta(cls, raw_file, book_info, overwrite=False, dont_build=None, search_index=True, search_index_tags=True, remote_gallery_url=None, days=0, findable=True): + from catalogue import tasks + if dont_build is None: dont_build = set() dont_build = set.union(set(dont_build), set(app_settings.DONT_BUILD)) @@ -762,7 +722,7 @@ class Book(models.Model): for master in root.iter(): if master.tag in master_tags: return master - + def update_references(self): from references.models import Entity, Reference master = self.get_master() @@ -786,7 +746,7 @@ class Book(models.Model): entity.populate() entity.save() Reference.objects.filter(book=self).exclude(entity__uri__in=found).delete() - + @property def references(self): return self.reference_set.all().select_related('entity') @@ -832,7 +792,7 @@ class Book(models.Model): yield self.parent else: return [] - + def clear_cache(self): clear_cached_renders(self.mini_box) clear_cached_renders(self.mini_box_nolink) @@ -1003,7 +963,7 @@ class Book(models.Model): return fragments[0] else: return None - + def fragment_data(self): fragment = self.choose_fragment() if fragment: @@ -1056,28 +1016,6 @@ class Book(models.Model): 'no_link': True, } -def add_file_fields(): - for format_ in Book.formats: - field_name = "%s_file" % format_ - # This weird globals() assignment makes Django migrations comfortable. - _upload_to = _ebook_upload_to('book/%s/%%s.%s' % (format_, format_)) - _upload_to.__name__ = '_%s_upload_to' % format_ - globals()[_upload_to.__name__] = _upload_to - - EbookField( - format_, _("%s file" % format_.upper()), - upload_to=_upload_to, - storage=bofh_storage, - max_length=255, - blank=True, - default='' - ).contribute_to_class(Book, field_name) - if format_ != 'xml': - models.CharField(max_length=255, editable=False, default='', db_index=True).contribute_to_class(Book, f'{field_name}_etag') - - -add_file_fields() - class BookPopularity(models.Model): book = models.OneToOneField(Book, models.CASCADE, related_name='popularity') diff --git a/src/catalogue/models/bookmedia.py b/src/catalogue/models/bookmedia.py index 702dd01f8..541940561 100644 --- a/src/catalogue/models/bookmedia.py +++ b/src/catalogue/models/bookmedia.py @@ -9,8 +9,7 @@ from django.utils.translation import ugettext_lazy as _ from slugify import slugify import mutagen from mutagen import id3 - -from catalogue.fields import OverwriteStorage +from fnpdjango.storage import BofhFileSystemStorage def _file_upload_to(i, _n): @@ -38,7 +37,7 @@ class BookMedia(models.Model): name = models.CharField(_('name'), max_length=512) part_name = models.CharField(_('part name'), default='', blank=True, max_length=512) index = models.IntegerField(_('index'), default=0) - file = models.FileField(_('file'), max_length=600, upload_to=_file_upload_to, storage=OverwriteStorage()) + file = models.FileField(_('file'), max_length=600, upload_to=_file_upload_to, storage=BofhFileSystemStorage()) duration = models.IntegerField(null=True, blank=True) uploaded_at = models.DateTimeField(_('creation date'), auto_now_add=True, editable=False, db_index=True) project_description = models.CharField(max_length=2048, blank=True) diff --git a/src/catalogue/tasks.py b/src/catalogue/tasks.py index b703e171f..e16f1ff5a 100644 --- a/src/catalogue/tasks.py +++ b/src/catalogue/tasks.py @@ -7,6 +7,7 @@ from celery.utils.log import get_task_logger from django.conf import settings from django.utils import timezone +from catalogue.models import Book from catalogue.utils import absolute_url, gallery_url from waiter.models import WaitedFile @@ -22,9 +23,16 @@ def touch_tag(tag): type(tag).objects.filter(pk=tag.pk).update(**update_dict) +@shared_task(ignore_result=True) +def build_field(pk, field_name): + book = Book.objects.get(pk=pk) + task_logger.info("build %s.%s" % (book.slug, field_name)) + field_file = getattr(book, field_name) + field_file.build() + + @shared_task def index_book(book_id, book_info=None, **kwargs): - from catalogue.models import Book try: return Book.objects.get(id=book_id).search_index(book_info, **kwargs) except Exception as e: @@ -39,7 +47,6 @@ def build_custom_pdf(book_id, customizations, file_name, waiter_id=None): try: from django.core.files import File from django.core.files.storage import DefaultStorage - from catalogue.models import Book task_logger.info(DefaultStorage().path(file_name)) if not DefaultStorage().exists(file_name): @@ -69,6 +76,5 @@ def update_counters(): @shared_task(ignore_result=True) def update_references(book_id): - from catalogue.models import Book Book.objects.get(id=book_id).update_references() diff --git a/src/catalogue/urls.py b/src/catalogue/urls.py index 7f1eddbae..3cb17502b 100644 --- a/src/catalogue/urls.py +++ b/src/catalogue/urls.py @@ -55,11 +55,11 @@ urlpatterns = [ path('pobierz//.', views.embargo_link, name='embargo_link'), # zip - path('zip/pdf.zip', views.download_zip, {'format': 'pdf', 'slug': None}, 'download_zip_pdf'), - path('zip/epub.zip', views.download_zip, {'format': 'epub', 'slug': None}, 'download_zip_epub'), - path('zip/mobi.zip', views.download_zip, {'format': 'mobi', 'slug': None}, 'download_zip_mobi'), - path('zip/mp3/.zip', views.download_zip, {'format': 'mp3'}, 'download_zip_mp3'), - path('zip/ogg/.zip', views.download_zip, {'format': 'ogg'}, 'download_zip_ogg'), + path('zip/pdf.zip', views.download_zip, {'file_format': 'pdf', 'slug': None}, 'download_zip_pdf'), + path('zip/epub.zip', views.download_zip, {'file_format': 'epub', 'slug': None}, 'download_zip_epub'), + path('zip/mobi.zip', views.download_zip, {'file_format': 'mobi', 'slug': None}, 'download_zip_mobi'), + path('zip/mp3/.zip', views.download_zip, {'media_format': 'mp3'}, 'download_zip_mp3'), + path('zip/ogg/.zip', views.download_zip, {'media_format': 'ogg'}, 'download_zip_ogg'), # Public interface. Do not change this URLs. path('lektura/.html', views.book_text, name='book_text'), diff --git a/src/catalogue/views.py b/src/catalogue/views.py index c63aa7a9e..816836d64 100644 --- a/src/catalogue/views.py +++ b/src/catalogue/views.py @@ -408,12 +408,12 @@ def embargo_link(request, key, format_, slug): return HttpResponse(media_file, content_type=constants.EBOOK_CONTENT_TYPES[format_]) -def download_zip(request, format, slug=None): - if format in Book.ebook_formats: - url = Book.zip_format(format) - elif format in ('mp3', 'ogg') and slug is not None: +def download_zip(request, file_format=None, media_format=None, slug=None): + if file_format: + url = Book.zip_format(file_format) + elif media_format and slug is not None: book = get_object_or_404(Book, slug=slug) - url = book.zip_audiobooks(format) + url = book.zip_audiobooks(media_format) else: raise Http404('No format specified for zip package') return HttpResponseRedirect(urlquote_plus(settings.MEDIA_URL + url, safe='/?=')) -- 2.20.1