From: Radek Czajka <rczajka@rczajka.pl>
Date: Fri, 30 Sep 2022 09:53:55 +0000 (+0200)
Subject: Housekeeping: reorganize format fields, simplify the building tasks.
X-Git-Url: https://git.mdrn.pl/wolnelektury.git/commitdiff_plain/4157358510703a54cde8f3b0f9814f2cd1c9f40a?hp=e1688333f557a60400fed09b4b087f4c8dddb432

Housekeeping: reorganize format fields, simplify the building tasks.
---

diff --git a/src/catalogue/__init__.py b/src/catalogue/__init__.py
index da120b574..e87986255 100644
--- a/src/catalogue/__init__.py
+++ b/src/catalogue/__init__.py
@@ -15,40 +15,12 @@ class Settings(AppSettings):
     DEFAULT_LANGUAGE = 'pol'
     # PDF needs TeXML + XeLaTeX, MOBI needs Calibre.
     DONT_BUILD = {'pdf', 'mobi'}
-    FORMAT_ZIPS = {
-        'epub': 'wolnelektury_pl_epub',
-        'pdf': 'wolnelektury_pl_pdf',
-        'mobi': 'wolnelektury_pl_mobi',
-        'fb2': 'wolnelektury_pl_fb2',
-    }
 
     REDAKCJA_URL = "http://redakcja.wolnelektury.pl"
     GOOD_LICENSES = {r'CC BY \d\.\d', r'CC BY-SA \d\.\d'}
     RELATED_RANDOM_PICTURE_CHANCE = .5
     GET_MP3_LENGTH = 'catalogue.utils.get_mp3_length'
 
-    def _more_DONT_BUILD(self, value):
-        for format_ in ['cover', 'pdf', 'epub', 'mobi', 'fb2', 'txt']:
-            attname = 'NO_BUILD_%s' % format_.upper()
-            if hasattr(settings, attname):
-                logging.warn("%s is deprecated, use CATALOGUE_DONT_BUILD instead", attname)
-                if getattr(settings, attname):
-                    value.add(format_)
-                else:
-                    value.remove(format_)
-        return value
-
-    def _more_FORMAT_ZIPS(self, value):
-        for format_ in ['epub', 'pdf', 'mobi', 'fb2']:
-            attname = 'ALL_%s_ZIP' % format_.upper()
-            if hasattr(settings, attname):
-                logging.warn(
-                    "%s is deprecated, use CATALOGUE_FORMAT_ZIPS[%s] instead",
-                    attname, format_
-                )
-                value[format_] = getattr(settings, attname)
-        return value
-
     def _more_GET_MP3_LENGTH(self, value):
         return import_string(value)
 
diff --git a/src/catalogue/fields.py b/src/catalogue/fields.py
index 3d10cd8b8..38cac775b 100644
--- a/src/catalogue/fields.py
+++ b/src/catalogue/fields.py
@@ -4,44 +4,53 @@
 import os
 from django.conf import settings
 from django.core.files import File
-from django.core.files.storage import FileSystemStorage
 from django.db import models
 from django.db.models.fields.files import FieldFile
-from catalogue import app_settings
+from django.utils.deconstruct import deconstructible
+from django.utils.translation import gettext_lazy as _
 from catalogue.constants import LANGUAGES_3TO2, EBOOK_FORMATS_WITH_CHILDREN, EBOOK_FORMATS_WITHOUT_CHILDREN
 from catalogue.utils import absolute_url, remove_zip, truncate_html_words, gallery_path, gallery_url
-#from celery import Task, shared_task
-from celery.task import Task, task
-from celery.utils.log import get_task_logger
 from waiter.utils import clear_cache
 
-task_logger = get_task_logger(__name__)
-
 ETAG_SCHEDULED_SUFFIX = '-scheduled'
 EBOOK_BUILD_PRIORITY = 0
 EBOOK_REBUILD_PRIORITY = 9
 
 
+@deconstructible
+class UploadToPath(object):
+    def __init__(self, path):
+        self.path = path
+
+    def __call__(self, instance, filename):
+        return self.path % instance.slug
+
+    def __eq__(self, other):
+        return isinstance(other, type(self)) and other.path == self.path
+
+
 class EbookFieldFile(FieldFile):
     """Represents contents of an ebook file field."""
 
     def build(self):
         """Build the ebook immediately."""
-        return self.field.builder.build(self)
+        etag = self.field.get_current_etag()
+        self.field.build(self)
+        self.update_etag(etag)
+        self.instance.clear_cache()
 
     def build_delay(self, priority=EBOOK_BUILD_PRIORITY):
         """Builds the ebook in a delayed task."""
+        from .tasks import build_field
+
         self.update_etag(
             "".join([self.field.get_current_etag(), ETAG_SCHEDULED_SUFFIX])
         )
-        return self.field.builder.apply_async(
-            [self.instance, self.field.attname],
+        return build_field.apply_async(
+            [self.instance.pk, self.field.attname],
             priority=priority
         )
 
-    def get_url(self):
-        return self.instance.media_url(self.field.attname.split('_')[0])
-
     def set_readable(self, readable):
         import os
         permissions = 0o644 if readable else 0o600
@@ -56,26 +65,59 @@ class EbookFieldFile(FieldFile):
 class EbookField(models.FileField):
     """Represents an ebook file field, attachable to a model."""
     attr_class = EbookFieldFile
-    registry = []
+    ext = None
+    librarian2_api = False
+    ZIP = None
 
-    def __init__(self, format_name, *args, **kwargs):
-        super(EbookField, self).__init__(*args, **kwargs)
-        self.format_name = format_name
+    def __init__(self, verbose_name_=None, with_etag=True, **kwargs):
+        # This is just for compatibility with older migrations,
+        # where first argument was for ebook format.
+        # Can be scrapped if old migrations are updated/removed.
+        verbose_name = verbose_name_ or _("%s file") % self.ext
+        kwargs.setdefault('verbose_name', verbose_name_ )
+
+        self.with_etag = with_etag
+        kwargs.setdefault('max_length', 255)
+        kwargs.setdefault('blank', True)
+        kwargs.setdefault('default', '')
+        kwargs.setdefault('upload_to', self.get_upload_to(self.ext))
+
+        super().__init__(**kwargs)
 
     def deconstruct(self):
-        name, path, args, kwargs = super(EbookField, self).deconstruct()
-        args.insert(0, self.format_name)
+        name, path, args, kwargs = super().deconstruct()
+        if kwargs.get('max_length') == 255:
+            del kwargs['max_length']
+        if kwargs.get('blank') is True:
+            del kwargs['blank']
+        if kwargs.get('default') == '':
+            del kwargs['default']
+        if self.get_upload_to(self.ext) == kwargs.get('upload_to'):
+            del kwargs['upload_to']
+        if not self.with_etag:
+            kwargs['with_etag'] = self.with_etag
+        # Compatibility
+        verbose_name = kwargs.get('verbose_name')
+        if verbose_name:
+            del kwargs['verbose_name']
+            if verbose_name != _("%s file") % self.ext:
+                args = [verbose_name] + args
         return name, path, args, kwargs
 
-    @property
-    def builder(self):
-        """Finds a celery task suitable for the format of the field."""
-        return BuildEbook.for_format(self.format_name)
+
+    @classmethod
+    def get_upload_to(cls, directory):
+        directory = getattr(cls, 'directory', cls.ext)
+        upload_template = f'book/{directory}/%s.{cls.ext}'
+        return UploadToPath(upload_template)
 
     def contribute_to_class(self, cls, name):
         super(EbookField, self).contribute_to_class(cls, name)
 
         self.etag_field_name = f'{name}_etag'
+        if self.with_etag:
+            self.etag_field = models.CharField(max_length=255, editable=False, default='', db_index=True)
+            self.etag_field.contribute_to_class(cls, f'{name}_etag')
 
         def has(model_instance):
             return bool(getattr(model_instance, self.attname, None))
@@ -84,8 +126,6 @@ class EbookField(models.FileField):
         has.short_description = self.name
         has.boolean = True
 
-        self.registry.append(self)
-
         setattr(cls, 'has_%s' % self.attname, has)
 
     def get_current_etag(self):
@@ -96,7 +136,7 @@ class EbookField(models.FileField):
     def schedule_stale(self, queryset=None):
         """Schedule building this format for all the books where etag is stale."""
         # If there is not ETag field, bail. That's true for xml file field.
-        if not hasattr(self.model, f'{self.attname}_etag'):
+        if not self.with_etag:
             return
 
         etag = self.get_current_etag()
@@ -117,51 +157,17 @@ class EbookField(models.FileField):
             fieldfile.build_delay(priority=priority)
 
     @classmethod
-    def schedule_all_stale(cls):
+    def schedule_all_stale(cls, model):
         """Schedules all stale ebooks of all formats to rebuild."""
-        for field in cls.registry:
-            field.schedule_stale()
-
-
-
-class BuildEbook(Task):
-    librarian2_api = False
-
-    formats = {}
-
-    @classmethod
-    def register(cls, format_name):
-        """A decorator for registering subclasses for particular formats."""
-        def wrapper(builder):
-            cls.formats[format_name] = builder
-            return builder
-        return wrapper
-
-    @classmethod
-    def for_format(cls, format_name):
-        """Returns a celery task suitable for specified format."""
-        return cls.formats.get(format_name, BuildEbookTask)
+        for field in model._meta.fields:
+            if isinstance(field, cls):
+                field.schedule_stale()
 
     @staticmethod
-    def transform(wldoc, fieldfile):
+    def transform(wldoc):
         """Transforms an librarian.WLDocument into an librarian.OutputFile.
-
-        By default, it just calls relevant wldoc.as_??? method.
-
         """
-        return getattr(wldoc, "as_%s" % fieldfile.field.format_name)()
-
-    def run(self, obj, field_name):
-        """Just run `build` on FieldFile, can't pass it directly to Celery."""
-        fieldfile = getattr(obj, field_name)
-
-        # Get etag value before actually building the file.
-        etag = fieldfile.field.get_current_etag()
-        task_logger.info("%s -> %s@%s" % (obj.slug, field_name, etag))
-        ret = self.build(getattr(obj, field_name))
-        fieldfile.update_etag(etag)
-        obj.clear_cache()
-        return ret
+        raise NotImplemented()
 
     def set_file_permissions(self, fieldfile):
         if fieldfile.instance.preview:
@@ -171,30 +177,45 @@ class BuildEbook(Task):
         book = fieldfile.instance
         out = self.transform(
             book.wldocument2() if self.librarian2_api else book.wldocument(),
-            fieldfile)
+        )
         fieldfile.save(None, File(open(out.get_filename(), 'rb')), save=False)
         self.set_file_permissions(fieldfile)
         if book.pk is not None:
-            book.save(update_fields=[fieldfile.field.attname])
-        if fieldfile.field.format_name in app_settings.FORMAT_ZIPS:
-            remove_zip(app_settings.FORMAT_ZIPS[fieldfile.field.format_name])
-# Don't decorate BuildEbook, because we want to subclass it.
-BuildEbookTask = task(BuildEbook, ignore_result=True)
+            book.save(update_fields=[self.attname])
+        if self.ZIP:
+            remove_zip(self.ZIP)
+
+
+class XmlField(EbookField):
+    ext = 'xml'
+
+    def build(self, fieldfile):
+        pass
 
 
-@BuildEbook.register('txt')
-@task(ignore_result=True)
-class BuildTxt(BuildEbook):
+class TxtField(EbookField):
+    ext = 'txt'
+
     @staticmethod
-    def transform(wldoc, fieldfile):
+    def transform(wldoc):
         return wldoc.as_text()
 
 
-@BuildEbook.register('pdf')
-@task(ignore_result=True)
-class BuildPdf(BuildEbook):
+class Fb2Field(EbookField):
+    ext = 'fb2'
+    ZIP = 'wolnelektury_pl_fb2'
+
+    @staticmethod
+    def transform(wldoc):
+        return wldoc.as_fb2()
+
+
+class PdfField(EbookField):
+    ext = 'pdf'
+    ZIP = 'wolnelektury_pl_pdf'
+
     @staticmethod
-    def transform(wldoc, fieldfile):
+    def transform(wldoc):
         return wldoc.as_pdf(
             morefloats=settings.LIBRARIAN_PDF_MOREFLOATS, cover=True,
             base_url=absolute_url(gallery_url(wldoc.book_info.url.slug)), customizations=['notoc'])
@@ -204,13 +225,13 @@ class BuildPdf(BuildEbook):
         clear_cache(fieldfile.instance.slug)
 
 
-@BuildEbook.register('epub')
-@task(ignore_result=True)
-class BuildEpub(BuildEbook):
+class EpubField(EbookField):
+    ext = 'epub'
     librarian2_api = True
+    ZIP = 'wolnelektury_pl_epub'
 
     @staticmethod
-    def transform(wldoc, fieldfile):
+    def transform(wldoc):
         from librarian.builders import EpubBuilder
         return EpubBuilder(
                 base_url='file://' + os.path.abspath(gallery_path(wldoc.meta.url.slug)) + '/',
@@ -218,13 +239,13 @@ class BuildEpub(BuildEbook):
             ).build(wldoc)
 
 
-@BuildEbook.register('mobi')
-@task(ignore_result=True)
-class BuildMobi(BuildEbook):
+class MobiField(EbookField):
+    ext = 'mobi'
     librarian2_api = True
+    ZIP = 'wolnelektury_pl_mobi'
 
     @staticmethod
-    def transform(wldoc, fieldfile):
+    def transform(wldoc):
         from librarian.builders import MobiBuilder
         return MobiBuilder(
                 base_url='file://' + os.path.abspath(gallery_path(wldoc.meta.url.slug)) + '/',
@@ -232,9 +253,9 @@ class BuildMobi(BuildEbook):
             ).build(wldoc)
 
 
-@BuildEbook.register('html')
-@task(ignore_result=True)
-class BuildHtml(BuildEbook):
+class HtmlField(EbookField):
+    ext = 'html'
+
     def build(self, fieldfile):
         from django.core.files.base import ContentFile
         from slugify import slugify
@@ -244,7 +265,7 @@ class BuildHtml(BuildEbook):
 
         book = fieldfile.instance
 
-        html_output = self.transform(book.wldocument(parse_dublincore=False), fieldfile)
+        html_output = self.transform(book.wldocument(parse_dublincore=False))
 
         # Delete old fragments, create from scratch if necessary.
         book.fragments.all().delete()
@@ -324,7 +345,7 @@ class BuildHtml(BuildEbook):
         return False
 
     @staticmethod
-    def transform(wldoc, fieldfile):
+    def transform(wldoc):
         # ugly, but we can't use wldoc.book_info here
         from librarian import DCNS
         url_elem = wldoc.edoc.getroot().find('.//' + DCNS('identifier.url'))
@@ -338,16 +359,19 @@ class BuildHtml(BuildEbook):
         return wldoc.as_html(gallery_path=gal_path, gallery_url=gal_url, base_url=absolute_url(gal_url))
 
 
-class BuildCover(BuildEbook):
+class CoverField(EbookField):
+    ext = 'jpg'
+    directory = 'cover'
+
     def set_file_permissions(self, fieldfile):
         pass
 
 
-@BuildEbook.register('cover_clean')
-@task(ignore_result=True)
-class BuildCoverClean(BuildCover):
-    @classmethod
-    def transform(cls, wldoc, fieldfile):
+class CoverCleanField(CoverField):
+    directory = 'cover_clean'
+
+    @staticmethod
+    def transform(wldoc):
         if wldoc.book_info.cover_box_position == 'none':
             from librarian.cover import WLCover
             return WLCover(wldoc.book_info, width=240).output_file()
@@ -355,62 +379,37 @@ class BuildCoverClean(BuildCover):
         return MarquiseCover(wldoc.book_info, width=240).output_file()
 
 
-@BuildEbook.register('cover_thumb')
-@task(ignore_result=True)
-class BuildCoverThumb(BuildCover):
-    @classmethod
-    def transform(cls, wldoc, fieldfile):
+class CoverThumbField(CoverField):
+    directory = 'cover_thumb'
+
+    @staticmethod
+    def transform(wldoc):
         from librarian.cover import WLCover
         return WLCover(wldoc.book_info, height=193).output_file()
 
 
-@BuildEbook.register('cover_api_thumb')
-@task(ignore_result=True)
-class BuildCoverApiThumb(BuildCover):
-    @classmethod
-    def transform(cls, wldoc, fieldfile):
+class CoverApiThumbField(CoverField):
+    directory = 'cover_api_thumb'
+
+    @staticmethod
+    def transform(wldoc):
         from librarian.cover import WLNoBoxCover
         return WLNoBoxCover(wldoc.book_info, height=500).output_file()
 
 
-@BuildEbook.register('simple_cover')
-@task(ignore_result=True)
-class BuildSimpleCover(BuildCover):
-    @classmethod
-    def transform(cls, wldoc, fieldfile):
+class SimpleCoverField(CoverField):
+    directory = 'cover_simple'
+
+    @staticmethod
+    def transform(wldoc):
         from librarian.cover import WLNoBoxCover
         return WLNoBoxCover(wldoc.book_info, height=1000).output_file()
 
 
-@BuildEbook.register('cover_ebookpoint')
-@task(ignore_result=True)
-class BuildCoverEbookpoint(BuildCover):
-    @classmethod
-    def transform(cls, wldoc, fieldfile):
+class CoverEbookpointField(CoverField):
+    directory = 'cover_ebookpoint'
+
+    @staticmethod
+    def transform(wldoc):
         from librarian.cover import EbookpointCover
         return EbookpointCover(wldoc.book_info).output_file()
-
-
-# not used, but needed for migrations
-class OverwritingFieldFile(FieldFile):
-    """
-        Deletes the old file before saving the new one.
-    """
-
-    def save(self, name, content, *args, **kwargs):
-        leave = kwargs.pop('leave', None)
-        # delete if there's a file already and there's a new one coming
-        if not leave and self and (not hasattr(content, 'path') or content.path != self.path):
-            self.delete(save=False)
-        return super(OverwritingFieldFile, self).save(name, content, *args, **kwargs)
-
-
-class OverwritingFileField(models.FileField):
-    attr_class = OverwritingFieldFile
-
-
-class OverwriteStorage(FileSystemStorage):
-
-    def get_available_name(self, name, max_length=None):
-        self.delete(name)
-        return name
diff --git a/src/catalogue/management/commands/schedule_stale_ebooks.py b/src/catalogue/management/commands/schedule_stale_ebooks.py
index 91836fbf3..4420a815e 100644
--- a/src/catalogue/management/commands/schedule_stale_ebooks.py
+++ b/src/catalogue/management/commands/schedule_stale_ebooks.py
@@ -5,10 +5,11 @@ from django.conf import settings
 from django.core.management.base import BaseCommand
 
 from catalogue.fields import EbookField
+from catalogue.models import Book
 
 
 class Command(BaseCommand):
     help = 'Schedule regenerating stale ebook files.'
 
     def handle(self, **options):
-        EbookField.schedule_all_stale()
+        EbookField.schedule_all_stale(Book)
diff --git a/src/catalogue/migrations/0001_initial.py b/src/catalogue/migrations/0001_initial.py
index ac8f9d738..e62b23358 100644
--- a/src/catalogue/migrations/0001_initial.py
+++ b/src/catalogue/migrations/0001_initial.py
@@ -35,16 +35,16 @@ class Migration(migrations.Migration):
                 ('extra_info', models.TextField(default='{}', verbose_name='Additional information')),
                 ('gazeta_link', models.CharField(max_length=240, blank=True)),
                 ('wiki_link', models.CharField(max_length=240, blank=True)),
-                ('cover', catalogue.fields.EbookField('cover', upload_to=catalogue.models.book._cover_upload_to, storage=fnpdjango.storage.BofhFileSystemStorage(), max_length=255, blank=True, null=True, verbose_name='cover')),
-                ('cover_thumb', catalogue.fields.EbookField('cover_thumb', max_length=255, upload_to=catalogue.models.book._cover_thumb_upload_to, null=True, verbose_name='cover thumbnail', blank=True)),
+                ('cover', catalogue.fields.EbookField('cover', storage=fnpdjango.storage.BofhFileSystemStorage(), max_length=255, blank=True, null=True, verbose_name='cover')),
+                ('cover_thumb', catalogue.fields.EbookField('cover_thumb', max_length=255, null=True, verbose_name='cover thumbnail', blank=True)),
                 ('_related_info', models.TextField(null=True, editable=False, blank=True)),
-                ('txt_file', catalogue.fields.EbookField('txt', default='', storage=fnpdjango.storage.BofhFileSystemStorage(), upload_to=catalogue.models.book._txt_upload_to, max_length=255, blank=True, verbose_name='TXT file')),
-                ('fb2_file', catalogue.fields.EbookField('fb2', default='', storage=fnpdjango.storage.BofhFileSystemStorage(), upload_to=catalogue.models.book._fb2_upload_to, max_length=255, blank=True, verbose_name='FB2 file')),
-                ('pdf_file', catalogue.fields.EbookField('pdf', default='', storage=fnpdjango.storage.BofhFileSystemStorage(), upload_to=catalogue.models.book._pdf_upload_to, max_length=255, blank=True, verbose_name='PDF file')),
-                ('epub_file', catalogue.fields.EbookField('epub', default='', storage=fnpdjango.storage.BofhFileSystemStorage(), upload_to=catalogue.models.book._epub_upload_to, max_length=255, blank=True, verbose_name='EPUB file')),
-                ('mobi_file', catalogue.fields.EbookField('mobi', default='', storage=fnpdjango.storage.BofhFileSystemStorage(), upload_to=catalogue.models.book._mobi_upload_to, max_length=255, blank=True, verbose_name='MOBI file')),
-                ('html_file', catalogue.fields.EbookField('html', default='', storage=fnpdjango.storage.BofhFileSystemStorage(), upload_to=catalogue.models.book._html_upload_to, max_length=255, blank=True, verbose_name='HTML file')),
-                ('xml_file', catalogue.fields.EbookField('xml', default='', storage=fnpdjango.storage.BofhFileSystemStorage(), upload_to=catalogue.models.book._xml_upload_to, max_length=255, blank=True, verbose_name='XML file')),
+                ('txt_file', catalogue.fields.EbookField('txt', default='', storage=fnpdjango.storage.BofhFileSystemStorage(), max_length=255, blank=True, verbose_name='TXT file')),
+                ('fb2_file', catalogue.fields.EbookField('fb2', default='', storage=fnpdjango.storage.BofhFileSystemStorage(), max_length=255, blank=True, verbose_name='FB2 file')),
+                ('pdf_file', catalogue.fields.EbookField('pdf', default='', storage=fnpdjango.storage.BofhFileSystemStorage(), max_length=255, blank=True, verbose_name='PDF file')),
+                ('epub_file', catalogue.fields.EbookField('epub', default='', storage=fnpdjango.storage.BofhFileSystemStorage(), max_length=255, blank=True, verbose_name='EPUB file')),
+                ('mobi_file', catalogue.fields.EbookField('mobi', default='', storage=fnpdjango.storage.BofhFileSystemStorage(), max_length=255, blank=True, verbose_name='MOBI file')),
+                ('html_file', catalogue.fields.EbookField('html', default='', storage=fnpdjango.storage.BofhFileSystemStorage(), max_length=255, blank=True, verbose_name='HTML file')),
+                ('xml_file', catalogue.fields.EbookField('xml', default='', storage=fnpdjango.storage.BofhFileSystemStorage(), max_length=255, blank=True, verbose_name='XML file')),
                 ('parent', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='children', blank=True, to='catalogue.Book', null=True)),
             ],
             options={
@@ -60,7 +60,7 @@ class Migration(migrations.Migration):
                 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                 ('type', models.CharField(db_index=True, max_length=20, verbose_name='type', choices=[('mp3', 'MP3 file'), ('ogg', 'Ogg Vorbis file'), ('daisy', 'DAISY file')])),
                 ('name', models.CharField(max_length=512, verbose_name='name')),
-                ('file', catalogue.fields.OverwritingFileField(upload_to=catalogue.models.bookmedia._file_upload_to, max_length=600, verbose_name='XML file')),
+                ('file', models.FileField(upload_to=catalogue.models.bookmedia._file_upload_to, max_length=600, verbose_name='XML file')),
                 ('uploaded_at', models.DateTimeField(auto_now_add=True, verbose_name='creation date', db_index=True)),
                 ('extra_info', models.TextField(default='{}', verbose_name='Additional information', editable=False)),
                 ('source_sha1', models.CharField(max_length=40, null=True, editable=False, blank=True)),
diff --git a/src/catalogue/migrations/0008_auto_20151221_1225.py b/src/catalogue/migrations/0008_auto_20151221_1225.py
index 04e79c38e..cdaaa0eb4 100644
--- a/src/catalogue/migrations/0008_auto_20151221_1225.py
+++ b/src/catalogue/migrations/0008_auto_20151221_1225.py
@@ -69,7 +69,7 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='bookmedia',
             name='file',
-            field=catalogue.fields.OverwritingFileField(upload_to=catalogue.models.bookmedia._file_upload_to, max_length=600, verbose_name='file'),
+            field=models.FileField(upload_to=catalogue.models.bookmedia._file_upload_to, max_length=600, verbose_name='file'),
         ),
         migrations.AlterField(
             model_name='collection',
diff --git a/src/catalogue/migrations/0014_auto_20170627_1442.py b/src/catalogue/migrations/0014_auto_20170627_1442.py
index 0affd0a19..cdfac230d 100644
--- a/src/catalogue/migrations/0014_auto_20170627_1442.py
+++ b/src/catalogue/migrations/0014_auto_20170627_1442.py
@@ -2,7 +2,7 @@
 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
 #
 from django.db import migrations, models
-import catalogue.fields
+import fnpdjango.storage
 import catalogue.models.bookmedia
 
 
@@ -16,6 +16,6 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='bookmedia',
             name='file',
-            field=models.FileField(storage=catalogue.fields.OverwriteStorage(), upload_to=catalogue.models.bookmedia._file_upload_to, max_length=600, verbose_name='file'),
+            field=models.FileField(storage=fnpdjango.storage.BofhFileSystemStorage(), upload_to=catalogue.models.bookmedia._file_upload_to, max_length=600, verbose_name='file'),
         ),
     ]
diff --git a/src/catalogue/migrations/0020_book_cover_api_thumb.py b/src/catalogue/migrations/0020_book_cover_api_thumb.py
index 2d73807c7..0dba17502 100644
--- a/src/catalogue/migrations/0020_book_cover_api_thumb.py
+++ b/src/catalogue/migrations/0020_book_cover_api_thumb.py
@@ -3,7 +3,6 @@
 #
 from django.db import migrations, models
 import catalogue.fields
-import catalogue.models.book
 
 
 class Migration(migrations.Migration):
@@ -16,6 +15,6 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='book',
             name='cover_api_thumb',
-            field=catalogue.fields.EbookField('cover_api_thumb', max_length=255, upload_to=catalogue.models.book.UploadToPath('book/cover_api_thumb/%s.jpg'), null=True, verbose_name='cover thumbnail for API', blank=True),
+            field=catalogue.fields.EbookField('cover_api_thumb', max_length=255, upload_to=catalogue.fields.UploadToPath('book/cover_api_thumb/%s.jpg'), null=True, verbose_name='cover thumbnail for API', blank=True),
         ),
     ]
diff --git a/src/catalogue/migrations/0021_auto_20171222_1404.py b/src/catalogue/migrations/0021_auto_20171222_1404.py
index 8f4e4f23a..e4a8762e2 100644
--- a/src/catalogue/migrations/0021_auto_20171222_1404.py
+++ b/src/catalogue/migrations/0021_auto_20171222_1404.py
@@ -3,7 +3,6 @@
 #
 from django.db import migrations, models
 import catalogue.fields
-import catalogue.models.book
 
 
 class Migration(migrations.Migration):
@@ -16,11 +15,11 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='book',
             name='simple_cover',
-            field=catalogue.fields.EbookField('simple_cover', max_length=255, upload_to=catalogue.models.book.UploadToPath('book/cover_simple/%s.jpg'), null=True, verbose_name='cover for mobile app', blank=True),
+            field=catalogue.fields.EbookField('simple_cover', max_length=255, upload_to=catalogue.fields.UploadToPath('book/cover_simple/%s.jpg'), null=True, verbose_name='cover for mobile app', blank=True),
         ),
         migrations.AlterField(
             model_name='book',
             name='cover_api_thumb',
-            field=catalogue.fields.EbookField('cover_api_thumb', max_length=255, upload_to=catalogue.models.book.UploadToPath('book/cover_api_thumb/%s.jpg'), null=True, verbose_name='cover thumbnail for mobile app', blank=True),
+            field=catalogue.fields.EbookField('cover_api_thumb', max_length=255, upload_to=catalogue.fields.UploadToPath('book/cover_api_thumb/%s.jpg'), null=True, verbose_name='cover thumbnail for mobile app', blank=True),
         ),
     ]
diff --git a/src/catalogue/migrations/0028_book_cover_ebookpoint.py b/src/catalogue/migrations/0028_book_cover_ebookpoint.py
index 6c6e74cc7..b9ba1b064 100644
--- a/src/catalogue/migrations/0028_book_cover_ebookpoint.py
+++ b/src/catalogue/migrations/0028_book_cover_ebookpoint.py
@@ -1,7 +1,6 @@
 # Generated by Django 2.2.10 on 2020-04-03 09:40
 
 import catalogue.fields
-import catalogue.models.book
 from django.db import migrations
 
 
@@ -15,6 +14,6 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='book',
             name='cover_ebookpoint',
-            field=catalogue.fields.EbookField('cover_ebookpoint', blank=True, max_length=255, null=True, upload_to=catalogue.models.book.UploadToPath('book/cover_ebookpoint/%s.jpg'), verbose_name='cover for Ebookpoint'),
+            field=catalogue.fields.EbookField('cover_ebookpoint', blank=True, max_length=255, null=True, upload_to=catalogue.fields.UploadToPath('book/cover_ebookpoint/%s.jpg'), verbose_name='cover for Ebookpoint'),
         ),
     ]
diff --git a/src/catalogue/migrations/0034_auto_20220310_1251.py b/src/catalogue/migrations/0034_auto_20220310_1251.py
index 758106ab7..9360229a1 100644
--- a/src/catalogue/migrations/0034_auto_20220310_1251.py
+++ b/src/catalogue/migrations/0034_auto_20220310_1251.py
@@ -1,7 +1,6 @@
 # Generated by Django 2.2.25 on 2022-03-10 11:51
 
 import catalogue.fields
-import catalogue.models.book
 from django.db import migrations, models
 
 
@@ -15,7 +14,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='book',
             name='cover_clean',
-            field=catalogue.fields.EbookField('cover_clean', blank=True, max_length=255, null=True, upload_to=catalogue.models.book.UploadToPath('book/cover_clean/%s.jpg'), verbose_name='clean cover'),
+            field=catalogue.fields.EbookField('cover_clean', blank=True, max_length=255, null=True, upload_to=catalogue.fields.UploadToPath('book/cover_clean/%s.jpg'), verbose_name='clean cover'),
         ),
         migrations.AddField(
             model_name='book',
diff --git a/src/catalogue/migrations/0038_auto_20220930_1050.py b/src/catalogue/migrations/0038_auto_20220930_1050.py
new file mode 100644
index 000000000..4963e6690
--- /dev/null
+++ b/src/catalogue/migrations/0038_auto_20220930_1050.py
@@ -0,0 +1,80 @@
+# Generated by Django 2.2.28 on 2022-09-30 08:50
+
+import catalogue.fields
+from django.db import migrations
+import fnpdjango.storage
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('catalogue', '0037_auto_20220928_0124'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='book',
+            name='cover',
+            field=catalogue.fields.CoverField('cover', storage=fnpdjango.storage.BofhFileSystemStorage()),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='cover_api_thumb',
+            field=catalogue.fields.CoverApiThumbField('cover thumbnail for mobile app'),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='cover_clean',
+            field=catalogue.fields.CoverCleanField('clean cover'),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='cover_ebookpoint',
+            field=catalogue.fields.CoverEbookpointField('cover for Ebookpoint'),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='cover_thumb',
+            field=catalogue.fields.CoverThumbField('cover thumbnail'),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='epub_file',
+            field=catalogue.fields.EpubField(storage=fnpdjango.storage.BofhFileSystemStorage()),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='fb2_file',
+            field=catalogue.fields.Fb2Field(storage=fnpdjango.storage.BofhFileSystemStorage()),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='html_file',
+            field=catalogue.fields.HtmlField(storage=fnpdjango.storage.BofhFileSystemStorage()),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='mobi_file',
+            field=catalogue.fields.MobiField(storage=fnpdjango.storage.BofhFileSystemStorage()),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='pdf_file',
+            field=catalogue.fields.PdfField(storage=fnpdjango.storage.BofhFileSystemStorage()),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='simple_cover',
+            field=catalogue.fields.SimpleCoverField('cover for mobile app'),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='txt_file',
+            field=catalogue.fields.TxtField(storage=fnpdjango.storage.BofhFileSystemStorage()),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='xml_file',
+            field=catalogue.fields.XmlField(storage=fnpdjango.storage.BofhFileSystemStorage(), with_etag=False),
+        ),
+    ]
diff --git a/src/catalogue/models/book.py b/src/catalogue/models/book.py
index 7d923ea3c..fc07fc5f7 100644
--- a/src/catalogue/models/book.py
+++ b/src/catalogue/models/book.py
@@ -16,45 +16,22 @@ from django.contrib.contenttypes.fields import GenericRelation
 from django.template.loader import render_to_string
 from django.urls import reverse
 from django.utils.translation import ugettext_lazy as _, get_language
-from django.utils.deconstruct import deconstructible
 from fnpdjango.storage import BofhFileSystemStorage
 from lxml import html
 from librarian.cover import WLCover
 from librarian.html import transform_abstrakt
 from newtagging import managers
 from catalogue import constants
-from catalogue.fields import EbookField
+from catalogue import fields
 from catalogue.models import Tag, Fragment, BookMedia
 from catalogue.utils import create_zip, gallery_url, gallery_path, split_tags, get_random_hash
 from catalogue.models.tag import prefetched_relations
 from catalogue import app_settings
-from catalogue import tasks
 from wolnelektury.utils import makedirs, cached_render, clear_cached_renders
 
 bofh_storage = BofhFileSystemStorage()
 
 
-@deconstructible
-class UploadToPath(object):
-    def __init__(self, path):
-        self.path = path
-
-    def __call__(self, instance, filename):
-        return self.path % instance.slug
-
-
-_cover_upload_to = UploadToPath('book/cover/%s.jpg')
-_cover_clean_upload_to = UploadToPath('book/cover_clean/%s.jpg')
-_cover_thumb_upload_to = UploadToPath('book/cover_thumb/%s.jpg')
-_cover_api_thumb_upload_to = UploadToPath('book/cover_api_thumb/%s.jpg')
-_simple_cover_upload_to = UploadToPath('book/cover_simple/%s.jpg')
-_cover_ebookpoint_upload_to = UploadToPath('book/cover_ebookpoint/%s.jpg')
-
-
-def _ebook_upload_to(upload_path):
-    return UploadToPath(upload_path)
-
-
 class Book(models.Model):
     """Represents a book imported from WL-XML."""
     title = models.CharField(_('title'), max_length=32767)
@@ -82,44 +59,24 @@ class Book(models.Model):
     findable = models.BooleanField(_('findable'), default=True, db_index=True)
 
     # files generated during publication
-    cover = EbookField(
-        'cover', _('cover'),
-        null=True, blank=True,
-        upload_to=_cover_upload_to,
-        storage=bofh_storage, max_length=255)
-    cover_etag = models.CharField(max_length=255, editable=False, default='', db_index=True)
+    xml_file = fields.XmlField(storage=bofh_storage, with_etag=False)
+    html_file = fields.HtmlField(storage=bofh_storage)
+    fb2_file = fields.Fb2Field(storage=bofh_storage)
+    txt_file = fields.TxtField(storage=bofh_storage)
+    epub_file = fields.EpubField(storage=bofh_storage)
+    mobi_file = fields.MobiField(storage=bofh_storage)
+    pdf_file = fields.PdfField(storage=bofh_storage)
+
+    cover = fields.CoverField(_('cover'), storage=bofh_storage)
     # Cleaner version of cover for thumbs
-    cover_clean = EbookField(
-        'cover_clean', _('clean cover'),
-        null=True, blank=True,
-        upload_to=_cover_clean_upload_to,
-        max_length=255
-    )
-    cover_clean_etag = models.CharField(max_length=255, editable=False, default='', db_index=True)
-    cover_thumb = EbookField(
-        'cover_thumb', _('cover thumbnail'),
-        null=True, blank=True,
-        upload_to=_cover_thumb_upload_to,
-        max_length=255)
-    cover_thumb_etag = models.CharField(max_length=255, editable=False, default='', db_index=True)
-    cover_api_thumb = EbookField(
-        'cover_api_thumb', _('cover thumbnail for mobile app'),
-        null=True, blank=True,
-        upload_to=_cover_api_thumb_upload_to,
-        max_length=255)
-    cover_api_thumb_etag = models.CharField(max_length=255, editable=False, default='', db_index=True)
-    simple_cover = EbookField(
-        'simple_cover', _('cover for mobile app'),
-        null=True, blank=True,
-        upload_to=_simple_cover_upload_to,
-        max_length=255)
-    simple_cover_etag = models.CharField(max_length=255, editable=False, default='', db_index=True)
-    cover_ebookpoint = EbookField(
-        'cover_ebookpoint', _('cover for Ebookpoint'),
-        null=True, blank=True,
-        upload_to=_cover_ebookpoint_upload_to,
-        max_length=255)
-    cover_ebookpoint_etag = models.CharField(max_length=255, editable=False, default='', db_index=True)
+    cover_clean = fields.CoverCleanField(_('clean cover'))
+    cover_thumb = fields.CoverThumbField(_('cover thumbnail'))
+    cover_api_thumb = fields.CoverApiThumbField(
+        _('cover thumbnail for mobile app'))
+    simple_cover = fields.SimpleCoverField(_('cover for mobile app'))
+    cover_ebookpoint = fields.CoverEbookpointField(
+        _('cover for Ebookpoint'))
+
     ebook_formats = constants.EBOOK_FORMATS
     formats = ebook_formats + ['html', 'xml']
 
@@ -295,7 +252,7 @@ class Book(models.Model):
 
         if self.parent.html_file:
             return self.parent
-        
+
         return self.parent.get_prev_text()
 
     def get_next_text(self, inside=True):
@@ -329,7 +286,7 @@ class Book(models.Model):
 
     def get_children(self):
         return self.children.all().order_by('parent_number')
-    
+
     @property
     def name(self):
         return self.title
@@ -463,7 +420,7 @@ class Book(models.Model):
     @property
     def media_daisy(self):
         return self.get_media('daisy')
-    
+
     @property
     def media_audio_epub(self):
         return self.get_media('audio.epub')
@@ -536,9 +493,10 @@ class Book(models.Model):
                 format_)
 
         field_name = "%s_file" % format_
+        field = getattr(Book, field_name)
         books = Book.objects.filter(parent=None).exclude(**{field_name: ""}).exclude(preview=True).exclude(findable=False)
         paths = [(pretty_file_name(b), getattr(b, field_name).path) for b in books.iterator()]
-        return create_zip(paths, app_settings.FORMAT_ZIPS[format_])
+        return create_zip(paths, field.ZIP)
 
     def zip_audiobooks(self, format_):
         bm = BookMedia.objects.filter(book=self, type=format_)
@@ -610,7 +568,7 @@ class Book(models.Model):
                 a.attrib['href'] = html_link + a.attrib['href']
             self.toc = html.tostring(toc, encoding='unicode')
             # div#toc
-            
+
     @classmethod
     def from_xml_file(cls, xml_file, **kwargs):
         from django.core.files import File
@@ -630,6 +588,8 @@ class Book(models.Model):
     @classmethod
     def from_text_and_meta(cls, raw_file, book_info, overwrite=False, dont_build=None, search_index=True,
                            search_index_tags=True, remote_gallery_url=None, days=0, findable=True):
+        from catalogue import tasks
+
         if dont_build is None:
             dont_build = set()
         dont_build = set.union(set(dont_build), set(app_settings.DONT_BUILD))
@@ -762,7 +722,7 @@ class Book(models.Model):
         for master in root.iter():
             if master.tag in master_tags:
                 return master
-    
+
     def update_references(self):
         from references.models import Entity, Reference
         master = self.get_master()
@@ -786,7 +746,7 @@ class Book(models.Model):
                 entity.populate()
                 entity.save()
         Reference.objects.filter(book=self).exclude(entity__uri__in=found).delete()
-    
+
     @property
     def references(self):
         return self.reference_set.all().select_related('entity')
@@ -832,7 +792,7 @@ class Book(models.Model):
             yield self.parent
         else:
             return []
-                    
+
     def clear_cache(self):
         clear_cached_renders(self.mini_box)
         clear_cached_renders(self.mini_box_nolink)
@@ -1003,7 +963,7 @@ class Book(models.Model):
             return fragments[0]
         else:
             return None
-        
+
     def fragment_data(self):
         fragment = self.choose_fragment()
         if fragment:
@@ -1056,28 +1016,6 @@ class Book(models.Model):
             'no_link': True,
         }
 
-def add_file_fields():
-    for format_ in Book.formats:
-        field_name = "%s_file" % format_
-        # This weird globals() assignment makes Django migrations comfortable.
-        _upload_to = _ebook_upload_to('book/%s/%%s.%s' % (format_, format_))
-        _upload_to.__name__ = '_%s_upload_to' % format_
-        globals()[_upload_to.__name__] = _upload_to
-
-        EbookField(
-            format_, _("%s file" % format_.upper()),
-            upload_to=_upload_to,
-            storage=bofh_storage,
-            max_length=255,
-            blank=True,
-            default=''
-        ).contribute_to_class(Book, field_name)
-        if format_ != 'xml':
-            models.CharField(max_length=255, editable=False, default='', db_index=True).contribute_to_class(Book, f'{field_name}_etag')
-
-
-add_file_fields()
-
 
 class BookPopularity(models.Model):
     book = models.OneToOneField(Book, models.CASCADE, related_name='popularity')
diff --git a/src/catalogue/models/bookmedia.py b/src/catalogue/models/bookmedia.py
index 702dd01f8..541940561 100644
--- a/src/catalogue/models/bookmedia.py
+++ b/src/catalogue/models/bookmedia.py
@@ -9,8 +9,7 @@ from django.utils.translation import ugettext_lazy as _
 from slugify import slugify
 import mutagen
 from mutagen import id3
-
-from catalogue.fields import OverwriteStorage
+from fnpdjango.storage import BofhFileSystemStorage
 
 
 def _file_upload_to(i, _n):
@@ -38,7 +37,7 @@ class BookMedia(models.Model):
     name = models.CharField(_('name'), max_length=512)
     part_name = models.CharField(_('part name'), default='', blank=True, max_length=512)
     index = models.IntegerField(_('index'), default=0)
-    file = models.FileField(_('file'), max_length=600, upload_to=_file_upload_to, storage=OverwriteStorage())
+    file = models.FileField(_('file'), max_length=600, upload_to=_file_upload_to, storage=BofhFileSystemStorage())
     duration = models.IntegerField(null=True, blank=True)
     uploaded_at = models.DateTimeField(_('creation date'), auto_now_add=True, editable=False, db_index=True)
     project_description = models.CharField(max_length=2048, blank=True)
diff --git a/src/catalogue/tasks.py b/src/catalogue/tasks.py
index b703e171f..e16f1ff5a 100644
--- a/src/catalogue/tasks.py
+++ b/src/catalogue/tasks.py
@@ -7,6 +7,7 @@ from celery.utils.log import get_task_logger
 from django.conf import settings
 from django.utils import timezone
 
+from catalogue.models import Book
 from catalogue.utils import absolute_url, gallery_url
 from waiter.models import WaitedFile
 
@@ -22,9 +23,16 @@ def touch_tag(tag):
     type(tag).objects.filter(pk=tag.pk).update(**update_dict)
 
 
+@shared_task(ignore_result=True)
+def build_field(pk, field_name):
+    book = Book.objects.get(pk=pk)
+    task_logger.info("build %s.%s" % (book.slug, field_name))
+    field_file = getattr(book, field_name)
+    field_file.build()
+
+
 @shared_task
 def index_book(book_id, book_info=None, **kwargs):
-    from catalogue.models import Book
     try:
         return Book.objects.get(id=book_id).search_index(book_info, **kwargs)
     except Exception as e:
@@ -39,7 +47,6 @@ def build_custom_pdf(book_id, customizations, file_name, waiter_id=None):
     try:
         from django.core.files import File
         from django.core.files.storage import DefaultStorage
-        from catalogue.models import Book
 
         task_logger.info(DefaultStorage().path(file_name))
         if not DefaultStorage().exists(file_name):
@@ -69,6 +76,5 @@ def update_counters():
 
 @shared_task(ignore_result=True)
 def update_references(book_id):
-    from catalogue.models import Book
     Book.objects.get(id=book_id).update_references()
 
diff --git a/src/catalogue/urls.py b/src/catalogue/urls.py
index 7f1eddbae..3cb17502b 100644
--- a/src/catalogue/urls.py
+++ b/src/catalogue/urls.py
@@ -55,11 +55,11 @@ urlpatterns = [
     path('pobierz/<key>/<slug:slug>.<slug:format_>', views.embargo_link, name='embargo_link'),
 
     # zip
-    path('zip/pdf.zip', views.download_zip, {'format': 'pdf', 'slug': None}, 'download_zip_pdf'),
-    path('zip/epub.zip', views.download_zip, {'format': 'epub', 'slug': None}, 'download_zip_epub'),
-    path('zip/mobi.zip', views.download_zip, {'format': 'mobi', 'slug': None}, 'download_zip_mobi'),
-    path('zip/mp3/<slug:slug>.zip', views.download_zip, {'format': 'mp3'}, 'download_zip_mp3'),
-    path('zip/ogg/<slug:slug>.zip', views.download_zip, {'format': 'ogg'}, 'download_zip_ogg'),
+    path('zip/pdf.zip', views.download_zip, {'file_format': 'pdf', 'slug': None}, 'download_zip_pdf'),
+    path('zip/epub.zip', views.download_zip, {'file_format': 'epub', 'slug': None}, 'download_zip_epub'),
+    path('zip/mobi.zip', views.download_zip, {'file_format': 'mobi', 'slug': None}, 'download_zip_mobi'),
+    path('zip/mp3/<slug:slug>.zip', views.download_zip, {'media_format': 'mp3'}, 'download_zip_mp3'),
+    path('zip/ogg/<slug:slug>.zip', views.download_zip, {'media_format': 'ogg'}, 'download_zip_ogg'),
 
     # Public interface. Do not change this URLs.
     path('lektura/<slug:slug>.html', views.book_text, name='book_text'),
diff --git a/src/catalogue/views.py b/src/catalogue/views.py
index c63aa7a9e..816836d64 100644
--- a/src/catalogue/views.py
+++ b/src/catalogue/views.py
@@ -408,12 +408,12 @@ def embargo_link(request, key, format_, slug):
     return HttpResponse(media_file, content_type=constants.EBOOK_CONTENT_TYPES[format_])
 
 
-def download_zip(request, format, slug=None):
-    if format in Book.ebook_formats:
-        url = Book.zip_format(format)
-    elif format in ('mp3', 'ogg') and slug is not None:
+def download_zip(request, file_format=None, media_format=None, slug=None):
+    if file_format:
+        url = Book.zip_format(file_format)
+    elif media_format and slug is not None:
         book = get_object_or_404(Book, slug=slug)
-        url = book.zip_audiobooks(format)
+        url = book.zip_audiobooks(media_format)
     else:
         raise Http404('No format specified for zip package')
     return HttpResponseRedirect(urlquote_plus(settings.MEDIA_URL + url, safe='/?='))