DEFAULT_LANGUAGE = 'pol'
# PDF needs TeXML + XeLaTeX, MOBI needs Calibre.
DONT_BUILD = {'pdf', 'mobi'}
- FORMAT_ZIPS = {
- 'epub': 'wolnelektury_pl_epub',
- 'pdf': 'wolnelektury_pl_pdf',
- 'mobi': 'wolnelektury_pl_mobi',
- 'fb2': 'wolnelektury_pl_fb2',
- }
REDAKCJA_URL = "http://redakcja.wolnelektury.pl"
GOOD_LICENSES = {r'CC BY \d\.\d', r'CC BY-SA \d\.\d'}
RELATED_RANDOM_PICTURE_CHANCE = .5
GET_MP3_LENGTH = 'catalogue.utils.get_mp3_length'
- def _more_DONT_BUILD(self, value):
- for format_ in ['cover', 'pdf', 'epub', 'mobi', 'fb2', 'txt']:
- attname = 'NO_BUILD_%s' % format_.upper()
- if hasattr(settings, attname):
- logging.warn("%s is deprecated, use CATALOGUE_DONT_BUILD instead", attname)
- if getattr(settings, attname):
- value.add(format_)
- else:
- value.remove(format_)
- return value
-
- def _more_FORMAT_ZIPS(self, value):
- for format_ in ['epub', 'pdf', 'mobi', 'fb2']:
- attname = 'ALL_%s_ZIP' % format_.upper()
- if hasattr(settings, attname):
- logging.warn(
- "%s is deprecated, use CATALOGUE_FORMAT_ZIPS[%s] instead",
- attname, format_
- )
- value[format_] = getattr(settings, attname)
- return value
-
def _more_GET_MP3_LENGTH(self, value):
return import_string(value)
import os
from django.conf import settings
from django.core.files import File
-from django.core.files.storage import FileSystemStorage
from django.db import models
from django.db.models.fields.files import FieldFile
-from catalogue import app_settings
+from django.utils.deconstruct import deconstructible
+from django.utils.translation import gettext_lazy as _
from catalogue.constants import LANGUAGES_3TO2, EBOOK_FORMATS_WITH_CHILDREN, EBOOK_FORMATS_WITHOUT_CHILDREN
from catalogue.utils import absolute_url, remove_zip, truncate_html_words, gallery_path, gallery_url
-#from celery import Task, shared_task
-from celery.task import Task, task
-from celery.utils.log import get_task_logger
from waiter.utils import clear_cache
-task_logger = get_task_logger(__name__)
-
ETAG_SCHEDULED_SUFFIX = '-scheduled'
EBOOK_BUILD_PRIORITY = 0
EBOOK_REBUILD_PRIORITY = 9
+@deconstructible
+class UploadToPath(object):
+ def __init__(self, path):
+ self.path = path
+
+ def __call__(self, instance, filename):
+ return self.path % instance.slug
+
+ def __eq__(self, other):
+ return isinstance(other, type(self)) and other.path == self.path
+
+
class EbookFieldFile(FieldFile):
"""Represents contents of an ebook file field."""
def build(self):
"""Build the ebook immediately."""
- return self.field.builder.build(self)
+ etag = self.field.get_current_etag()
+ self.field.build(self)
+ self.update_etag(etag)
+ self.instance.clear_cache()
def build_delay(self, priority=EBOOK_BUILD_PRIORITY):
"""Builds the ebook in a delayed task."""
+ from .tasks import build_field
+
self.update_etag(
"".join([self.field.get_current_etag(), ETAG_SCHEDULED_SUFFIX])
)
- return self.field.builder.apply_async(
- [self.instance, self.field.attname],
+ return build_field.apply_async(
+ [self.instance.pk, self.field.attname],
priority=priority
)
- def get_url(self):
- return self.instance.media_url(self.field.attname.split('_')[0])
-
def set_readable(self, readable):
import os
permissions = 0o644 if readable else 0o600
class EbookField(models.FileField):
"""Represents an ebook file field, attachable to a model."""
attr_class = EbookFieldFile
- registry = []
+ ext = None
+ librarian2_api = False
+ ZIP = None
- def __init__(self, format_name, *args, **kwargs):
- super(EbookField, self).__init__(*args, **kwargs)
- self.format_name = format_name
+ def __init__(self, verbose_name_=None, with_etag=True, **kwargs):
+ # This is just for compatibility with older migrations,
+ # where first argument was for ebook format.
+ # Can be scrapped if old migrations are updated/removed.
+ verbose_name = verbose_name_ or _("%s file") % self.ext
+ kwargs.setdefault('verbose_name', verbose_name_ )
+
+ self.with_etag = with_etag
+ kwargs.setdefault('max_length', 255)
+ kwargs.setdefault('blank', True)
+ kwargs.setdefault('default', '')
+ kwargs.setdefault('upload_to', self.get_upload_to(self.ext))
+
+ super().__init__(**kwargs)
def deconstruct(self):
- name, path, args, kwargs = super(EbookField, self).deconstruct()
- args.insert(0, self.format_name)
+ name, path, args, kwargs = super().deconstruct()
+ if kwargs.get('max_length') == 255:
+ del kwargs['max_length']
+ if kwargs.get('blank') is True:
+ del kwargs['blank']
+ if kwargs.get('default') == '':
+ del kwargs['default']
+ if self.get_upload_to(self.ext) == kwargs.get('upload_to'):
+ del kwargs['upload_to']
+ if not self.with_etag:
+ kwargs['with_etag'] = self.with_etag
+ # Compatibility
+ verbose_name = kwargs.get('verbose_name')
+ if verbose_name:
+ del kwargs['verbose_name']
+ if verbose_name != _("%s file") % self.ext:
+ args = [verbose_name] + args
return name, path, args, kwargs
- @property
- def builder(self):
- """Finds a celery task suitable for the format of the field."""
- return BuildEbook.for_format(self.format_name)
+
+ @classmethod
+ def get_upload_to(cls, directory):
+ directory = getattr(cls, 'directory', cls.ext)
+ upload_template = f'book/{directory}/%s.{cls.ext}'
+ return UploadToPath(upload_template)
def contribute_to_class(self, cls, name):
super(EbookField, self).contribute_to_class(cls, name)
self.etag_field_name = f'{name}_etag'
+ if self.with_etag:
+ self.etag_field = models.CharField(max_length=255, editable=False, default='', db_index=True)
+ self.etag_field.contribute_to_class(cls, f'{name}_etag')
def has(model_instance):
return bool(getattr(model_instance, self.attname, None))
has.short_description = self.name
has.boolean = True
- self.registry.append(self)
-
setattr(cls, 'has_%s' % self.attname, has)
def get_current_etag(self):
def schedule_stale(self, queryset=None):
"""Schedule building this format for all the books where etag is stale."""
# If there is not ETag field, bail. That's true for xml file field.
- if not hasattr(self.model, f'{self.attname}_etag'):
+ if not self.with_etag:
return
etag = self.get_current_etag()
fieldfile.build_delay(priority=priority)
@classmethod
- def schedule_all_stale(cls):
+ def schedule_all_stale(cls, model):
"""Schedules all stale ebooks of all formats to rebuild."""
- for field in cls.registry:
- field.schedule_stale()
-
-
-
-class BuildEbook(Task):
- librarian2_api = False
-
- formats = {}
-
- @classmethod
- def register(cls, format_name):
- """A decorator for registering subclasses for particular formats."""
- def wrapper(builder):
- cls.formats[format_name] = builder
- return builder
- return wrapper
-
- @classmethod
- def for_format(cls, format_name):
- """Returns a celery task suitable for specified format."""
- return cls.formats.get(format_name, BuildEbookTask)
+ for field in model._meta.fields:
+ if isinstance(field, cls):
+ field.schedule_stale()
@staticmethod
- def transform(wldoc, fieldfile):
+ def transform(wldoc):
"""Transforms an librarian.WLDocument into an librarian.OutputFile.
-
- By default, it just calls relevant wldoc.as_??? method.
-
"""
- return getattr(wldoc, "as_%s" % fieldfile.field.format_name)()
-
- def run(self, obj, field_name):
- """Just run `build` on FieldFile, can't pass it directly to Celery."""
- fieldfile = getattr(obj, field_name)
-
- # Get etag value before actually building the file.
- etag = fieldfile.field.get_current_etag()
- task_logger.info("%s -> %s@%s" % (obj.slug, field_name, etag))
- ret = self.build(getattr(obj, field_name))
- fieldfile.update_etag(etag)
- obj.clear_cache()
- return ret
+ raise NotImplemented()
def set_file_permissions(self, fieldfile):
if fieldfile.instance.preview:
book = fieldfile.instance
out = self.transform(
book.wldocument2() if self.librarian2_api else book.wldocument(),
- fieldfile)
+ )
fieldfile.save(None, File(open(out.get_filename(), 'rb')), save=False)
self.set_file_permissions(fieldfile)
if book.pk is not None:
- book.save(update_fields=[fieldfile.field.attname])
- if fieldfile.field.format_name in app_settings.FORMAT_ZIPS:
- remove_zip(app_settings.FORMAT_ZIPS[fieldfile.field.format_name])
-# Don't decorate BuildEbook, because we want to subclass it.
-BuildEbookTask = task(BuildEbook, ignore_result=True)
+ book.save(update_fields=[self.attname])
+ if self.ZIP:
+ remove_zip(self.ZIP)
+
+
+class XmlField(EbookField):
+ ext = 'xml'
+
+ def build(self, fieldfile):
+ pass
-@BuildEbook.register('txt')
-@task(ignore_result=True)
-class BuildTxt(BuildEbook):
+class TxtField(EbookField):
+ ext = 'txt'
+
@staticmethod
- def transform(wldoc, fieldfile):
+ def transform(wldoc):
return wldoc.as_text()
-@BuildEbook.register('pdf')
-@task(ignore_result=True)
-class BuildPdf(BuildEbook):
+class Fb2Field(EbookField):
+ ext = 'fb2'
+ ZIP = 'wolnelektury_pl_fb2'
+
+ @staticmethod
+ def transform(wldoc):
+ return wldoc.as_fb2()
+
+
+class PdfField(EbookField):
+ ext = 'pdf'
+ ZIP = 'wolnelektury_pl_pdf'
+
@staticmethod
- def transform(wldoc, fieldfile):
+ def transform(wldoc):
return wldoc.as_pdf(
morefloats=settings.LIBRARIAN_PDF_MOREFLOATS, cover=True,
base_url=absolute_url(gallery_url(wldoc.book_info.url.slug)), customizations=['notoc'])
clear_cache(fieldfile.instance.slug)
-@BuildEbook.register('epub')
-@task(ignore_result=True)
-class BuildEpub(BuildEbook):
+class EpubField(EbookField):
+ ext = 'epub'
librarian2_api = True
+ ZIP = 'wolnelektury_pl_epub'
@staticmethod
- def transform(wldoc, fieldfile):
+ def transform(wldoc):
from librarian.builders import EpubBuilder
return EpubBuilder(
base_url='file://' + os.path.abspath(gallery_path(wldoc.meta.url.slug)) + '/',
).build(wldoc)
-@BuildEbook.register('mobi')
-@task(ignore_result=True)
-class BuildMobi(BuildEbook):
+class MobiField(EbookField):
+ ext = 'mobi'
librarian2_api = True
+ ZIP = 'wolnelektury_pl_mobi'
@staticmethod
- def transform(wldoc, fieldfile):
+ def transform(wldoc):
from librarian.builders import MobiBuilder
return MobiBuilder(
base_url='file://' + os.path.abspath(gallery_path(wldoc.meta.url.slug)) + '/',
).build(wldoc)
-@BuildEbook.register('html')
-@task(ignore_result=True)
-class BuildHtml(BuildEbook):
+class HtmlField(EbookField):
+ ext = 'html'
+
def build(self, fieldfile):
from django.core.files.base import ContentFile
from slugify import slugify
book = fieldfile.instance
- html_output = self.transform(book.wldocument(parse_dublincore=False), fieldfile)
+ html_output = self.transform(book.wldocument(parse_dublincore=False))
# Delete old fragments, create from scratch if necessary.
book.fragments.all().delete()
return False
@staticmethod
- def transform(wldoc, fieldfile):
+ def transform(wldoc):
# ugly, but we can't use wldoc.book_info here
from librarian import DCNS
url_elem = wldoc.edoc.getroot().find('.//' + DCNS('identifier.url'))
return wldoc.as_html(gallery_path=gal_path, gallery_url=gal_url, base_url=absolute_url(gal_url))
-class BuildCover(BuildEbook):
+class CoverField(EbookField):
+ ext = 'jpg'
+ directory = 'cover'
+
def set_file_permissions(self, fieldfile):
pass
-@BuildEbook.register('cover_clean')
-@task(ignore_result=True)
-class BuildCoverClean(BuildCover):
- @classmethod
- def transform(cls, wldoc, fieldfile):
+class CoverCleanField(CoverField):
+ directory = 'cover_clean'
+
+ @staticmethod
+ def transform(wldoc):
if wldoc.book_info.cover_box_position == 'none':
from librarian.cover import WLCover
return WLCover(wldoc.book_info, width=240).output_file()
return MarquiseCover(wldoc.book_info, width=240).output_file()
-@BuildEbook.register('cover_thumb')
-@task(ignore_result=True)
-class BuildCoverThumb(BuildCover):
- @classmethod
- def transform(cls, wldoc, fieldfile):
+class CoverThumbField(CoverField):
+ directory = 'cover_thumb'
+
+ @staticmethod
+ def transform(wldoc):
from librarian.cover import WLCover
return WLCover(wldoc.book_info, height=193).output_file()
-@BuildEbook.register('cover_api_thumb')
-@task(ignore_result=True)
-class BuildCoverApiThumb(BuildCover):
- @classmethod
- def transform(cls, wldoc, fieldfile):
+class CoverApiThumbField(CoverField):
+ directory = 'cover_api_thumb'
+
+ @staticmethod
+ def transform(wldoc):
from librarian.cover import WLNoBoxCover
return WLNoBoxCover(wldoc.book_info, height=500).output_file()
-@BuildEbook.register('simple_cover')
-@task(ignore_result=True)
-class BuildSimpleCover(BuildCover):
- @classmethod
- def transform(cls, wldoc, fieldfile):
+class SimpleCoverField(CoverField):
+ directory = 'cover_simple'
+
+ @staticmethod
+ def transform(wldoc):
from librarian.cover import WLNoBoxCover
return WLNoBoxCover(wldoc.book_info, height=1000).output_file()
-@BuildEbook.register('cover_ebookpoint')
-@task(ignore_result=True)
-class BuildCoverEbookpoint(BuildCover):
- @classmethod
- def transform(cls, wldoc, fieldfile):
+class CoverEbookpointField(CoverField):
+ directory = 'cover_ebookpoint'
+
+ @staticmethod
+ def transform(wldoc):
from librarian.cover import EbookpointCover
return EbookpointCover(wldoc.book_info).output_file()
-
-
-# not used, but needed for migrations
-class OverwritingFieldFile(FieldFile):
- """
- Deletes the old file before saving the new one.
- """
-
- def save(self, name, content, *args, **kwargs):
- leave = kwargs.pop('leave', None)
- # delete if there's a file already and there's a new one coming
- if not leave and self and (not hasattr(content, 'path') or content.path != self.path):
- self.delete(save=False)
- return super(OverwritingFieldFile, self).save(name, content, *args, **kwargs)
-
-
-class OverwritingFileField(models.FileField):
- attr_class = OverwritingFieldFile
-
-
-class OverwriteStorage(FileSystemStorage):
-
- def get_available_name(self, name, max_length=None):
- self.delete(name)
- return name
from django.core.management.base import BaseCommand
from catalogue.fields import EbookField
+from catalogue.models import Book
class Command(BaseCommand):
help = 'Schedule regenerating stale ebook files.'
def handle(self, **options):
- EbookField.schedule_all_stale()
+ EbookField.schedule_all_stale(Book)
('extra_info', models.TextField(default='{}', verbose_name='Additional information')),
('gazeta_link', models.CharField(max_length=240, blank=True)),
('wiki_link', models.CharField(max_length=240, blank=True)),
- ('cover', catalogue.fields.EbookField('cover', upload_to=catalogue.models.book._cover_upload_to, storage=fnpdjango.storage.BofhFileSystemStorage(), max_length=255, blank=True, null=True, verbose_name='cover')),
- ('cover_thumb', catalogue.fields.EbookField('cover_thumb', max_length=255, upload_to=catalogue.models.book._cover_thumb_upload_to, null=True, verbose_name='cover thumbnail', blank=True)),
+ ('cover', catalogue.fields.EbookField('cover', storage=fnpdjango.storage.BofhFileSystemStorage(), max_length=255, blank=True, null=True, verbose_name='cover')),
+ ('cover_thumb', catalogue.fields.EbookField('cover_thumb', max_length=255, null=True, verbose_name='cover thumbnail', blank=True)),
('_related_info', models.TextField(null=True, editable=False, blank=True)),
- ('txt_file', catalogue.fields.EbookField('txt', default='', storage=fnpdjango.storage.BofhFileSystemStorage(), upload_to=catalogue.models.book._txt_upload_to, max_length=255, blank=True, verbose_name='TXT file')),
- ('fb2_file', catalogue.fields.EbookField('fb2', default='', storage=fnpdjango.storage.BofhFileSystemStorage(), upload_to=catalogue.models.book._fb2_upload_to, max_length=255, blank=True, verbose_name='FB2 file')),
- ('pdf_file', catalogue.fields.EbookField('pdf', default='', storage=fnpdjango.storage.BofhFileSystemStorage(), upload_to=catalogue.models.book._pdf_upload_to, max_length=255, blank=True, verbose_name='PDF file')),
- ('epub_file', catalogue.fields.EbookField('epub', default='', storage=fnpdjango.storage.BofhFileSystemStorage(), upload_to=catalogue.models.book._epub_upload_to, max_length=255, blank=True, verbose_name='EPUB file')),
- ('mobi_file', catalogue.fields.EbookField('mobi', default='', storage=fnpdjango.storage.BofhFileSystemStorage(), upload_to=catalogue.models.book._mobi_upload_to, max_length=255, blank=True, verbose_name='MOBI file')),
- ('html_file', catalogue.fields.EbookField('html', default='', storage=fnpdjango.storage.BofhFileSystemStorage(), upload_to=catalogue.models.book._html_upload_to, max_length=255, blank=True, verbose_name='HTML file')),
- ('xml_file', catalogue.fields.EbookField('xml', default='', storage=fnpdjango.storage.BofhFileSystemStorage(), upload_to=catalogue.models.book._xml_upload_to, max_length=255, blank=True, verbose_name='XML file')),
+ ('txt_file', catalogue.fields.EbookField('txt', default='', storage=fnpdjango.storage.BofhFileSystemStorage(), max_length=255, blank=True, verbose_name='TXT file')),
+ ('fb2_file', catalogue.fields.EbookField('fb2', default='', storage=fnpdjango.storage.BofhFileSystemStorage(), max_length=255, blank=True, verbose_name='FB2 file')),
+ ('pdf_file', catalogue.fields.EbookField('pdf', default='', storage=fnpdjango.storage.BofhFileSystemStorage(), max_length=255, blank=True, verbose_name='PDF file')),
+ ('epub_file', catalogue.fields.EbookField('epub', default='', storage=fnpdjango.storage.BofhFileSystemStorage(), max_length=255, blank=True, verbose_name='EPUB file')),
+ ('mobi_file', catalogue.fields.EbookField('mobi', default='', storage=fnpdjango.storage.BofhFileSystemStorage(), max_length=255, blank=True, verbose_name='MOBI file')),
+ ('html_file', catalogue.fields.EbookField('html', default='', storage=fnpdjango.storage.BofhFileSystemStorage(), max_length=255, blank=True, verbose_name='HTML file')),
+ ('xml_file', catalogue.fields.EbookField('xml', default='', storage=fnpdjango.storage.BofhFileSystemStorage(), max_length=255, blank=True, verbose_name='XML file')),
('parent', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='children', blank=True, to='catalogue.Book', null=True)),
],
options={
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('type', models.CharField(db_index=True, max_length=20, verbose_name='type', choices=[('mp3', 'MP3 file'), ('ogg', 'Ogg Vorbis file'), ('daisy', 'DAISY file')])),
('name', models.CharField(max_length=512, verbose_name='name')),
- ('file', catalogue.fields.OverwritingFileField(upload_to=catalogue.models.bookmedia._file_upload_to, max_length=600, verbose_name='XML file')),
+ ('file', models.FileField(upload_to=catalogue.models.bookmedia._file_upload_to, max_length=600, verbose_name='XML file')),
('uploaded_at', models.DateTimeField(auto_now_add=True, verbose_name='creation date', db_index=True)),
('extra_info', models.TextField(default='{}', verbose_name='Additional information', editable=False)),
('source_sha1', models.CharField(max_length=40, null=True, editable=False, blank=True)),
migrations.AlterField(
model_name='bookmedia',
name='file',
- field=catalogue.fields.OverwritingFileField(upload_to=catalogue.models.bookmedia._file_upload_to, max_length=600, verbose_name='file'),
+ field=models.FileField(upload_to=catalogue.models.bookmedia._file_upload_to, max_length=600, verbose_name='file'),
),
migrations.AlterField(
model_name='collection',
# Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
#
from django.db import migrations, models
-import catalogue.fields
+import fnpdjango.storage
import catalogue.models.bookmedia
migrations.AlterField(
model_name='bookmedia',
name='file',
- field=models.FileField(storage=catalogue.fields.OverwriteStorage(), upload_to=catalogue.models.bookmedia._file_upload_to, max_length=600, verbose_name='file'),
+ field=models.FileField(storage=fnpdjango.storage.BofhFileSystemStorage(), upload_to=catalogue.models.bookmedia._file_upload_to, max_length=600, verbose_name='file'),
),
]
#
from django.db import migrations, models
import catalogue.fields
-import catalogue.models.book
class Migration(migrations.Migration):
migrations.AddField(
model_name='book',
name='cover_api_thumb',
- field=catalogue.fields.EbookField('cover_api_thumb', max_length=255, upload_to=catalogue.models.book.UploadToPath('book/cover_api_thumb/%s.jpg'), null=True, verbose_name='cover thumbnail for API', blank=True),
+ field=catalogue.fields.EbookField('cover_api_thumb', max_length=255, upload_to=catalogue.fields.UploadToPath('book/cover_api_thumb/%s.jpg'), null=True, verbose_name='cover thumbnail for API', blank=True),
),
]
#
from django.db import migrations, models
import catalogue.fields
-import catalogue.models.book
class Migration(migrations.Migration):
migrations.AddField(
model_name='book',
name='simple_cover',
- field=catalogue.fields.EbookField('simple_cover', max_length=255, upload_to=catalogue.models.book.UploadToPath('book/cover_simple/%s.jpg'), null=True, verbose_name='cover for mobile app', blank=True),
+ field=catalogue.fields.EbookField('simple_cover', max_length=255, upload_to=catalogue.fields.UploadToPath('book/cover_simple/%s.jpg'), null=True, verbose_name='cover for mobile app', blank=True),
),
migrations.AlterField(
model_name='book',
name='cover_api_thumb',
- field=catalogue.fields.EbookField('cover_api_thumb', max_length=255, upload_to=catalogue.models.book.UploadToPath('book/cover_api_thumb/%s.jpg'), null=True, verbose_name='cover thumbnail for mobile app', blank=True),
+ field=catalogue.fields.EbookField('cover_api_thumb', max_length=255, upload_to=catalogue.fields.UploadToPath('book/cover_api_thumb/%s.jpg'), null=True, verbose_name='cover thumbnail for mobile app', blank=True),
),
]
# Generated by Django 2.2.10 on 2020-04-03 09:40
import catalogue.fields
-import catalogue.models.book
from django.db import migrations
migrations.AddField(
model_name='book',
name='cover_ebookpoint',
- field=catalogue.fields.EbookField('cover_ebookpoint', blank=True, max_length=255, null=True, upload_to=catalogue.models.book.UploadToPath('book/cover_ebookpoint/%s.jpg'), verbose_name='cover for Ebookpoint'),
+ field=catalogue.fields.EbookField('cover_ebookpoint', blank=True, max_length=255, null=True, upload_to=catalogue.fields.UploadToPath('book/cover_ebookpoint/%s.jpg'), verbose_name='cover for Ebookpoint'),
),
]
# Generated by Django 2.2.25 on 2022-03-10 11:51
import catalogue.fields
-import catalogue.models.book
from django.db import migrations, models
migrations.AddField(
model_name='book',
name='cover_clean',
- field=catalogue.fields.EbookField('cover_clean', blank=True, max_length=255, null=True, upload_to=catalogue.models.book.UploadToPath('book/cover_clean/%s.jpg'), verbose_name='clean cover'),
+ field=catalogue.fields.EbookField('cover_clean', blank=True, max_length=255, null=True, upload_to=catalogue.fields.UploadToPath('book/cover_clean/%s.jpg'), verbose_name='clean cover'),
),
migrations.AddField(
model_name='book',
--- /dev/null
+# Generated by Django 2.2.28 on 2022-09-30 08:50
+
+import catalogue.fields
+from django.db import migrations
+import fnpdjango.storage
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('catalogue', '0037_auto_20220928_0124'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='book',
+ name='cover',
+ field=catalogue.fields.CoverField('cover', storage=fnpdjango.storage.BofhFileSystemStorage()),
+ ),
+ migrations.AlterField(
+ model_name='book',
+ name='cover_api_thumb',
+ field=catalogue.fields.CoverApiThumbField('cover thumbnail for mobile app'),
+ ),
+ migrations.AlterField(
+ model_name='book',
+ name='cover_clean',
+ field=catalogue.fields.CoverCleanField('clean cover'),
+ ),
+ migrations.AlterField(
+ model_name='book',
+ name='cover_ebookpoint',
+ field=catalogue.fields.CoverEbookpointField('cover for Ebookpoint'),
+ ),
+ migrations.AlterField(
+ model_name='book',
+ name='cover_thumb',
+ field=catalogue.fields.CoverThumbField('cover thumbnail'),
+ ),
+ migrations.AlterField(
+ model_name='book',
+ name='epub_file',
+ field=catalogue.fields.EpubField(storage=fnpdjango.storage.BofhFileSystemStorage()),
+ ),
+ migrations.AlterField(
+ model_name='book',
+ name='fb2_file',
+ field=catalogue.fields.Fb2Field(storage=fnpdjango.storage.BofhFileSystemStorage()),
+ ),
+ migrations.AlterField(
+ model_name='book',
+ name='html_file',
+ field=catalogue.fields.HtmlField(storage=fnpdjango.storage.BofhFileSystemStorage()),
+ ),
+ migrations.AlterField(
+ model_name='book',
+ name='mobi_file',
+ field=catalogue.fields.MobiField(storage=fnpdjango.storage.BofhFileSystemStorage()),
+ ),
+ migrations.AlterField(
+ model_name='book',
+ name='pdf_file',
+ field=catalogue.fields.PdfField(storage=fnpdjango.storage.BofhFileSystemStorage()),
+ ),
+ migrations.AlterField(
+ model_name='book',
+ name='simple_cover',
+ field=catalogue.fields.SimpleCoverField('cover for mobile app'),
+ ),
+ migrations.AlterField(
+ model_name='book',
+ name='txt_file',
+ field=catalogue.fields.TxtField(storage=fnpdjango.storage.BofhFileSystemStorage()),
+ ),
+ migrations.AlterField(
+ model_name='book',
+ name='xml_file',
+ field=catalogue.fields.XmlField(storage=fnpdjango.storage.BofhFileSystemStorage(), with_etag=False),
+ ),
+ ]
from django.template.loader import render_to_string
from django.urls import reverse
from django.utils.translation import ugettext_lazy as _, get_language
-from django.utils.deconstruct import deconstructible
from fnpdjango.storage import BofhFileSystemStorage
from lxml import html
from librarian.cover import WLCover
from librarian.html import transform_abstrakt
from newtagging import managers
from catalogue import constants
-from catalogue.fields import EbookField
+from catalogue import fields
from catalogue.models import Tag, Fragment, BookMedia
from catalogue.utils import create_zip, gallery_url, gallery_path, split_tags, get_random_hash
from catalogue.models.tag import prefetched_relations
from catalogue import app_settings
-from catalogue import tasks
from wolnelektury.utils import makedirs, cached_render, clear_cached_renders
bofh_storage = BofhFileSystemStorage()
-@deconstructible
-class UploadToPath(object):
- def __init__(self, path):
- self.path = path
-
- def __call__(self, instance, filename):
- return self.path % instance.slug
-
-
-_cover_upload_to = UploadToPath('book/cover/%s.jpg')
-_cover_clean_upload_to = UploadToPath('book/cover_clean/%s.jpg')
-_cover_thumb_upload_to = UploadToPath('book/cover_thumb/%s.jpg')
-_cover_api_thumb_upload_to = UploadToPath('book/cover_api_thumb/%s.jpg')
-_simple_cover_upload_to = UploadToPath('book/cover_simple/%s.jpg')
-_cover_ebookpoint_upload_to = UploadToPath('book/cover_ebookpoint/%s.jpg')
-
-
-def _ebook_upload_to(upload_path):
- return UploadToPath(upload_path)
-
-
class Book(models.Model):
"""Represents a book imported from WL-XML."""
title = models.CharField(_('title'), max_length=32767)
findable = models.BooleanField(_('findable'), default=True, db_index=True)
# files generated during publication
- cover = EbookField(
- 'cover', _('cover'),
- null=True, blank=True,
- upload_to=_cover_upload_to,
- storage=bofh_storage, max_length=255)
- cover_etag = models.CharField(max_length=255, editable=False, default='', db_index=True)
+ xml_file = fields.XmlField(storage=bofh_storage, with_etag=False)
+ html_file = fields.HtmlField(storage=bofh_storage)
+ fb2_file = fields.Fb2Field(storage=bofh_storage)
+ txt_file = fields.TxtField(storage=bofh_storage)
+ epub_file = fields.EpubField(storage=bofh_storage)
+ mobi_file = fields.MobiField(storage=bofh_storage)
+ pdf_file = fields.PdfField(storage=bofh_storage)
+
+ cover = fields.CoverField(_('cover'), storage=bofh_storage)
# Cleaner version of cover for thumbs
- cover_clean = EbookField(
- 'cover_clean', _('clean cover'),
- null=True, blank=True,
- upload_to=_cover_clean_upload_to,
- max_length=255
- )
- cover_clean_etag = models.CharField(max_length=255, editable=False, default='', db_index=True)
- cover_thumb = EbookField(
- 'cover_thumb', _('cover thumbnail'),
- null=True, blank=True,
- upload_to=_cover_thumb_upload_to,
- max_length=255)
- cover_thumb_etag = models.CharField(max_length=255, editable=False, default='', db_index=True)
- cover_api_thumb = EbookField(
- 'cover_api_thumb', _('cover thumbnail for mobile app'),
- null=True, blank=True,
- upload_to=_cover_api_thumb_upload_to,
- max_length=255)
- cover_api_thumb_etag = models.CharField(max_length=255, editable=False, default='', db_index=True)
- simple_cover = EbookField(
- 'simple_cover', _('cover for mobile app'),
- null=True, blank=True,
- upload_to=_simple_cover_upload_to,
- max_length=255)
- simple_cover_etag = models.CharField(max_length=255, editable=False, default='', db_index=True)
- cover_ebookpoint = EbookField(
- 'cover_ebookpoint', _('cover for Ebookpoint'),
- null=True, blank=True,
- upload_to=_cover_ebookpoint_upload_to,
- max_length=255)
- cover_ebookpoint_etag = models.CharField(max_length=255, editable=False, default='', db_index=True)
+ cover_clean = fields.CoverCleanField(_('clean cover'))
+ cover_thumb = fields.CoverThumbField(_('cover thumbnail'))
+ cover_api_thumb = fields.CoverApiThumbField(
+ _('cover thumbnail for mobile app'))
+ simple_cover = fields.SimpleCoverField(_('cover for mobile app'))
+ cover_ebookpoint = fields.CoverEbookpointField(
+ _('cover for Ebookpoint'))
+
ebook_formats = constants.EBOOK_FORMATS
formats = ebook_formats + ['html', 'xml']
if self.parent.html_file:
return self.parent
-
+
return self.parent.get_prev_text()
def get_next_text(self, inside=True):
def get_children(self):
return self.children.all().order_by('parent_number')
-
+
@property
def name(self):
return self.title
@property
def media_daisy(self):
return self.get_media('daisy')
-
+
@property
def media_audio_epub(self):
return self.get_media('audio.epub')
format_)
field_name = "%s_file" % format_
+ field = getattr(Book, field_name)
books = Book.objects.filter(parent=None).exclude(**{field_name: ""}).exclude(preview=True).exclude(findable=False)
paths = [(pretty_file_name(b), getattr(b, field_name).path) for b in books.iterator()]
- return create_zip(paths, app_settings.FORMAT_ZIPS[format_])
+ return create_zip(paths, field.ZIP)
def zip_audiobooks(self, format_):
bm = BookMedia.objects.filter(book=self, type=format_)
a.attrib['href'] = html_link + a.attrib['href']
self.toc = html.tostring(toc, encoding='unicode')
# div#toc
-
+
@classmethod
def from_xml_file(cls, xml_file, **kwargs):
from django.core.files import File
@classmethod
def from_text_and_meta(cls, raw_file, book_info, overwrite=False, dont_build=None, search_index=True,
search_index_tags=True, remote_gallery_url=None, days=0, findable=True):
+ from catalogue import tasks
+
if dont_build is None:
dont_build = set()
dont_build = set.union(set(dont_build), set(app_settings.DONT_BUILD))
for master in root.iter():
if master.tag in master_tags:
return master
-
+
def update_references(self):
from references.models import Entity, Reference
master = self.get_master()
entity.populate()
entity.save()
Reference.objects.filter(book=self).exclude(entity__uri__in=found).delete()
-
+
@property
def references(self):
return self.reference_set.all().select_related('entity')
yield self.parent
else:
return []
-
+
def clear_cache(self):
clear_cached_renders(self.mini_box)
clear_cached_renders(self.mini_box_nolink)
return fragments[0]
else:
return None
-
+
def fragment_data(self):
fragment = self.choose_fragment()
if fragment:
'no_link': True,
}
-def add_file_fields():
- for format_ in Book.formats:
- field_name = "%s_file" % format_
- # This weird globals() assignment makes Django migrations comfortable.
- _upload_to = _ebook_upload_to('book/%s/%%s.%s' % (format_, format_))
- _upload_to.__name__ = '_%s_upload_to' % format_
- globals()[_upload_to.__name__] = _upload_to
-
- EbookField(
- format_, _("%s file" % format_.upper()),
- upload_to=_upload_to,
- storage=bofh_storage,
- max_length=255,
- blank=True,
- default=''
- ).contribute_to_class(Book, field_name)
- if format_ != 'xml':
- models.CharField(max_length=255, editable=False, default='', db_index=True).contribute_to_class(Book, f'{field_name}_etag')
-
-
-add_file_fields()
-
class BookPopularity(models.Model):
book = models.OneToOneField(Book, models.CASCADE, related_name='popularity')
from slugify import slugify
import mutagen
from mutagen import id3
-
-from catalogue.fields import OverwriteStorage
+from fnpdjango.storage import BofhFileSystemStorage
def _file_upload_to(i, _n):
name = models.CharField(_('name'), max_length=512)
part_name = models.CharField(_('part name'), default='', blank=True, max_length=512)
index = models.IntegerField(_('index'), default=0)
- file = models.FileField(_('file'), max_length=600, upload_to=_file_upload_to, storage=OverwriteStorage())
+ file = models.FileField(_('file'), max_length=600, upload_to=_file_upload_to, storage=BofhFileSystemStorage())
duration = models.IntegerField(null=True, blank=True)
uploaded_at = models.DateTimeField(_('creation date'), auto_now_add=True, editable=False, db_index=True)
project_description = models.CharField(max_length=2048, blank=True)
from django.conf import settings
from django.utils import timezone
+from catalogue.models import Book
from catalogue.utils import absolute_url, gallery_url
from waiter.models import WaitedFile
type(tag).objects.filter(pk=tag.pk).update(**update_dict)
+@shared_task(ignore_result=True)
+def build_field(pk, field_name):
+ book = Book.objects.get(pk=pk)
+ task_logger.info("build %s.%s" % (book.slug, field_name))
+ field_file = getattr(book, field_name)
+ field_file.build()
+
+
@shared_task
def index_book(book_id, book_info=None, **kwargs):
- from catalogue.models import Book
try:
return Book.objects.get(id=book_id).search_index(book_info, **kwargs)
except Exception as e:
try:
from django.core.files import File
from django.core.files.storage import DefaultStorage
- from catalogue.models import Book
task_logger.info(DefaultStorage().path(file_name))
if not DefaultStorage().exists(file_name):
@shared_task(ignore_result=True)
def update_references(book_id):
- from catalogue.models import Book
Book.objects.get(id=book_id).update_references()
path('pobierz/<key>/<slug:slug>.<slug:format_>', views.embargo_link, name='embargo_link'),
# zip
- path('zip/pdf.zip', views.download_zip, {'format': 'pdf', 'slug': None}, 'download_zip_pdf'),
- path('zip/epub.zip', views.download_zip, {'format': 'epub', 'slug': None}, 'download_zip_epub'),
- path('zip/mobi.zip', views.download_zip, {'format': 'mobi', 'slug': None}, 'download_zip_mobi'),
- path('zip/mp3/<slug:slug>.zip', views.download_zip, {'format': 'mp3'}, 'download_zip_mp3'),
- path('zip/ogg/<slug:slug>.zip', views.download_zip, {'format': 'ogg'}, 'download_zip_ogg'),
+ path('zip/pdf.zip', views.download_zip, {'file_format': 'pdf', 'slug': None}, 'download_zip_pdf'),
+ path('zip/epub.zip', views.download_zip, {'file_format': 'epub', 'slug': None}, 'download_zip_epub'),
+ path('zip/mobi.zip', views.download_zip, {'file_format': 'mobi', 'slug': None}, 'download_zip_mobi'),
+ path('zip/mp3/<slug:slug>.zip', views.download_zip, {'media_format': 'mp3'}, 'download_zip_mp3'),
+ path('zip/ogg/<slug:slug>.zip', views.download_zip, {'media_format': 'ogg'}, 'download_zip_ogg'),
# Public interface. Do not change this URLs.
path('lektura/<slug:slug>.html', views.book_text, name='book_text'),
return HttpResponse(media_file, content_type=constants.EBOOK_CONTENT_TYPES[format_])
-def download_zip(request, format, slug=None):
- if format in Book.ebook_formats:
- url = Book.zip_format(format)
- elif format in ('mp3', 'ogg') and slug is not None:
+def download_zip(request, file_format=None, media_format=None, slug=None):
+ if file_format:
+ url = Book.zip_format(file_format)
+ elif media_format and slug is not None:
book = get_object_or_404(Book, slug=slug)
- url = book.zip_audiobooks(format)
+ url = book.zip_audiobooks(media_format)
else:
raise Http404('No format specified for zip package')
return HttpResponseRedirect(urlquote_plus(settings.MEDIA_URL + url, safe='/?='))
fnp / wolnelektury.git / commitdiff