src/catalogue/fields.py

   1 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
   2 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
   3 #
   4 import os
   5 from django.conf import settings
   6 from django.core.files import File
   7 from django.core.files.storage import FileSystemStorage
   8 from django.db import models
   9 from django.db.models.fields.files import FieldFile
  10 from catalogue import app_settings
  11 from catalogue.constants import LANGUAGES_3TO2, EBOOK_FORMATS_WITH_CHILDREN, EBOOK_FORMATS_WITHOUT_CHILDREN
  12 from catalogue.utils import absolute_url, remove_zip, truncate_html_words, gallery_path, gallery_url
  13 from celery.task import Task, task
  14 from celery.utils.log import get_task_logger
  15 from waiter.utils import clear_cache
  16
  17 task_logger = get_task_logger(__name__)
  18
  19 ETAG_SCHEDULED_SUFFIX = '-scheduled'
  20 EBOOK_BUILD_PRIORITY = 0
  21 EBOOK_REBUILD_PRIORITY = 9
  22
  23
  24 class EbookFieldFile(FieldFile):
  25     """Represents contents of an ebook file field."""
  26
  27     def build(self):
  28         """Build the ebook immediately."""
  29         return self.field.builder.build(self)
  30
  31     def build_delay(self, priority=EBOOK_BUILD_PRIORITY):
  32         """Builds the ebook in a delayed task."""
  33         self.update_etag(
  34             "".join([self.field.get_current_etag(), ETAG_SCHEDULED_SUFFIX])
  35         )
  36         return self.field.builder.apply_async(
  37             [self.instance, self.field.attname],
  38             priority=priority
  39         )
  40
  41     def get_url(self):
  42         return self.instance.media_url(self.field.attname.split('_')[0])
  43
  44     def set_readable(self, readable):
  45         import os
  46         permissions = 0o644 if readable else 0o600
  47         os.chmod(self.path, permissions)
  48
  49     def update_etag(self, etag):
  50         setattr(self.instance, self.field.etag_field_name, etag)
  51         if self.instance.pk:
  52             self.instance.save(update_fields=[self.field.etag_field_name])
  53
  54
  55 class EbookField(models.FileField):
  56     """Represents an ebook file field, attachable to a model."""
  57     attr_class = EbookFieldFile
  58     registry = []
  59     librarian2_api = False
  60
  61     def __init__(self, format_name, *args, **kwargs):
  62         super(EbookField, self).__init__(*args, **kwargs)
  63         self.format_name = format_name
  64
  65     def deconstruct(self):
  66         name, path, args, kwargs = super(EbookField, self).deconstruct()
  67         args.insert(0, self.format_name)
  68         return name, path, args, kwargs
  69
  70     @property
  71     def builder(self):
  72         """Finds a celery task suitable for the format of the field."""
  73         return BuildEbook.for_format(self.format_name)
  74
  75     def contribute_to_class(self, cls, name):
  76         super(EbookField, self).contribute_to_class(cls, name)
  77
  78         self.etag_field_name = f'{name}_etag'
  79
  80         def has(model_instance):
  81             return bool(getattr(model_instance, self.attname, None))
  82         has.__doc__ = None
  83         has.__name__ = str("has_%s" % self.attname)
  84         has.short_description = self.name
  85         has.boolean = True
  86
  87         self.registry.append(self)
  88
  89         setattr(cls, 'has_%s' % self.attname, has)
  90
  91     def get_current_etag(self):
  92         import pkg_resources
  93         librarian_version = pkg_resources.get_distribution("librarian").version
  94         return librarian_version
  95
  96     def schedule_stale(self, queryset=None):
  97         """Schedule building this format for all the books where etag is stale."""
  98         # If there is not ETag field, bail. That's true for xml file field.
  99         if not hasattr(self.model, f'{self.attname}_etag'):
 100             return
 101
 102         etag = self.get_current_etag()
 103         if queryset is None:
 104             queryset = self.model.objects.all()
 105
 106         if self.format_name in EBOOK_FORMATS_WITHOUT_CHILDREN + ['html']:
 107             queryset = queryset.filter(children=None)
 108
 109         queryset = queryset.exclude(**{
 110             f'{self.etag_field_name}__in': [
 111                 etag, f'{etag}{ETAG_SCHEDULED_SUFFIX}'
 112             ]
 113         })
 114         for obj in queryset:
 115             fieldfile = getattr(obj, self.attname)
 116             priority = EBOOK_REBUILD_PRIORITY if fieldfile else EBOOK_BUILD_PRIORITY
 117             fieldfile.build_delay(priority=priority)
 118
 119     @classmethod
 120     def schedule_all_stale(cls):
 121         """Schedules all stale ebooks of all formats to rebuild."""
 122         for field in cls.registry:
 123             field.schedule_stale()
 124
 125
 126
 127 class BuildEbook(Task):
 128     formats = {}
 129
 130     @classmethod
 131     def register(cls, format_name):
 132         """A decorator for registering subclasses for particular formats."""
 133         def wrapper(builder):
 134             cls.formats[format_name] = builder
 135             return builder
 136         return wrapper
 137
 138     @classmethod
 139     def for_format(cls, format_name):
 140         """Returns a celery task suitable for specified format."""
 141         return cls.formats.get(format_name, BuildEbookTask)
 142
 143     @staticmethod
 144     def transform(wldoc, fieldfile):
 145         """Transforms an librarian.WLDocument into an librarian.OutputFile.
 146
 147         By default, it just calls relevant wldoc.as_??? method.
 148
 149         """
 150         return getattr(wldoc, "as_%s" % fieldfile.field.format_name)()
 151
 152     def run(self, obj, field_name):
 153         """Just run `build` on FieldFile, can't pass it directly to Celery."""
 154         fieldfile = getattr(obj, field_name)
 155
 156         # Get etag value before actually building the file.
 157         etag = fieldfile.field.get_current_etag()
 158         task_logger.info("%s -> %s@%s" % (obj.slug, field_name, etag))
 159         ret = self.build(getattr(obj, field_name))
 160         fieldfile.update_etag(etag)
 161         obj.clear_cache()
 162         return ret
 163
 164     def set_file_permissions(self, fieldfile):
 165         if fieldfile.instance.preview:
 166             fieldfile.set_readable(False)
 167
 168     def build(self, fieldfile):
 169         book = fieldfile.instance
 170         out = self.transform(
 171             book.wldocument2() if self.librarian2_api else book.wldocument(),
 172             fieldfile)
 173         fieldfile.save(None, File(open(out.get_filename(), 'rb')), save=False)
 174         self.set_file_permissions(fieldfile)
 175         if book.pk is not None:
 176             book.save(update_fields=[fieldfile.field.attname])
 177         if fieldfile.field.format_name in app_settings.FORMAT_ZIPS:
 178             remove_zip(app_settings.FORMAT_ZIPS[fieldfile.field.format_name])
 179 # Don't decorate BuildEbook, because we want to subclass it.
 180 BuildEbookTask = task(BuildEbook, ignore_result=True)
 181
 182
 183 @BuildEbook.register('txt')
 184 @task(ignore_result=True)
 185 class BuildTxt(BuildEbook):
 186     @staticmethod
 187     def transform(wldoc, fieldfile):
 188         return wldoc.as_text()
 189
 190
 191 @BuildEbook.register('pdf')
 192 @task(ignore_result=True)
 193 class BuildPdf(BuildEbook):
 194     @staticmethod
 195     def transform(wldoc, fieldfile):
 196         return wldoc.as_pdf(
 197             morefloats=settings.LIBRARIAN_PDF_MOREFLOATS, cover=True,
 198             base_url=absolute_url(gallery_url(wldoc.book_info.url.slug)), customizations=['notoc'])
 199
 200     def build(self, fieldfile):
 201         BuildEbook.build(self, fieldfile)
 202         clear_cache(fieldfile.instance.slug)
 203
 204
 205 @BuildEbook.register('epub')
 206 @task(ignore_result=True)
 207 class BuildEpub(BuildEbook):
 208     librarian2_api = True
 209
 210     @staticmethod
 211     def transform(wldoc, fieldfile):
 212         from librarian.builders import EpubBuilder
 213         return EpubBuilder(
 214                 base_url='file://' + os.path.abspath(gallery_path(wldoc.meta.url.slug)) + '/',
 215                 fundraising=settings.EPUB_FUNDRAISING
 216             ).build(wldoc)
 217
 218
 219 @BuildEbook.register('mobi')
 220 @task(ignore_result=True)
 221 class BuildMobi(BuildEbook):
 222     @staticmethod
 223     def transform(wldoc, fieldfile):
 224         return wldoc.as_mobi(cover=True, base_url=absolute_url(gallery_url(wldoc.book_info.url.slug)))
 225
 226
 227 @BuildEbook.register('html')
 228 @task(ignore_result=True)
 229 class BuildHtml(BuildEbook):
 230     def build(self, fieldfile):
 231         from django.core.files.base import ContentFile
 232         from slugify import slugify
 233         from sortify import sortify
 234         from librarian import html
 235         from catalogue.models import Fragment, Tag
 236
 237         book = fieldfile.instance
 238
 239         html_output = self.transform(book.wldocument(parse_dublincore=False), fieldfile)
 240
 241         # Delete old fragments, create from scratch if necessary.
 242         book.fragments.all().delete()
 243
 244         if html_output:
 245             meta_tags = list(book.tags.filter(
 246                 category__in=('author', 'epoch', 'genre', 'kind')))
 247
 248             lang = book.language
 249             lang = LANGUAGES_3TO2.get(lang, lang)
 250             if lang not in [ln[0] for ln in settings.LANGUAGES]:
 251                 lang = None
 252
 253             fieldfile.save(None, ContentFile(html_output.get_bytes()), save=False)
 254             self.set_file_permissions(fieldfile)
 255             type(book).objects.filter(pk=book.pk).update(**{
 256                 fieldfile.field.attname: fieldfile
 257             })
 258
 259             # Extract fragments
 260             closed_fragments, open_fragments = html.extract_fragments(fieldfile.path)
 261             for fragment in closed_fragments.values():
 262                 try:
 263                     theme_names = [s.strip() for s in fragment.themes.split(',')]
 264                 except AttributeError:
 265                     continue
 266                 themes = []
 267                 for theme_name in theme_names:
 268                     if not theme_name:
 269                         continue
 270                     if lang == settings.LANGUAGE_CODE:
 271                         # Allow creating themes if book in default language.
 272                         tag, created = Tag.objects.get_or_create(
 273                             slug=slugify(theme_name),
 274                             category='theme'
 275                         )
 276                         if created:
 277                             tag.name = theme_name
 278                             setattr(tag, "name_%s" % lang, theme_name)
 279                             tag.sort_key = sortify(theme_name.lower())
 280                             tag.for_books = True
 281                             tag.save()
 282                         themes.append(tag)
 283                     elif lang is not None:
 284                         # Don't create unknown themes in non-default languages.
 285                         try:
 286                             tag = Tag.objects.get(
 287                                 category='theme',
 288                                 **{"name_%s" % lang: theme_name}
 289                             )
 290                         except Tag.DoesNotExist:
 291                             pass
 292                         else:
 293                             themes.append(tag)
 294                 if not themes:
 295                     continue
 296
 297                 text = fragment.to_string()
 298                 short_text = truncate_html_words(text, 15)
 299                 if text == short_text:
 300                     short_text = ''
 301                 new_fragment = Fragment.objects.create(
 302                     anchor=fragment.id,
 303                     book=book,
 304                     text=text,
 305                     short_text=short_text
 306                 )
 307
 308                 new_fragment.save()
 309                 new_fragment.tags = set(meta_tags + themes)
 310                 for theme in themes:
 311                     if not theme.for_books:
 312                         theme.for_books = True
 313                         theme.save()
 314             book.html_built.send(sender=type(self), instance=book)
 315             return True
 316         return False
 317
 318     @staticmethod
 319     def transform(wldoc, fieldfile):
 320         # ugly, but we can't use wldoc.book_info here
 321         from librarian import DCNS
 322         url_elem = wldoc.edoc.getroot().find('.//' + DCNS('identifier.url'))
 323         if url_elem is None:
 324             gal_url = ''
 325             gal_path = ''
 326         else:
 327             slug = url_elem.text.rstrip('/').rsplit('/', 1)[1]
 328             gal_url = gallery_url(slug=slug)
 329             gal_path = gallery_path(slug=slug)
 330         return wldoc.as_html(gallery_path=gal_path, gallery_url=gal_url, base_url=absolute_url(gal_url))
 331
 332
 333 class BuildCover(BuildEbook):
 334     def set_file_permissions(self, fieldfile):
 335         pass
 336
 337
 338 @BuildEbook.register('cover_thumb')
 339 @task(ignore_result=True)
 340 class BuildCoverThumb(BuildCover):
 341     @classmethod
 342     def transform(cls, wldoc, fieldfile):
 343         from librarian.cover import WLCover
 344         return WLCover(wldoc.book_info, height=193).output_file()
 345
 346
 347 @BuildEbook.register('cover_api_thumb')
 348 @task(ignore_result=True)
 349 class BuildCoverApiThumb(BuildCover):
 350     @classmethod
 351     def transform(cls, wldoc, fieldfile):
 352         from librarian.cover import WLNoBoxCover
 353         return WLNoBoxCover(wldoc.book_info, height=500).output_file()
 354
 355
 356 @BuildEbook.register('simple_cover')
 357 @task(ignore_result=True)
 358 class BuildSimpleCover(BuildCover):
 359     @classmethod
 360     def transform(cls, wldoc, fieldfile):
 361         from librarian.cover import WLNoBoxCover
 362         return WLNoBoxCover(wldoc.book_info, height=1000).output_file()
 363
 364
 365 @BuildEbook.register('cover_ebookpoint')
 366 @task(ignore_result=True)
 367 class BuildCoverEbookpoint(BuildCover):
 368     @classmethod
 369     def transform(cls, wldoc, fieldfile):
 370         from librarian.cover import EbookpointCover
 371         return EbookpointCover(wldoc.book_info).output_file()
 372
 373
 374 # not used, but needed for migrations
 375 class OverwritingFieldFile(FieldFile):
 376     """
 377         Deletes the old file before saving the new one.
 378     """
 379
 380     def save(self, name, content, *args, **kwargs):
 381         leave = kwargs.pop('leave', None)
 382         # delete if there's a file already and there's a new one coming
 383         if not leave and self and (not hasattr(content, 'path') or content.path != self.path):
 384             self.delete(save=False)
 385         return super(OverwritingFieldFile, self).save(name, content, *args, **kwargs)
 386
 387
 388 class OverwritingFileField(models.FileField):
 389     attr_class = OverwritingFieldFile
 390
 391
 392 class OverwriteStorage(FileSystemStorage):
 393
 394     def get_available_name(self, name, max_length=None):
 395         self.delete(name)
 396         return name