src/catalogue/fields.py

   1 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
   2 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
   3 #
   4 import os
   5 from django.conf import settings
   6 from django.core.files import File
   7 from django.db import models
   8 from django.db.models.fields.files import FieldFile
   9 from django.utils.deconstruct import deconstructible
  10 from django.utils.translation import gettext_lazy as _
  11 from catalogue.constants import LANGUAGES_3TO2, EBOOK_FORMATS_WITH_CHILDREN, EBOOK_FORMATS_WITHOUT_CHILDREN
  12 from catalogue.utils import absolute_url, remove_zip, truncate_html_words, gallery_path, gallery_url
  13 from waiter.utils import clear_cache
  14
  15 ETAG_SCHEDULED_SUFFIX = '-scheduled'
  16 EBOOK_BUILD_PRIORITY = 0
  17 EBOOK_REBUILD_PRIORITY = 9
  18
  19
  20 @deconstructible
  21 class UploadToPath(object):
  22     def __init__(self, path):
  23         self.path = path
  24
  25     def __call__(self, instance, filename):
  26         return self.path % instance.slug
  27
  28     def __eq__(self, other):
  29         return isinstance(other, type(self)) and other.path == self.path
  30
  31
  32 class EbookFieldFile(FieldFile):
  33     """Represents contents of an ebook file field."""
  34
  35     def build(self):
  36         """Build the ebook immediately."""
  37         etag = self.field.get_current_etag()
  38         self.field.build(self)
  39         self.update_etag(etag)
  40         self.instance.clear_cache()
  41
  42     def build_delay(self, priority=EBOOK_BUILD_PRIORITY):
  43         """Builds the ebook in a delayed task."""
  44         from .tasks import build_field
  45
  46         self.update_etag(
  47             "".join([self.field.get_current_etag(), ETAG_SCHEDULED_SUFFIX])
  48         )
  49         return build_field.apply_async(
  50             [self.instance.pk, self.field.attname],
  51             priority=priority
  52         )
  53
  54     def set_readable(self, readable):
  55         import os
  56         permissions = 0o644 if readable else 0o600
  57         os.chmod(self.path, permissions)
  58
  59     def update_etag(self, etag):
  60         setattr(self.instance, self.field.etag_field_name, etag)
  61         if self.instance.pk:
  62             self.instance.save(update_fields=[self.field.etag_field_name])
  63
  64
  65 class EbookField(models.FileField):
  66     """Represents an ebook file field, attachable to a model."""
  67     attr_class = EbookFieldFile
  68     ext = None
  69     librarian2_api = False
  70     ZIP = None
  71
  72     def __init__(self, verbose_name_=None, with_etag=True, etag_field_name=None, **kwargs):
  73         # This is just for compatibility with older migrations,
  74         # where first argument was for ebook format.
  75         # Can be scrapped if old migrations are updated/removed.
  76         verbose_name = verbose_name_ or _("%s file") % self.ext
  77         kwargs.setdefault('verbose_name', verbose_name_ )
  78
  79         # Another compatibility fix:
  80         # old migrations use EbookField directly, creating etag fields.
  81         if type(self) is EbookField:
  82             with_etag = False
  83
  84         self.with_etag = with_etag
  85         self.etag_field_name = etag_field_name
  86         kwargs.setdefault('max_length', 255)
  87         kwargs.setdefault('blank', True)
  88         kwargs.setdefault('default', '')
  89         kwargs.setdefault('upload_to', self.get_upload_to(self.ext))
  90
  91         super().__init__(**kwargs)
  92
  93     def deconstruct(self):
  94         name, path, args, kwargs = super().deconstruct()
  95         if kwargs.get('max_length') == 255:
  96             del kwargs['max_length']
  97         if kwargs.get('blank') is True:
  98             del kwargs['blank']
  99         if kwargs.get('default') == '':
 100             del kwargs['default']
 101         if self.get_upload_to(self.ext) == kwargs.get('upload_to'):
 102             del kwargs['upload_to']
 103         # with_etag creates a second field, which then deconstructs to manage
 104         # its own migrations. So for migrations, etag_field_name is explicitly
 105         # set to avoid double creation of the etag field.
 106         if self.with_etag:
 107             kwargs['etag_field_name'] = self.etag_field_name
 108         else:
 109             kwargs['with_etag'] = self.with_etag
 110
 111         # Compatibility
 112         verbose_name = kwargs.get('verbose_name')
 113         if verbose_name:
 114             del kwargs['verbose_name']
 115             if verbose_name != _("%s file") % self.ext:
 116                 args = [verbose_name] + args
 117         return name, path, args, kwargs
 118
 119
 120     @classmethod
 121     def get_upload_to(cls, directory):
 122         directory = getattr(cls, 'directory', cls.ext)
 123         upload_template = f'book/{directory}/%s.{cls.ext}'
 124         return UploadToPath(upload_template)
 125
 126     def contribute_to_class(self, cls, name):
 127         super(EbookField, self).contribute_to_class(cls, name)
 128
 129         if self.with_etag and not self.etag_field_name:
 130             self.etag_field_name = f'{name}_etag'
 131             self.etag_field = models.CharField(max_length=255, editable=False, default='', db_index=True)
 132             self.etag_field.contribute_to_class(cls, f'{name}_etag')
 133
 134         def has(model_instance):
 135             return bool(getattr(model_instance, self.attname, None))
 136         has.__doc__ = None
 137         has.__name__ = str("has_%s" % self.attname)
 138         has.short_description = self.name
 139         has.boolean = True
 140
 141         setattr(cls, 'has_%s' % self.attname, has)
 142
 143     def get_current_etag(self):
 144         import pkg_resources
 145         librarian_version = pkg_resources.get_distribution("librarian").version
 146         return librarian_version
 147
 148     def schedule_stale(self, queryset=None):
 149         """Schedule building this format for all the books where etag is stale."""
 150         # If there is not ETag field, bail. That's true for xml file field.
 151         if not self.with_etag:
 152             return
 153
 154         etag = self.get_current_etag()
 155         if queryset is None:
 156             queryset = self.model.objects.all()
 157
 158         if self.format_name in EBOOK_FORMATS_WITHOUT_CHILDREN + ['html']:
 159             queryset = queryset.filter(children=None)
 160
 161         queryset = queryset.exclude(**{
 162             f'{self.etag_field_name}__in': [
 163                 etag, f'{etag}{ETAG_SCHEDULED_SUFFIX}'
 164             ]
 165         })
 166         for obj in queryset:
 167             fieldfile = getattr(obj, self.attname)
 168             priority = EBOOK_REBUILD_PRIORITY if fieldfile else EBOOK_BUILD_PRIORITY
 169             fieldfile.build_delay(priority=priority)
 170
 171     @classmethod
 172     def schedule_all_stale(cls, model):
 173         """Schedules all stale ebooks of all formats to rebuild."""
 174         for field in model._meta.fields:
 175             if isinstance(field, cls):
 176                 field.schedule_stale()
 177
 178     @staticmethod
 179     def transform(wldoc):
 180         """Transforms an librarian.WLDocument into an librarian.OutputFile.
 181         """
 182         raise NotImplemented()
 183
 184     def set_file_permissions(self, fieldfile):
 185         if fieldfile.instance.preview:
 186             fieldfile.set_readable(False)
 187
 188     def build(self, fieldfile):
 189         book = fieldfile.instance
 190         out = self.transform(
 191             book.wldocument2() if self.librarian2_api else book.wldocument(),
 192         )
 193         fieldfile.save(None, File(open(out.get_filename(), 'rb')), save=False)
 194         self.set_file_permissions(fieldfile)
 195         if book.pk is not None:
 196             book.save(update_fields=[self.attname])
 197         if self.ZIP:
 198             remove_zip(self.ZIP)
 199
 200
 201 class XmlField(EbookField):
 202     ext = 'xml'
 203
 204     def build(self, fieldfile):
 205         pass
 206
 207
 208 class TxtField(EbookField):
 209     ext = 'txt'
 210
 211     @staticmethod
 212     def transform(wldoc):
 213         return wldoc.as_text()
 214
 215
 216 class Fb2Field(EbookField):
 217     ext = 'fb2'
 218     ZIP = 'wolnelektury_pl_fb2'
 219
 220     @staticmethod
 221     def transform(wldoc):
 222         return wldoc.as_fb2()
 223
 224
 225 class PdfField(EbookField):
 226     ext = 'pdf'
 227     ZIP = 'wolnelektury_pl_pdf'
 228
 229     @staticmethod
 230     def transform(wldoc):
 231         return wldoc.as_pdf(
 232             morefloats=settings.LIBRARIAN_PDF_MOREFLOATS, cover=True,
 233             base_url=absolute_url(gallery_url(wldoc.book_info.url.slug)), customizations=['notoc'])
 234
 235     def build(self, fieldfile):
 236         super().build(fieldfile)
 237         clear_cache(fieldfile.instance.slug)
 238
 239
 240 class EpubField(EbookField):
 241     ext = 'epub'
 242     librarian2_api = True
 243     ZIP = 'wolnelektury_pl_epub'
 244
 245     @staticmethod
 246     def transform(wldoc):
 247         from librarian.builders import EpubBuilder
 248         return EpubBuilder(
 249                 base_url='file://' + os.path.abspath(gallery_path(wldoc.meta.url.slug)) + '/',
 250                 fundraising=settings.EPUB_FUNDRAISING
 251             ).build(wldoc)
 252
 253
 254 class MobiField(EbookField):
 255     ext = 'mobi'
 256     librarian2_api = True
 257     ZIP = 'wolnelektury_pl_mobi'
 258
 259     @staticmethod
 260     def transform(wldoc):
 261         from librarian.builders import MobiBuilder
 262         return MobiBuilder(
 263                 base_url='file://' + os.path.abspath(gallery_path(wldoc.meta.url.slug)) + '/',
 264                 fundraising=settings.EPUB_FUNDRAISING
 265             ).build(wldoc)
 266
 267
 268 class HtmlField(EbookField):
 269     ext = 'html'
 270
 271     def build(self, fieldfile):
 272         from django.core.files.base import ContentFile
 273         from slugify import slugify
 274         from sortify import sortify
 275         from librarian import html
 276         from catalogue.models import Fragment, Tag
 277
 278         book = fieldfile.instance
 279
 280         html_output = self.transform(book.wldocument(parse_dublincore=False))
 281
 282         # Delete old fragments, create from scratch if necessary.
 283         book.fragments.all().delete()
 284
 285         if html_output:
 286             meta_tags = list(book.tags.filter(
 287                 category__in=('author', 'epoch', 'genre', 'kind')))
 288
 289             lang = book.language
 290             lang = LANGUAGES_3TO2.get(lang, lang)
 291             if lang not in [ln[0] for ln in settings.LANGUAGES]:
 292                 lang = None
 293
 294             fieldfile.save(None, ContentFile(html_output.get_bytes()), save=False)
 295             self.set_file_permissions(fieldfile)
 296             type(book).objects.filter(pk=book.pk).update(**{
 297                 fieldfile.field.attname: fieldfile
 298             })
 299
 300             # Extract fragments
 301             closed_fragments, open_fragments = html.extract_fragments(fieldfile.path)
 302             for fragment in closed_fragments.values():
 303                 try:
 304                     theme_names = [s.strip() for s in fragment.themes.split(',')]
 305                 except AttributeError:
 306                     continue
 307                 themes = []
 308                 for theme_name in theme_names:
 309                     if not theme_name:
 310                         continue
 311                     if lang == settings.LANGUAGE_CODE:
 312                         # Allow creating themes if book in default language.
 313                         tag, created = Tag.objects.get_or_create(
 314                             slug=slugify(theme_name),
 315                             category='theme'
 316                         )
 317                         if created:
 318                             tag.name = theme_name
 319                             setattr(tag, "name_%s" % lang, theme_name)
 320                             tag.sort_key = sortify(theme_name.lower())
 321                             tag.for_books = True
 322                             tag.save()
 323                         themes.append(tag)
 324                     elif lang is not None:
 325                         # Don't create unknown themes in non-default languages.
 326                         try:
 327                             tag = Tag.objects.get(
 328                                 category='theme',
 329                                 **{"name_%s" % lang: theme_name}
 330                             )
 331                         except Tag.DoesNotExist:
 332                             pass
 333                         else:
 334                             themes.append(tag)
 335                 if not themes:
 336                     continue
 337
 338                 text = fragment.to_string()
 339                 short_text = truncate_html_words(text, 15)
 340                 if text == short_text:
 341                     short_text = ''
 342                 new_fragment = Fragment.objects.create(
 343                     anchor=fragment.id,
 344                     book=book,
 345                     text=text,
 346                     short_text=short_text
 347                 )
 348
 349                 new_fragment.save()
 350                 new_fragment.tags = set(meta_tags + themes)
 351                 for theme in themes:
 352                     if not theme.for_books:
 353                         theme.for_books = True
 354                         theme.save()
 355             book.html_built.send(sender=type(self), instance=book)
 356             return True
 357         return False
 358
 359     @staticmethod
 360     def transform(wldoc):
 361         # ugly, but we can't use wldoc.book_info here
 362         from librarian import DCNS
 363         url_elem = wldoc.edoc.getroot().find('.//' + DCNS('identifier.url'))
 364         if url_elem is None:
 365             gal_url = ''
 366             gal_path = ''
 367         else:
 368             slug = url_elem.text.rstrip('/').rsplit('/', 1)[1]
 369             gal_url = gallery_url(slug=slug)
 370             gal_path = gallery_path(slug=slug)
 371         return wldoc.as_html(gallery_path=gal_path, gallery_url=gal_url, base_url=absolute_url(gal_url))
 372
 373
 374 class CoverField(EbookField):
 375     ext = 'jpg'
 376     directory = 'cover'
 377
 378     def set_file_permissions(self, fieldfile):
 379         pass
 380
 381
 382 class CoverCleanField(CoverField):
 383     directory = 'cover_clean'
 384
 385     @staticmethod
 386     def transform(wldoc):
 387         if wldoc.book_info.cover_box_position == 'none':
 388             from librarian.cover import WLCover
 389             return WLCover(wldoc.book_info, width=240).output_file()
 390         from librarian.covers.marquise import MarquiseCover
 391         return MarquiseCover(wldoc.book_info, width=240).output_file()
 392
 393
 394 class CoverThumbField(CoverField):
 395     directory = 'cover_thumb'
 396
 397     @staticmethod
 398     def transform(wldoc):
 399         from librarian.cover import WLCover
 400         return WLCover(wldoc.book_info, height=193).output_file()
 401
 402
 403 class CoverApiThumbField(CoverField):
 404     directory = 'cover_api_thumb'
 405
 406     @staticmethod
 407     def transform(wldoc):
 408         from librarian.cover import WLNoBoxCover
 409         return WLNoBoxCover(wldoc.book_info, height=500).output_file()
 410
 411
 412 class SimpleCoverField(CoverField):
 413     directory = 'cover_simple'
 414
 415     @staticmethod
 416     def transform(wldoc):
 417         from librarian.cover import WLNoBoxCover
 418         return WLNoBoxCover(wldoc.book_info, height=1000).output_file()
 419
 420
 421 class CoverEbookpointField(CoverField):
 422     directory = 'cover_ebookpoint'
 423
 424     @staticmethod
 425     def transform(wldoc):
 426         from librarian.cover import EbookpointCover
 427         return EbookpointCover(wldoc.book_info).output_file()