src/catalogue/fields.py

   1 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
   2 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
   3 #
   4 import os
   5 import pkg_resources
   6 import random
   7 from django.apps import apps
   8 from django.conf import settings
   9 from django.core.files import File
  10 from django.db import models
  11 from django.db.models.fields.files import FieldFile
  12 from django.utils.deconstruct import deconstructible
  13 from django.utils.translation import gettext_lazy as _
  14 from catalogue.constants import LANGUAGES_3TO2
  15 from catalogue.utils import absolute_url, remove_zip, truncate_html_words, gallery_path, gallery_url
  16 from waiter.utils import clear_cache
  17
  18 ETAG_SCHEDULED_SUFFIX = '-scheduled'
  19 EBOOK_BUILD_PRIORITY = 0
  20 EBOOK_REBUILD_PRIORITY = 9
  21
  22
  23 @deconstructible
  24 class UploadToPath(object):
  25     def __init__(self, path):
  26         self.path = path
  27
  28     def __call__(self, instance, filename):
  29         return self.path % instance.slug
  30
  31     def __eq__(self, other):
  32         return isinstance(other, type(self)) and other.path == self.path
  33
  34
  35 class EbookFieldFile(FieldFile):
  36     """Represents contents of an ebook file field."""
  37
  38     def build(self):
  39         """Build the ebook immediately."""
  40         etag = self.field.get_current_etag()
  41         self.field.build(self)
  42         self.update_etag(etag)
  43         self.instance.clear_cache()
  44
  45     def build_delay(self, priority=EBOOK_BUILD_PRIORITY):
  46         """Builds the ebook in a delayed task."""
  47         from .tasks import build_field
  48
  49         self.update_etag(
  50             "".join([self.field.get_current_etag(), ETAG_SCHEDULED_SUFFIX])
  51         )
  52         return build_field.apply_async(
  53             [self.instance.pk, self.field.attname],
  54             priority=priority
  55         )
  56
  57     def set_readable(self, readable):
  58         import os
  59         permissions = 0o644 if readable else 0o600
  60         os.chmod(self.path, permissions)
  61
  62     def update_etag(self, etag):
  63         setattr(self.instance, self.field.etag_field_name, etag)
  64         if self.instance.pk:
  65             self.instance.save(update_fields=[self.field.etag_field_name])
  66
  67
  68 class EbookField(models.FileField):
  69     """Represents an ebook file field, attachable to a model."""
  70     attr_class = EbookFieldFile
  71     ext = None
  72     for_parents = True
  73     librarian2_api = False
  74     ZIP = None
  75
  76     def __init__(self, verbose_name=None, with_etag=True, etag_field_name=None, **kwargs):
  77         kwargs.setdefault('verbose_name', verbose_name)
  78         self.with_etag = with_etag
  79         self.etag_field_name = etag_field_name
  80         kwargs.setdefault('max_length', 255)
  81         kwargs.setdefault('blank', True)
  82         kwargs.setdefault('default', '')
  83         kwargs.setdefault('upload_to', self.get_upload_to(self.ext))
  84
  85         super().__init__(**kwargs)
  86
  87     def deconstruct(self):
  88         name, path, args, kwargs = super().deconstruct()
  89         if kwargs.get('max_length') == 255:
  90             del kwargs['max_length']
  91         if kwargs.get('blank') is True:
  92             del kwargs['blank']
  93         if kwargs.get('default') == '':
  94             del kwargs['default']
  95         if self.get_upload_to(self.ext) == kwargs.get('upload_to'):
  96             del kwargs['upload_to']
  97         # with_etag creates a second field, which then deconstructs to manage
  98         # its own migrations. So for migrations, etag_field_name is explicitly
  99         # set to avoid double creation of the etag field.
 100         if self.with_etag:
 101             kwargs['etag_field_name'] = self.etag_field_name
 102         else:
 103             kwargs['with_etag'] = self.with_etag
 104
 105         return name, path, args, kwargs
 106
 107     @classmethod
 108     def get_upload_to(cls, directory):
 109         directory = getattr(cls, 'directory', cls.ext)
 110         upload_template = f'book/{directory}/%s.{cls.ext}'
 111         return UploadToPath(upload_template)
 112
 113     def contribute_to_class(self, cls, name):
 114         super(EbookField, self).contribute_to_class(cls, name)
 115
 116         if self.with_etag and not self.etag_field_name:
 117             self.etag_field_name = f'{name}_etag'
 118             self.etag_field = models.CharField(max_length=255, editable=False, default='', db_index=True)
 119             self.etag_field.contribute_to_class(cls, f'{name}_etag')
 120
 121         def has(model_instance):
 122             return bool(getattr(model_instance, self.attname, None))
 123         has.__doc__ = None
 124         has.__name__ = str("has_%s" % self.attname)
 125         has.short_description = self.name
 126         has.boolean = True
 127
 128         setattr(cls, 'has_%s' % self.attname, has)
 129
 130     def get_current_etag(self):
 131         MediaInsertSet = apps.get_model('annoy', 'MediaInsertSet')
 132         librarian_version = pkg_resources.get_distribution("librarian").version
 133         etag = librarian_version
 134         mis = MediaInsertSet.get_for_format(self.ext)
 135         if mis is not None:
 136             etag += '_' + mis.etag
 137         return etag
 138
 139     def find_stale(self, limit):
 140         """Find some books where this format is stale."""
 141         # If there is not ETag field, bail. That's true for xml file field.
 142         if not self.with_etag:
 143             return []
 144
 145         etag = self.get_current_etag()
 146
 147         queryset = self.model.objects.all()
 148         if not self.for_parents:
 149             queryset = queryset.filter(children=None)
 150
 151         queryset = queryset.exclude(**{
 152             f'{self.etag_field_name}__in': [
 153                 etag, f'{etag}{ETAG_SCHEDULED_SUFFIX}'
 154            ]
 155         })
 156
 157         queryset = queryset.order_by('?')[:limit]
 158         return queryset
 159
 160     @classmethod
 161     def find_all_stale(cls, model, limit):
 162         """Schedules all stale ebooks of all formats to rebuild."""
 163         found = []
 164         for field in model._meta.fields:
 165             if isinstance(field, cls):
 166                 for instance in field.find_stale(limit):
 167                     found.append((
 168                         field.name,
 169                         instance
 170                     ))
 171         random.shuffle(found)
 172         found = found[:limit]
 173         return found
 174
 175     @staticmethod
 176     def transform(wldoc):
 177         """Transforms an librarian.WLDocument into an librarian.OutputFile.
 178         """
 179         raise NotImplemented()
 180
 181     def set_file_permissions(self, fieldfile):
 182         if fieldfile.instance.preview:
 183             fieldfile.set_readable(False)
 184
 185     def build(self, fieldfile):
 186         book = fieldfile.instance
 187         out = self.transform(
 188             book.wldocument2() if self.librarian2_api else book.wldocument(),
 189         )
 190         with open(out.get_filename(), 'rb') as f:
 191             fieldfile.save(None, File(f), save=False)
 192         self.set_file_permissions(fieldfile)
 193         if book.pk is not None:
 194             book.save(update_fields=[self.attname])
 195         if self.ZIP:
 196             remove_zip(self.ZIP)
 197
 198
 199 class XmlField(EbookField):
 200     ext = 'xml'
 201
 202     def build(self, fieldfile):
 203         pass
 204
 205
 206 class TxtField(EbookField):
 207     ext = 'txt'
 208     for_parents = False
 209
 210     @staticmethod
 211     def transform(wldoc):
 212         return wldoc.as_text()
 213
 214
 215 class Fb2Field(EbookField):
 216     ext = 'fb2'
 217     for_parents = False
 218     ZIP = 'wolnelektury_pl_fb2'
 219
 220     @staticmethod
 221     def transform(wldoc):
 222         return wldoc.as_fb2()
 223
 224
 225 class PdfField(EbookField):
 226     ext = 'pdf'
 227     ZIP = 'wolnelektury_pl_pdf'
 228
 229     @staticmethod
 230     def transform(wldoc):
 231         return wldoc.as_pdf(
 232             morefloats=settings.LIBRARIAN_PDF_MOREFLOATS, cover=True,
 233             base_url=absolute_url(gallery_url(wldoc.book_info.url.slug)), customizations=['notoc'])
 234
 235     def build(self, fieldfile):
 236         super().build(fieldfile)
 237         clear_cache(fieldfile.instance.slug)
 238
 239
 240 class EpubField(EbookField):
 241     ext = 'epub'
 242     librarian2_api = True
 243     ZIP = 'wolnelektury_pl_epub'
 244
 245     @staticmethod
 246     def transform(wldoc):
 247         from librarian.builders import EpubBuilder
 248         MediaInsertSet = apps.get_model('annoy', 'MediaInsertSet')
 249         return EpubBuilder(
 250                 base_url='file://' + os.path.abspath(gallery_path(wldoc.meta.url.slug)) + '/',
 251                 fundraising=MediaInsertSet.get_texts_for('epub')
 252             ).build(wldoc)
 253
 254
 255 class MobiField(EbookField):
 256     ext = 'mobi'
 257     librarian2_api = True
 258     ZIP = 'wolnelektury_pl_mobi'
 259
 260     @staticmethod
 261     def transform(wldoc):
 262         from librarian.builders import MobiBuilder
 263         MediaInsertSet = apps.get_model('annoy', 'MediaInsertSet')
 264         return MobiBuilder(
 265                 base_url='file://' + os.path.abspath(gallery_path(wldoc.meta.url.slug)) + '/',
 266                 fundraising=MediaInsertSet.get_texts_for('mobi')
 267             ).build(wldoc)
 268
 269
 270 class HtmlField(EbookField):
 271     ext = 'html'
 272     for_parents = False
 273
 274     def build(self, fieldfile):
 275         from django.core.files.base import ContentFile
 276         from slugify import slugify
 277         from sortify import sortify
 278         from librarian import html
 279         from catalogue.models import Fragment, Tag
 280
 281         book = fieldfile.instance
 282
 283         html_output = self.transform(book.wldocument(parse_dublincore=False))
 284
 285         # Delete old fragments, create from scratch if necessary.
 286         book.fragments.all().delete()
 287
 288         if html_output:
 289             meta_tags = list(book.tags.filter(
 290                 category__in=('author', 'epoch', 'genre', 'kind')))
 291
 292             lang = book.language
 293             lang = LANGUAGES_3TO2.get(lang, lang)
 294             if lang not in [ln[0] for ln in settings.LANGUAGES]:
 295                 lang = None
 296
 297             fieldfile.save(None, ContentFile(html_output.get_bytes()), save=False)
 298             self.set_file_permissions(fieldfile)
 299             type(book).objects.filter(pk=book.pk).update(**{
 300                 fieldfile.field.attname: fieldfile
 301             })
 302
 303             # Extract fragments
 304             closed_fragments, open_fragments = html.extract_fragments(fieldfile.path)
 305             for fragment in closed_fragments.values():
 306                 try:
 307                     theme_names = [s.strip() for s in fragment.themes.split(',')]
 308                 except AttributeError:
 309                     continue
 310                 themes = []
 311                 for theme_name in theme_names:
 312                     if not theme_name:
 313                         continue
 314                     if lang == settings.LANGUAGE_CODE:
 315                         # Allow creating themes if book in default language.
 316                         tag, created = Tag.objects.get_or_create(
 317                             slug=slugify(theme_name),
 318                             category='theme'
 319                         )
 320                         if created:
 321                             tag.name = theme_name
 322                             setattr(tag, "name_%s" % lang, theme_name)
 323                             tag.sort_key = sortify(theme_name.lower())
 324                             tag.for_books = True
 325                             tag.save()
 326                         themes.append(tag)
 327                     elif lang is not None:
 328                         # Don't create unknown themes in non-default languages.
 329                         try:
 330                             tag = Tag.objects.get(
 331                                 category='theme',
 332                                 **{"name_%s" % lang: theme_name}
 333                             )
 334                         except Tag.DoesNotExist:
 335                             pass
 336                         else:
 337                             themes.append(tag)
 338                 if not themes:
 339                     continue
 340
 341                 text = fragment.to_string()
 342                 short_text = truncate_html_words(text, 15)
 343                 if text == short_text:
 344                     short_text = ''
 345                 new_fragment = Fragment.objects.create(
 346                     anchor=fragment.id,
 347                     book=book,
 348                     text=text,
 349                     short_text=short_text
 350                 )
 351
 352                 new_fragment.save()
 353                 new_fragment.tags = set(meta_tags + themes)
 354                 for theme in themes:
 355                     if not theme.for_books:
 356                         theme.for_books = True
 357                         theme.save()
 358             book.html_built.send(sender=type(self), instance=book)
 359             return True
 360         return False
 361
 362     @staticmethod
 363     def transform(wldoc):
 364         # ugly, but we can't use wldoc.book_info here
 365         from librarian import DCNS
 366         url_elem = wldoc.edoc.getroot().find('.//' + DCNS('identifier.url'))
 367         if url_elem is None:
 368             gal_url = ''
 369             gal_path = ''
 370         else:
 371             slug = url_elem.text.rstrip('/').rsplit('/', 1)[1]
 372             gal_url = gallery_url(slug=slug)
 373             gal_path = gallery_path(slug=slug)
 374         return wldoc.as_html(gallery_path=gal_path, gallery_url=gal_url, base_url=absolute_url(gal_url))
 375
 376
 377 class CoverField(EbookField):
 378     ext = 'jpg'
 379     directory = 'cover'
 380
 381     @staticmethod
 382     def transform(wldoc):
 383         return wldoc.as_cover()
 384
 385     def set_file_permissions(self, fieldfile):
 386         pass
 387
 388
 389 class CoverCleanField(CoverField):
 390     directory = 'cover_clean'
 391
 392     @staticmethod
 393     def transform(wldoc):
 394         from librarian.covers.marquise import MarquiseCover
 395         return MarquiseCover(wldoc.book_info, width=360).output_file()
 396
 397
 398 class CoverThumbField(CoverField):
 399     directory = 'cover_thumb'
 400
 401     @staticmethod
 402     def transform(wldoc):
 403         from librarian.cover import WLCover
 404         return WLCover(wldoc.book_info, height=193).output_file()
 405
 406
 407 class CoverApiThumbField(CoverField):
 408     directory = 'cover_api_thumb'
 409
 410     @staticmethod
 411     def transform(wldoc):
 412         from librarian.cover import WLNoBoxCover
 413         return WLNoBoxCover(wldoc.book_info, height=500).output_file()
 414
 415
 416 class SimpleCoverField(CoverField):
 417     directory = 'cover_simple'
 418
 419     @staticmethod
 420     def transform(wldoc):
 421         from librarian.cover import WLNoBoxCover
 422         return WLNoBoxCover(wldoc.book_info, height=1000).output_file()
 423
 424
 425 class CoverEbookpointField(CoverField):
 426     directory = 'cover_ebookpoint'
 427
 428     @staticmethod
 429     def transform(wldoc):
 430         from librarian.cover import EbookpointCover
 431         return EbookpointCover(wldoc.book_info).output_file()