src/catalogue/fields.py

   1 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
   2 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
   3 #
   4 from django.conf import settings
   5 from django.core.files import File
   6 from django.core.files.storage import FileSystemStorage
   7 from django.db import models
   8 from django.db.models.fields.files import FieldFile
   9 from catalogue import app_settings
  10 from catalogue.constants import LANGUAGES_3TO2
  11 from catalogue.utils import remove_zip, truncate_html_words, gallery_path, gallery_url
  12 from celery.task import Task, task
  13 from celery.utils.log import get_task_logger
  14 from waiter.utils import clear_cache
  15
  16 task_logger = get_task_logger(__name__)
  17
  18
  19 class EbookFieldFile(FieldFile):
  20     """Represents contents of an ebook file field."""
  21
  22     def build(self):
  23         """Build the ebook immediately."""
  24         return self.field.builder.build(self)
  25
  26     def build_delay(self):
  27         """Builds the ebook in a delayed task."""
  28         return self.field.builder.delay(self.instance, self.field.attname)
  29
  30     def get_url(self):
  31         return self.instance.media_url(self.field.attname.split('_')[0])
  32
  33     def set_readable(self, readable):
  34         import os
  35         permissions = 0o644 if readable else 0o600
  36         os.chmod(self.path, permissions)
  37
  38
  39 class EbookField(models.FileField):
  40     """Represents an ebook file field, attachable to a model."""
  41     attr_class = EbookFieldFile
  42
  43     def __init__(self, format_name, *args, **kwargs):
  44         super(EbookField, self).__init__(*args, **kwargs)
  45         self.format_name = format_name
  46
  47     def deconstruct(self):
  48         name, path, args, kwargs = super(EbookField, self).deconstruct()
  49         args.insert(0, self.format_name)
  50         return name, path, args, kwargs
  51
  52     @property
  53     def builder(self):
  54         """Finds a celery task suitable for the format of the field."""
  55         return BuildEbook.for_format(self.format_name)
  56
  57     def contribute_to_class(self, cls, name):
  58         super(EbookField, self).contribute_to_class(cls, name)
  59
  60         def has(model_instance):
  61             return bool(getattr(model_instance, self.attname, None))
  62         has.__doc__ = None
  63         has.__name__ = str("has_%s" % self.attname)
  64         has.short_description = self.name
  65         has.boolean = True
  66         setattr(cls, 'has_%s' % self.attname, has)
  67
  68
  69 class BuildEbook(Task):
  70     formats = {}
  71
  72     @classmethod
  73     def register(cls, format_name):
  74         """A decorator for registering subclasses for particular formats."""
  75         def wrapper(builder):
  76             cls.formats[format_name] = builder
  77             return builder
  78         return wrapper
  79
  80     @classmethod
  81     def for_format(cls, format_name):
  82         """Returns a celery task suitable for specified format."""
  83         return cls.formats.get(format_name, BuildEbookTask)
  84
  85     @staticmethod
  86     def transform(wldoc, fieldfile):
  87         """Transforms an librarian.WLDocument into an librarian.OutputFile.
  88
  89         By default, it just calls relevant wldoc.as_??? method.
  90
  91         """
  92         return getattr(wldoc, "as_%s" % fieldfile.field.format_name)()
  93
  94     def run(self, obj, field_name):
  95         """Just run `build` on FieldFile, can't pass it directly to Celery."""
  96         task_logger.info("%s -> %s" % (obj.slug, field_name))
  97         ret = self.build(getattr(obj, field_name))
  98         obj.clear_cache()
  99         return ret
 100
 101     def set_file_permissions(self, fieldfile):
 102         if fieldfile.instance.preview:
 103             fieldfile.set_readable(False)
 104
 105     def build(self, fieldfile):
 106         book = fieldfile.instance
 107         out = self.transform(book.wldocument(), fieldfile)
 108         fieldfile.save(None, File(open(out.get_filename(), 'rb')), save=False)
 109         self.set_file_permissions(fieldfile)
 110         if book.pk is not None:
 111             books = type(book).objects.filter(pk=book.pk)
 112             books.update(**{
 113                 fieldfile.field.attname: fieldfile
 114             })
 115             for book in books:
 116                 book.save()  # just to trigger post-save
 117         if fieldfile.field.format_name in app_settings.FORMAT_ZIPS:
 118             remove_zip(app_settings.FORMAT_ZIPS[fieldfile.field.format_name])
 119 # Don't decorate BuildEbook, because we want to subclass it.
 120 BuildEbookTask = task(BuildEbook, ignore_result=True)
 121
 122
 123 @BuildEbook.register('txt')
 124 @task(ignore_result=True)
 125 class BuildTxt(BuildEbook):
 126     @staticmethod
 127     def transform(wldoc, fieldfile):
 128         return wldoc.as_text()
 129
 130
 131 @BuildEbook.register('pdf')
 132 @task(ignore_result=True)
 133 class BuildPdf(BuildEbook):
 134     @staticmethod
 135     def transform(wldoc, fieldfile):
 136         return wldoc.as_pdf(
 137             morefloats=settings.LIBRARIAN_PDF_MOREFLOATS, cover=True,
 138             ilustr_path=gallery_path(wldoc.book_info.url.slug), customizations=['notoc'])
 139
 140     def build(self, fieldfile):
 141         BuildEbook.build(self, fieldfile)
 142         clear_cache(fieldfile.instance.slug)
 143
 144
 145 @BuildEbook.register('epub')
 146 @task(ignore_result=True)
 147 class BuildEpub(BuildEbook):
 148     @staticmethod
 149     def transform(wldoc, fieldfile):
 150         return wldoc.as_epub(cover=True, ilustr_path=gallery_path(wldoc.book_info.url.slug))
 151
 152
 153 @BuildEbook.register('mobi')
 154 @task(ignore_result=True)
 155 class BuildMobi(BuildEbook):
 156     @staticmethod
 157     def transform(wldoc, fieldfile):
 158         return wldoc.as_mobi(cover=True, ilustr_path=gallery_path(wldoc.book_info.url.slug))
 159
 160
 161 @BuildEbook.register('html')
 162 @task(ignore_result=True)
 163 class BuildHtml(BuildEbook):
 164     def build(self, fieldfile):
 165         from django.core.files.base import ContentFile
 166         from slugify import slugify
 167         from sortify import sortify
 168         from librarian import html
 169         from catalogue.models import Fragment, Tag
 170
 171         book = fieldfile.instance
 172
 173         html_output = self.transform(book.wldocument(parse_dublincore=False), fieldfile)
 174
 175         # Delete old fragments, create from scratch if necessary.
 176         book.fragments.all().delete()
 177
 178         if html_output:
 179             meta_tags = list(book.tags.filter(
 180                 category__in=('author', 'epoch', 'genre', 'kind')))
 181
 182             lang = book.language
 183             lang = LANGUAGES_3TO2.get(lang, lang)
 184             if lang not in [ln[0] for ln in settings.LANGUAGES]:
 185                 lang = None
 186
 187             fieldfile.save(None, ContentFile(html_output.get_bytes()), save=False)
 188             self.set_file_permissions(fieldfile)
 189             type(book).objects.filter(pk=book.pk).update(**{
 190                 fieldfile.field.attname: fieldfile
 191             })
 192
 193             # Extract fragments
 194             closed_fragments, open_fragments = html.extract_fragments(fieldfile.path)
 195             for fragment in closed_fragments.values():
 196                 try:
 197                     theme_names = [s.strip() for s in fragment.themes.split(',')]
 198                 except AttributeError:
 199                     continue
 200                 themes = []
 201                 for theme_name in theme_names:
 202                     if not theme_name:
 203                         continue
 204                     if lang == settings.LANGUAGE_CODE:
 205                         # Allow creating themes if book in default language.
 206                         tag, created = Tag.objects.get_or_create(
 207                             slug=slugify(theme_name),
 208                             category='theme'
 209                         )
 210                         if created:
 211                             tag.name = theme_name
 212                             setattr(tag, "name_%s" % lang, theme_name)
 213                             tag.sort_key = sortify(theme_name.lower())
 214                             tag.for_books = True
 215                             tag.save()
 216                         themes.append(tag)
 217                     elif lang is not None:
 218                         # Don't create unknown themes in non-default languages.
 219                         try:
 220                             tag = Tag.objects.get(
 221                                 category='theme',
 222                                 **{"name_%s" % lang: theme_name}
 223                             )
 224                         except Tag.DoesNotExist:
 225                             pass
 226                         else:
 227                             themes.append(tag)
 228                 if not themes:
 229                     continue
 230
 231                 text = fragment.to_string()
 232                 short_text = truncate_html_words(text, 15)
 233                 if text == short_text:
 234                     short_text = ''
 235                 new_fragment = Fragment.objects.create(
 236                     anchor=fragment.id,
 237                     book=book,
 238                     text=text,
 239                     short_text=short_text
 240                 )
 241
 242                 new_fragment.save()
 243                 new_fragment.tags = set(meta_tags + themes)
 244                 for theme in themes:
 245                     if not theme.for_books:
 246                         theme.for_books = True
 247                         theme.save()
 248             book.html_built.send(sender=type(self), instance=book)
 249             return True
 250         return False
 251
 252     @staticmethod
 253     def transform(wldoc, fieldfile):
 254         # ugly, but we can't use wldoc.book_info here
 255         from librarian import DCNS
 256         url_elem = wldoc.edoc.getroot().find('.//' + DCNS('identifier.url'))
 257         if url_elem is None:
 258             gallery = ''
 259         else:
 260             gallery = gallery_url(slug=url_elem.text.rsplit('/', 1)[1])
 261         return wldoc.as_html(options={'gallery': "'%s'" % gallery})
 262
 263
 264 class BuildCover(BuildEbook):
 265     def set_file_permissions(self, fieldfile):
 266         pass
 267
 268
 269 @BuildEbook.register('cover_thumb')
 270 @task(ignore_result=True)
 271 class BuildCoverThumb(BuildCover):
 272     @classmethod
 273     def transform(cls, wldoc, fieldfile):
 274         from librarian.cover import WLCover
 275         return WLCover(wldoc.book_info, height=193).output_file()
 276
 277
 278 @BuildEbook.register('cover_api_thumb')
 279 @task(ignore_result=True)
 280 class BuildCoverApiThumb(BuildCover):
 281     @classmethod
 282     def transform(cls, wldoc, fieldfile):
 283         from librarian.cover import WLNoBoxCover
 284         return WLNoBoxCover(wldoc.book_info, height=500).output_file()
 285
 286
 287 @BuildEbook.register('simple_cover')
 288 @task(ignore_result=True)
 289 class BuildSimpleCover(BuildCover):
 290     @classmethod
 291     def transform(cls, wldoc, fieldfile):
 292         from librarian.cover import WLNoBoxCover
 293         return WLNoBoxCover(wldoc.book_info, height=1000).output_file()
 294
 295
 296 @BuildEbook.register('cover_ebookpoint')
 297 @task(ignore_result=True)
 298 class BuildCoverEbookpoint(BuildCover):
 299     @classmethod
 300     def transform(cls, wldoc, fieldfile):
 301         from librarian.cover import EbookpointCover
 302         return EbookpointCover(wldoc.book_info).output_file()
 303
 304
 305 # not used, but needed for migrations
 306 class OverwritingFieldFile(FieldFile):
 307     """
 308         Deletes the old file before saving the new one.
 309     """
 310
 311     def save(self, name, content, *args, **kwargs):
 312         leave = kwargs.pop('leave', None)
 313         # delete if there's a file already and there's a new one coming
 314         if not leave and self and (not hasattr(content, 'path') or content.path != self.path):
 315             self.delete(save=False)
 316         return super(OverwritingFieldFile, self).save(name, content, *args, **kwargs)
 317
 318
 319 class OverwritingFileField(models.FileField):
 320     attr_class = OverwritingFieldFile
 321
 322
 323 class OverwriteStorage(FileSystemStorage):
 324
 325     def get_available_name(self, name, max_length=None):
 326         self.delete(name)
 327         return name