src/catalogue/fields.py

   1 # -*- coding: utf-8 -*-
   2 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
   3 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
   4 #
   5 from django.conf import settings
   6 from django.core.files import File
   7 from django.core.files.storage import FileSystemStorage
   8 from django.db import models
   9 from django.db.models.fields.files import FieldFile
  10 from catalogue import app_settings
  11 from catalogue.constants import LANGUAGES_3TO2
  12 from catalogue.utils import remove_zip, truncate_html_words, gallery_path, gallery_url
  13 from celery.task import Task, task
  14 from celery.utils.log import get_task_logger
  15 from waiter.utils import clear_cache
  16
  17 task_logger = get_task_logger(__name__)
  18
  19
  20 class EbookFieldFile(FieldFile):
  21     """Represents contents of an ebook file field."""
  22
  23     def build(self):
  24         """Build the ebook immediately."""
  25         return self.field.builder.build(self)
  26
  27     def build_delay(self):
  28         """Builds the ebook in a delayed task."""
  29         return self.field.builder.delay(self.instance, self.field.attname)
  30
  31     def get_url(self):
  32         return self.instance.media_url(self.field.attname.split('_')[0])
  33
  34     def set_readable(self, readable):
  35         import os
  36         permissions = 0o644 if readable else 0o600
  37         os.chmod(self.path, permissions)
  38
  39
  40 class EbookField(models.FileField):
  41     """Represents an ebook file field, attachable to a model."""
  42     attr_class = EbookFieldFile
  43
  44     def __init__(self, format_name, *args, **kwargs):
  45         super(EbookField, self).__init__(*args, **kwargs)
  46         self.format_name = format_name
  47
  48     def deconstruct(self):
  49         name, path, args, kwargs = super(EbookField, self).deconstruct()
  50         args.insert(0, self.format_name)
  51         return name, path, args, kwargs
  52
  53     @property
  54     def builder(self):
  55         """Finds a celery task suitable for the format of the field."""
  56         return BuildEbook.for_format(self.format_name)
  57
  58     def contribute_to_class(self, cls, name):
  59         super(EbookField, self).contribute_to_class(cls, name)
  60
  61         def has(model_instance):
  62             return bool(getattr(model_instance, self.attname, None))
  63         has.__doc__ = None
  64         has.__name__ = str("has_%s" % self.attname)
  65         has.short_description = self.name
  66         has.boolean = True
  67         setattr(cls, 'has_%s' % self.attname, has)
  68
  69
  70 class BuildEbook(Task):
  71     formats = {}
  72
  73     @classmethod
  74     def register(cls, format_name):
  75         """A decorator for registering subclasses for particular formats."""
  76         def wrapper(builder):
  77             cls.formats[format_name] = builder
  78             return builder
  79         return wrapper
  80
  81     @classmethod
  82     def for_format(cls, format_name):
  83         """Returns a celery task suitable for specified format."""
  84         return cls.formats.get(format_name, BuildEbookTask)
  85
  86     @staticmethod
  87     def transform(wldoc, fieldfile):
  88         """Transforms an librarian.WLDocument into an librarian.OutputFile.
  89
  90         By default, it just calls relevant wldoc.as_??? method.
  91
  92         """
  93         return getattr(wldoc, "as_%s" % fieldfile.field.format_name)()
  94
  95     def run(self, obj, field_name):
  96         """Just run `build` on FieldFile, can't pass it directly to Celery."""
  97         task_logger.info("%s -> %s" % (obj.slug, field_name))
  98         ret = self.build(getattr(obj, field_name))
  99         obj.flush_includes()
 100         return ret
 101
 102     def set_file_permissions(self, fieldfile):
 103         if fieldfile.instance.preview:
 104             fieldfile.set_readable(False)
 105
 106     def build(self, fieldfile):
 107         book = fieldfile.instance
 108         out = self.transform(book.wldocument(), fieldfile)
 109         fieldfile.save(None, File(open(out.get_filename(), 'rb')), save=False)
 110         self.set_file_permissions(fieldfile)
 111         if book.pk is not None:
 112             books = type(book).objects.filter(pk=book.pk)
 113             books.update(**{
 114                 fieldfile.field.attname: fieldfile
 115             })
 116             for book in books:
 117                 book.save()  # just to trigger post-save
 118         if fieldfile.field.format_name in app_settings.FORMAT_ZIPS:
 119             remove_zip(app_settings.FORMAT_ZIPS[fieldfile.field.format_name])
 120 # Don't decorate BuildEbook, because we want to subclass it.
 121 BuildEbookTask = task(BuildEbook, ignore_result=True)
 122
 123
 124 @BuildEbook.register('txt')
 125 @task(ignore_result=True)
 126 class BuildTxt(BuildEbook):
 127     @staticmethod
 128     def transform(wldoc, fieldfile):
 129         return wldoc.as_text()
 130
 131
 132 @BuildEbook.register('pdf')
 133 @task(ignore_result=True)
 134 class BuildPdf(BuildEbook):
 135     @staticmethod
 136     def transform(wldoc, fieldfile):
 137         return wldoc.as_pdf(morefloats=settings.LIBRARIAN_PDF_MOREFLOATS, cover=True,
 138                             ilustr_path=gallery_path(wldoc.book_info.url.slug), customizations=['notoc'])
 139
 140     def build(self, fieldfile):
 141         BuildEbook.build(self, fieldfile)
 142         clear_cache(fieldfile.instance.slug)
 143
 144
 145 @BuildEbook.register('epub')
 146 @task(ignore_result=True)
 147 class BuildEpub(BuildEbook):
 148     @staticmethod
 149     def transform(wldoc, fieldfile):
 150         return wldoc.as_epub(cover=True, ilustr_path=gallery_path(wldoc.book_info.url.slug))
 151
 152
 153 @BuildEbook.register('mobi')
 154 @task(ignore_result=True)
 155 class BuildMobi(BuildEbook):
 156     @staticmethod
 157     def transform(wldoc, fieldfile):
 158         return wldoc.as_mobi(cover=True, ilustr_path=gallery_path(wldoc.book_info.url.slug))
 159
 160
 161 @BuildEbook.register('html')
 162 @task(ignore_result=True)
 163 class BuildHtml(BuildEbook):
 164     def build(self, fieldfile):
 165         from django.core.files.base import ContentFile
 166         from slugify import slugify
 167         from sortify import sortify
 168         from librarian import html
 169         from catalogue.models import Fragment, Tag
 170
 171         book = fieldfile.instance
 172
 173         html_output = self.transform(book.wldocument(parse_dublincore=False), fieldfile)
 174
 175         # Delete old fragments, create from scratch if necessary.
 176         book.fragments.all().delete()
 177
 178         if html_output:
 179             meta_tags = list(book.tags.filter(
 180                 category__in=('author', 'epoch', 'genre', 'kind')))
 181
 182             lang = book.language
 183             lang = LANGUAGES_3TO2.get(lang, lang)
 184             if lang not in [ln[0] for ln in settings.LANGUAGES]:
 185                 lang = None
 186
 187             fieldfile.save(None, ContentFile(html_output.get_bytes()), save=False)
 188             self.set_file_permissions(fieldfile)
 189             type(book).objects.filter(pk=book.pk).update(**{
 190                 fieldfile.field.attname: fieldfile
 191             })
 192
 193             # Extract fragments
 194             closed_fragments, open_fragments = html.extract_fragments(fieldfile.path)
 195             for fragment in closed_fragments.values():
 196                 try:
 197                     theme_names = [s.strip() for s in fragment.themes.split(',')]
 198                 except AttributeError:
 199                     continue
 200                 themes = []
 201                 for theme_name in theme_names:
 202                     if not theme_name:
 203                         continue
 204                     if lang == settings.LANGUAGE_CODE:
 205                         # Allow creating themes if book in default language.
 206                         tag, created = Tag.objects.get_or_create(
 207                                             slug=slugify(theme_name),
 208                                             category='theme')
 209                         if created:
 210                             tag.name = theme_name
 211                             setattr(tag, "name_%s" % lang, theme_name)
 212                             tag.sort_key = sortify(theme_name.lower())
 213                             tag.for_books = True
 214                             tag.save()
 215                         themes.append(tag)
 216                     elif lang is not None:
 217                         # Don't create unknown themes in non-default languages.
 218                         try:
 219                             tag = Tag.objects.get(category='theme', **{"name_%s" % lang: theme_name})
 220                         except Tag.DoesNotExist:
 221                             pass
 222                         else:
 223                             themes.append(tag)
 224                 if not themes:
 225                     continue
 226
 227                 text = fragment.to_string()
 228                 short_text = truncate_html_words(text, 15)
 229                 if text == short_text:
 230                     short_text = ''
 231                 new_fragment = Fragment.objects.create(anchor=fragment.id, book=book, text=text, short_text=short_text)
 232
 233                 new_fragment.save()
 234                 new_fragment.tags = set(meta_tags + themes)
 235                 for theme in themes:
 236                     if not theme.for_books:
 237                         theme.for_books = True
 238                         theme.save()
 239             book.html_built.send(sender=type(self), instance=book)
 240             return True
 241         return False
 242
 243     @staticmethod
 244     def transform(wldoc, fieldfile):
 245         # ugly, but we can't use wldoc.book_info here
 246         from librarian import DCNS
 247         url_elem = wldoc.edoc.getroot().find('.//' + DCNS('identifier.url'))
 248         if url_elem is None:
 249             gallery = ''
 250         else:
 251             gallery = gallery_url(slug=url_elem.text.rsplit('/', 1)[1])
 252         return wldoc.as_html(options={'gallery': "'%s'" % gallery})
 253
 254
 255 class BuildCover(BuildEbook):
 256     def set_file_permissions(self, fieldfile):
 257         pass
 258
 259
 260 @BuildEbook.register('cover_thumb')
 261 @task(ignore_result=True)
 262 class BuildCoverThumb(BuildCover):
 263     @classmethod
 264     def transform(cls, wldoc, fieldfile):
 265         from librarian.cover import WLCover
 266         return WLCover(wldoc.book_info, height=193).output_file()
 267
 268
 269 @BuildEbook.register('cover_api_thumb')
 270 @task(ignore_result=True)
 271 class BuildCoverApiThumb(BuildCover):
 272     @classmethod
 273     def transform(cls, wldoc, fieldfile):
 274         from librarian.cover import WLNoBoxCover
 275         return WLNoBoxCover(wldoc.book_info, height=500).output_file()
 276
 277
 278 @BuildEbook.register('simple_cover')
 279 @task(ignore_result=True)
 280 class BuildSimpleCover(BuildCover):
 281     @classmethod
 282     def transform(cls, wldoc, fieldfile):
 283         from librarian.cover import WLNoBoxCover
 284         return WLNoBoxCover(wldoc.book_info, height=1000).output_file()
 285
 286
 287 # not used, but needed for migrations
 288 class OverwritingFieldFile(FieldFile):
 289     """
 290         Deletes the old file before saving the new one.
 291     """
 292
 293     def save(self, name, content, *args, **kwargs):
 294         leave = kwargs.pop('leave', None)
 295         # delete if there's a file already and there's a new one coming
 296         if not leave and self and (not hasattr(content, 'path') or content.path != self.path):
 297             self.delete(save=False)
 298         return super(OverwritingFieldFile, self).save(name, content, *args, **kwargs)
 299
 300
 301 class OverwritingFileField(models.FileField):
 302     attr_class = OverwritingFieldFile
 303
 304
 305 class OverwriteStorage(FileSystemStorage):
 306
 307     def get_available_name(self, name, max_length=None):
 308         self.delete(name)
 309         return name