src/catalogue/fields.py

   1 # -*- coding: utf-8 -*-
   2 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
   3 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
   4 #
   5 from django.conf import settings
   6 from django.core.files import File
   7 from django.core.files.storage import FileSystemStorage
   8 from django.db import models
   9 from django.db.models.fields.files import FieldFile
  10 from catalogue import app_settings
  11 from catalogue.constants import LANGUAGES_3TO2
  12 from catalogue.utils import remove_zip, truncate_html_words, gallery_path, gallery_url
  13 from celery.task import Task, task
  14 from celery.utils.log import get_task_logger
  15 from waiter.utils import clear_cache
  16
  17 task_logger = get_task_logger(__name__)
  18
  19
  20 class EbookFieldFile(FieldFile):
  21     """Represents contents of an ebook file field."""
  22
  23     def build(self):
  24         """Build the ebook immediately."""
  25         return self.field.builder.build(self)
  26
  27     def build_delay(self):
  28         """Builds the ebook in a delayed task."""
  29         return self.field.builder.delay(self.instance, self.field.attname)
  30
  31     def get_url(self):
  32         return self.instance.media_url(self.field.attname.split('_')[0])
  33
  34     def set_readable(self, readable):
  35         import os
  36         permissions = 0o644 if readable else 0o600
  37         os.chmod(self.path, permissions)
  38
  39
  40 class EbookField(models.FileField):
  41     """Represents an ebook file field, attachable to a model."""
  42     attr_class = EbookFieldFile
  43
  44     def __init__(self, format_name, *args, **kwargs):
  45         super(EbookField, self).__init__(*args, **kwargs)
  46         self.format_name = format_name
  47
  48     def deconstruct(self):
  49         name, path, args, kwargs = super(EbookField, self).deconstruct()
  50         args.insert(0, self.format_name)
  51         return name, path, args, kwargs
  52
  53     @property
  54     def builder(self):
  55         """Finds a celery task suitable for the format of the field."""
  56         return BuildEbook.for_format(self.format_name)
  57
  58     def contribute_to_class(self, cls, name):
  59         super(EbookField, self).contribute_to_class(cls, name)
  60
  61         def has(model_instance):
  62             return bool(getattr(model_instance, self.attname, None))
  63         has.__doc__ = None
  64         has.__name__ = str("has_%s" % self.attname)
  65         has.short_description = self.name
  66         has.boolean = True
  67         setattr(cls, 'has_%s' % self.attname, has)
  68
  69
  70 class BuildEbook(Task):
  71     formats = {}
  72
  73     @classmethod
  74     def register(cls, format_name):
  75         """A decorator for registering subclasses for particular formats."""
  76         def wrapper(builder):
  77             cls.formats[format_name] = builder
  78             return builder
  79         return wrapper
  80
  81     @classmethod
  82     def for_format(cls, format_name):
  83         """Returns a celery task suitable for specified format."""
  84         return cls.formats.get(format_name, BuildEbookTask)
  85
  86     @staticmethod
  87     def transform(wldoc, fieldfile):
  88         """Transforms an librarian.WLDocument into an librarian.OutputFile.
  89
  90         By default, it just calls relevant wldoc.as_??? method.
  91
  92         """
  93         return getattr(wldoc, "as_%s" % fieldfile.field.format_name)()
  94
  95     def run(self, obj, field_name):
  96         """Just run `build` on FieldFile, can't pass it directly to Celery."""
  97         task_logger.info("%s -> %s" % (obj.slug, field_name))
  98         ret = self.build(getattr(obj, field_name))
  99         obj.flush_includes()
 100         return ret
 101
 102     def set_file_permissions(self, fieldfile):
 103         if fieldfile.instance.preview:
 104             fieldfile.set_readable(False)
 105
 106     def build(self, fieldfile):
 107         book = fieldfile.instance
 108         out = self.transform(book.wldocument(), fieldfile)
 109         fieldfile.save(None, File(open(out.get_filename())), save=False)
 110         self.set_file_permissions(fieldfile)
 111         if book.pk is not None:
 112             type(book).objects.filter(pk=book.pk).update(**{
 113                 fieldfile.field.attname: fieldfile
 114             })
 115         if fieldfile.field.format_name in app_settings.FORMAT_ZIPS:
 116             remove_zip(app_settings.FORMAT_ZIPS[fieldfile.field.format_name])
 117 # Don't decorate BuildEbook, because we want to subclass it.
 118 BuildEbookTask = task(BuildEbook, ignore_result=True)
 119
 120
 121 @BuildEbook.register('txt')
 122 @task(ignore_result=True)
 123 class BuildTxt(BuildEbook):
 124     @staticmethod
 125     def transform(wldoc, fieldfile):
 126         return wldoc.as_text()
 127
 128
 129 @BuildEbook.register('pdf')
 130 @task(ignore_result=True)
 131 class BuildPdf(BuildEbook):
 132     @staticmethod
 133     def transform(wldoc, fieldfile):
 134         return wldoc.as_pdf(morefloats=settings.LIBRARIAN_PDF_MOREFLOATS, cover=True,
 135                             ilustr_path=gallery_path(wldoc.book_info.url.slug))
 136
 137     def build(self, fieldfile):
 138         BuildEbook.build(self, fieldfile)
 139         clear_cache(fieldfile.instance.slug)
 140
 141
 142 @BuildEbook.register('epub')
 143 @task(ignore_result=True)
 144 class BuildEpub(BuildEbook):
 145     @staticmethod
 146     def transform(wldoc, fieldfile):
 147         return wldoc.as_epub(cover=True, ilustr_path=gallery_path(wldoc.book_info.url.slug))
 148
 149
 150 @BuildEbook.register('mobi')
 151 @task(ignore_result=True)
 152 class BuildMobi(BuildEbook):
 153     @staticmethod
 154     def transform(wldoc, fieldfile):
 155         return wldoc.as_mobi(cover=True, ilustr_path=gallery_path(wldoc.book_info.url.slug))
 156
 157
 158 @BuildEbook.register('html')
 159 @task(ignore_result=True)
 160 class BuildHtml(BuildEbook):
 161     def build(self, fieldfile):
 162         from django.core.files.base import ContentFile
 163         from fnpdjango.utils.text.slughifi import slughifi
 164         from sortify import sortify
 165         from librarian import html
 166         from catalogue.models import Fragment, Tag
 167
 168         book = fieldfile.instance
 169
 170         html_output = self.transform(book.wldocument(parse_dublincore=False), fieldfile)
 171
 172         # Delete old fragments, create from scratch if necessary.
 173         book.fragments.all().delete()
 174
 175         if html_output:
 176             meta_tags = list(book.tags.filter(
 177                 category__in=('author', 'epoch', 'genre', 'kind')))
 178
 179             lang = book.language
 180             lang = LANGUAGES_3TO2.get(lang, lang)
 181             if lang not in [ln[0] for ln in settings.LANGUAGES]:
 182                 lang = None
 183
 184             fieldfile.save(None, ContentFile(html_output.get_string()), save=False)
 185             self.set_file_permissions(fieldfile)
 186             type(book).objects.filter(pk=book.pk).update(**{
 187                 fieldfile.field.attname: fieldfile
 188             })
 189
 190             # Extract fragments
 191             closed_fragments, open_fragments = html.extract_fragments(fieldfile.path)
 192             for fragment in closed_fragments.values():
 193                 try:
 194                     theme_names = [s.strip() for s in fragment.themes.split(',')]
 195                 except AttributeError:
 196                     continue
 197                 themes = []
 198                 for theme_name in theme_names:
 199                     if not theme_name:
 200                         continue
 201                     if lang == settings.LANGUAGE_CODE:
 202                         # Allow creating themes if book in default language.
 203                         tag, created = Tag.objects.get_or_create(
 204                                             slug=slughifi(theme_name),
 205                                             category='theme')
 206                         if created:
 207                             tag.name = theme_name
 208                             setattr(tag, "name_%s" % lang, theme_name)
 209                             tag.sort_key = sortify(theme_name.lower())
 210                             tag.for_books = True
 211                             tag.save()
 212                         themes.append(tag)
 213                     elif lang is not None:
 214                         # Don't create unknown themes in non-default languages.
 215                         try:
 216                             tag = Tag.objects.get(category='theme', **{"name_%s" % lang: theme_name})
 217                         except Tag.DoesNotExist:
 218                             pass
 219                         else:
 220                             themes.append(tag)
 221                 if not themes:
 222                     continue
 223
 224                 text = fragment.to_string()
 225                 short_text = truncate_html_words(text, 15)
 226                 if text == short_text:
 227                     short_text = ''
 228                 new_fragment = Fragment.objects.create(anchor=fragment.id, book=book, text=text, short_text=short_text)
 229
 230                 new_fragment.save()
 231                 new_fragment.tags = set(meta_tags + themes)
 232                 for theme in themes:
 233                     if not theme.for_books:
 234                         theme.for_books = True
 235                         theme.save()
 236             book.html_built.send(sender=type(self), instance=book)
 237             return True
 238         return False
 239
 240     @staticmethod
 241     def transform(wldoc, fieldfile):
 242         # ugly, but we can't use wldoc.book_info here
 243         from librarian import DCNS
 244         url_elem = wldoc.edoc.getroot().find('.//' + DCNS('identifier.url'))
 245         if url_elem is None:
 246             gallery = ''
 247         else:
 248             gallery = gallery_url(slug=url_elem.text.rsplit('/', 1)[1])
 249         return wldoc.as_html(options={'gallery': "'%s'" % gallery})
 250
 251
 252 class BuildCover(BuildEbook):
 253     def set_file_permissions(self, fieldfile):
 254         pass
 255
 256
 257 @BuildEbook.register('cover_thumb')
 258 @task(ignore_result=True)
 259 class BuildCoverThumb(BuildCover):
 260     @classmethod
 261     def transform(cls, wldoc, fieldfile):
 262         from librarian.cover import WLCover
 263         return WLCover(wldoc.book_info, height=193).output_file()
 264
 265
 266 @BuildEbook.register('cover_api_thumb')
 267 @task(ignore_result=True)
 268 class BuildCoverApiThumb(BuildCover):
 269     @classmethod
 270     def transform(cls, wldoc, fieldfile):
 271         from librarian.cover import WLNoBoxCover
 272         return WLNoBoxCover(wldoc.book_info, height=500).output_file()
 273
 274
 275 @BuildEbook.register('simple_cover')
 276 @task(ignore_result=True)
 277 class BuildSimpleCover(BuildCover):
 278     @classmethod
 279     def transform(cls, wldoc, fieldfile):
 280         from librarian.cover import WLNoBoxCover
 281         return WLNoBoxCover(wldoc.book_info, height=1000).output_file()
 282
 283
 284 # not used, but needed for migrations
 285 class OverwritingFieldFile(FieldFile):
 286     """
 287         Deletes the old file before saving the new one.
 288     """
 289
 290     def save(self, name, content, *args, **kwargs):
 291         leave = kwargs.pop('leave', None)
 292         # delete if there's a file already and there's a new one coming
 293         if not leave and self and (not hasattr(content, 'path') or content.path != self.path):
 294             self.delete(save=False)
 295         return super(OverwritingFieldFile, self).save(name, content, *args, **kwargs)
 296
 297
 298 class OverwritingFileField(models.FileField):
 299     attr_class = OverwritingFieldFile
 300
 301
 302 class OverwriteStorage(FileSystemStorage):
 303
 304     def get_available_name(self, name, max_length=None):
 305         self.delete(name)
 306         return name