More Py3 compatibility fixes.
[wolnelektury.git] / src / catalogue / fields.py
1 # -*- coding: utf-8 -*-
2 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
3 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
4 #
5 from django.conf import settings
6 from django.core.files import File
7 from django.core.files.storage import FileSystemStorage
8 from django.db import models
9 from django.db.models.fields.files import FieldFile
10 from catalogue import app_settings
11 from catalogue.constants import LANGUAGES_3TO2
12 from catalogue.utils import remove_zip, truncate_html_words, gallery_path, gallery_url
13 from celery.task import Task, task
14 from celery.utils.log import get_task_logger
15 from waiter.utils import clear_cache
16
17 task_logger = get_task_logger(__name__)
18
19
20 class EbookFieldFile(FieldFile):
21     """Represents contents of an ebook file field."""
22
23     def build(self):
24         """Build the ebook immediately."""
25         return self.field.builder.build(self)
26
27     def build_delay(self):
28         """Builds the ebook in a delayed task."""
29         return self.field.builder.delay(self.instance, self.field.attname)
30
31     def get_url(self):
32         return self.instance.media_url(self.field.attname.split('_')[0])
33
34     def set_readable(self, readable):
35         import os
36         permissions = 0o644 if readable else 0o600
37         os.chmod(self.path, permissions)
38
39
40 class EbookField(models.FileField):
41     """Represents an ebook file field, attachable to a model."""
42     attr_class = EbookFieldFile
43
44     def __init__(self, format_name, *args, **kwargs):
45         super(EbookField, self).__init__(*args, **kwargs)
46         self.format_name = format_name
47
48     def deconstruct(self):
49         name, path, args, kwargs = super(EbookField, self).deconstruct()
50         args.insert(0, self.format_name)
51         return name, path, args, kwargs
52
53     @property
54     def builder(self):
55         """Finds a celery task suitable for the format of the field."""
56         return BuildEbook.for_format(self.format_name)
57
58     def contribute_to_class(self, cls, name):
59         super(EbookField, self).contribute_to_class(cls, name)
60
61         def has(model_instance):
62             return bool(getattr(model_instance, self.attname, None))
63         has.__doc__ = None
64         has.__name__ = str("has_%s" % self.attname)
65         has.short_description = self.name
66         has.boolean = True
67         setattr(cls, 'has_%s' % self.attname, has)
68
69
70 class BuildEbook(Task):
71     formats = {}
72
73     @classmethod
74     def register(cls, format_name):
75         """A decorator for registering subclasses for particular formats."""
76         def wrapper(builder):
77             cls.formats[format_name] = builder
78             return builder
79         return wrapper
80
81     @classmethod
82     def for_format(cls, format_name):
83         """Returns a celery task suitable for specified format."""
84         return cls.formats.get(format_name, BuildEbookTask)
85
86     @staticmethod
87     def transform(wldoc, fieldfile):
88         """Transforms an librarian.WLDocument into an librarian.OutputFile.
89
90         By default, it just calls relevant wldoc.as_??? method.
91
92         """
93         return getattr(wldoc, "as_%s" % fieldfile.field.format_name)()
94
95     def run(self, obj, field_name):
96         """Just run `build` on FieldFile, can't pass it directly to Celery."""
97         task_logger.info("%s -> %s" % (obj.slug, field_name))
98         ret = self.build(getattr(obj, field_name))
99         obj.flush_includes()
100         return ret
101
102     def set_file_permissions(self, fieldfile):
103         if fieldfile.instance.preview:
104             fieldfile.set_readable(False)
105
106     def build(self, fieldfile):
107         book = fieldfile.instance
108         out = self.transform(book.wldocument(), fieldfile)
109         fieldfile.save(None, File(open(out.get_filename(), 'rb')), save=False)
110         self.set_file_permissions(fieldfile)
111         if book.pk is not None:
112             books = type(book).objects.filter(pk=book.pk)
113             books.update(**{
114                 fieldfile.field.attname: fieldfile
115             })
116             for book in books:
117                 book.save()  # just to trigger post-save
118         if fieldfile.field.format_name in app_settings.FORMAT_ZIPS:
119             remove_zip(app_settings.FORMAT_ZIPS[fieldfile.field.format_name])
120 # Don't decorate BuildEbook, because we want to subclass it.
121 BuildEbookTask = task(BuildEbook, ignore_result=True)
122
123
124 @BuildEbook.register('txt')
125 @task(ignore_result=True)
126 class BuildTxt(BuildEbook):
127     @staticmethod
128     def transform(wldoc, fieldfile):
129         return wldoc.as_text()
130
131
132 @BuildEbook.register('pdf')
133 @task(ignore_result=True)
134 class BuildPdf(BuildEbook):
135     @staticmethod
136     def transform(wldoc, fieldfile):
137         return wldoc.as_pdf(morefloats=settings.LIBRARIAN_PDF_MOREFLOATS, cover=True,
138                             ilustr_path=gallery_path(wldoc.book_info.url.slug), customizations=['notoc'])
139
140     def build(self, fieldfile):
141         BuildEbook.build(self, fieldfile)
142         clear_cache(fieldfile.instance.slug)
143
144
145 @BuildEbook.register('epub')
146 @task(ignore_result=True)
147 class BuildEpub(BuildEbook):
148     @staticmethod
149     def transform(wldoc, fieldfile):
150         return wldoc.as_epub(cover=True, ilustr_path=gallery_path(wldoc.book_info.url.slug))
151
152
153 @BuildEbook.register('mobi')
154 @task(ignore_result=True)
155 class BuildMobi(BuildEbook):
156     @staticmethod
157     def transform(wldoc, fieldfile):
158         return wldoc.as_mobi(cover=True, ilustr_path=gallery_path(wldoc.book_info.url.slug))
159
160
161 @BuildEbook.register('html')
162 @task(ignore_result=True)
163 class BuildHtml(BuildEbook):
164     def build(self, fieldfile):
165         from django.core.files.base import ContentFile
166         from slugify import slugify
167         from sortify import sortify
168         from librarian import html
169         from catalogue.models import Fragment, Tag
170
171         book = fieldfile.instance
172
173         html_output = self.transform(book.wldocument(parse_dublincore=False), fieldfile)
174
175         # Delete old fragments, create from scratch if necessary.
176         book.fragments.all().delete()
177
178         if html_output:
179             meta_tags = list(book.tags.filter(
180                 category__in=('author', 'epoch', 'genre', 'kind')))
181
182             lang = book.language
183             lang = LANGUAGES_3TO2.get(lang, lang)
184             if lang not in [ln[0] for ln in settings.LANGUAGES]:
185                 lang = None
186
187             fieldfile.save(None, ContentFile(html_output.get_bytes()), save=False)
188             self.set_file_permissions(fieldfile)
189             type(book).objects.filter(pk=book.pk).update(**{
190                 fieldfile.field.attname: fieldfile
191             })
192
193             # Extract fragments
194             closed_fragments, open_fragments = html.extract_fragments(fieldfile.path)
195             for fragment in closed_fragments.values():
196                 try:
197                     theme_names = [s.strip() for s in fragment.themes.split(',')]
198                 except AttributeError:
199                     continue
200                 themes = []
201                 for theme_name in theme_names:
202                     if not theme_name:
203                         continue
204                     if lang == settings.LANGUAGE_CODE:
205                         # Allow creating themes if book in default language.
206                         tag, created = Tag.objects.get_or_create(
207                                             slug=slugify(theme_name),
208                                             category='theme')
209                         if created:
210                             tag.name = theme_name
211                             setattr(tag, "name_%s" % lang, theme_name)
212                             tag.sort_key = sortify(theme_name.lower())
213                             tag.for_books = True
214                             tag.save()
215                         themes.append(tag)
216                     elif lang is not None:
217                         # Don't create unknown themes in non-default languages.
218                         try:
219                             tag = Tag.objects.get(category='theme', **{"name_%s" % lang: theme_name})
220                         except Tag.DoesNotExist:
221                             pass
222                         else:
223                             themes.append(tag)
224                 if not themes:
225                     continue
226
227                 text = fragment.to_string()
228                 short_text = truncate_html_words(text, 15)
229                 if text == short_text:
230                     short_text = ''
231                 new_fragment = Fragment.objects.create(anchor=fragment.id, book=book, text=text, short_text=short_text)
232
233                 new_fragment.save()
234                 new_fragment.tags = set(meta_tags + themes)
235                 for theme in themes:
236                     if not theme.for_books:
237                         theme.for_books = True
238                         theme.save()
239             book.html_built.send(sender=type(self), instance=book)
240             return True
241         return False
242
243     @staticmethod
244     def transform(wldoc, fieldfile):
245         # ugly, but we can't use wldoc.book_info here
246         from librarian import DCNS
247         url_elem = wldoc.edoc.getroot().find('.//' + DCNS('identifier.url'))
248         if url_elem is None:
249             gallery = ''
250         else:
251             gallery = gallery_url(slug=url_elem.text.rsplit('/', 1)[1])
252         return wldoc.as_html(options={'gallery': "'%s'" % gallery})
253
254
255 class BuildCover(BuildEbook):
256     def set_file_permissions(self, fieldfile):
257         pass
258
259
260 @BuildEbook.register('cover_thumb')
261 @task(ignore_result=True)
262 class BuildCoverThumb(BuildCover):
263     @classmethod
264     def transform(cls, wldoc, fieldfile):
265         from librarian.cover import WLCover
266         return WLCover(wldoc.book_info, height=193).output_file()
267
268
269 @BuildEbook.register('cover_api_thumb')
270 @task(ignore_result=True)
271 class BuildCoverApiThumb(BuildCover):
272     @classmethod
273     def transform(cls, wldoc, fieldfile):
274         from librarian.cover import WLNoBoxCover
275         return WLNoBoxCover(wldoc.book_info, height=500).output_file()
276
277
278 @BuildEbook.register('simple_cover')
279 @task(ignore_result=True)
280 class BuildSimpleCover(BuildCover):
281     @classmethod
282     def transform(cls, wldoc, fieldfile):
283         from librarian.cover import WLNoBoxCover
284         return WLNoBoxCover(wldoc.book_info, height=1000).output_file()
285
286
287 # not used, but needed for migrations
288 class OverwritingFieldFile(FieldFile):
289     """
290         Deletes the old file before saving the new one.
291     """
292
293     def save(self, name, content, *args, **kwargs):
294         leave = kwargs.pop('leave', None)
295         # delete if there's a file already and there's a new one coming
296         if not leave and self and (not hasattr(content, 'path') or content.path != self.path):
297             self.delete(save=False)
298         return super(OverwritingFieldFile, self).save(name, content, *args, **kwargs)
299
300
301 class OverwritingFileField(models.FileField):
302     attr_class = OverwritingFieldFile
303
304
305 class OverwriteStorage(FileSystemStorage):
306
307     def get_available_name(self, name, max_length=None):
308         self.delete(name)
309         return name