Almost ready.
[wolnelektury.git] / src / catalogue / fields.py
1 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
2 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
3 #
4 import os
5 from django.conf import settings
6 from django.core.files import File
7 from django.core.files.storage import FileSystemStorage
8 from django.db import models
9 from django.db.models.fields.files import FieldFile
10 from catalogue import app_settings
11 from catalogue.constants import LANGUAGES_3TO2, EBOOK_FORMATS_WITH_CHILDREN, EBOOK_FORMATS_WITHOUT_CHILDREN
12 from catalogue.utils import absolute_url, remove_zip, truncate_html_words, gallery_path, gallery_url
13 from celery.task import Task, task
14 from celery.utils.log import get_task_logger
15 from waiter.utils import clear_cache
16
17 task_logger = get_task_logger(__name__)
18
19 ETAG_SCHEDULED_SUFFIX = '-scheduled'
20 EBOOK_BUILD_PRIORITY = 0
21 EBOOK_REBUILD_PRIORITY = 9
22
23
24 class EbookFieldFile(FieldFile):
25     """Represents contents of an ebook file field."""
26
27     def build(self):
28         """Build the ebook immediately."""
29         return self.field.builder.build(self)
30
31     def build_delay(self, priority=EBOOK_BUILD_PRIORITY):
32         """Builds the ebook in a delayed task."""
33         self.update_etag(
34             "".join([self.field.get_current_etag(), ETAG_SCHEDULED_SUFFIX])
35         )
36         return self.field.builder.apply_async(
37             [self.instance, self.field.attname],
38             priority=priority
39         )
40
41     def get_url(self):
42         return self.instance.media_url(self.field.attname.split('_')[0])
43
44     def set_readable(self, readable):
45         import os
46         permissions = 0o644 if readable else 0o600
47         os.chmod(self.path, permissions)
48
49     def update_etag(self, etag):
50         setattr(self.instance, self.field.etag_field_name, etag)
51         if self.instance.pk:
52             self.instance.save(update_fields=[self.field.etag_field_name])
53
54
55 class EbookField(models.FileField):
56     """Represents an ebook file field, attachable to a model."""
57     attr_class = EbookFieldFile
58     registry = []
59
60     def __init__(self, format_name, *args, **kwargs):
61         super(EbookField, self).__init__(*args, **kwargs)
62         self.format_name = format_name
63
64     def deconstruct(self):
65         name, path, args, kwargs = super(EbookField, self).deconstruct()
66         args.insert(0, self.format_name)
67         return name, path, args, kwargs
68
69     @property
70     def builder(self):
71         """Finds a celery task suitable for the format of the field."""
72         return BuildEbook.for_format(self.format_name)
73
74     def contribute_to_class(self, cls, name):
75         super(EbookField, self).contribute_to_class(cls, name)
76
77         self.etag_field_name = f'{name}_etag'
78
79         def has(model_instance):
80             return bool(getattr(model_instance, self.attname, None))
81         has.__doc__ = None
82         has.__name__ = str("has_%s" % self.attname)
83         has.short_description = self.name
84         has.boolean = True
85
86         self.registry.append(self)
87
88         setattr(cls, 'has_%s' % self.attname, has)
89
90     def get_current_etag(self):
91         import pkg_resources
92         librarian_version = pkg_resources.get_distribution("librarian").version
93         return librarian_version
94
95     def schedule_stale(self, queryset=None):
96         """Schedule building this format for all the books where etag is stale."""
97         # If there is not ETag field, bail. That's true for xml file field.
98         if not hasattr(self.model, f'{self.attname}_etag'):
99             return
100
101         etag = self.get_current_etag()
102         if queryset is None:
103             queryset = self.model.objects.all()
104
105         if self.format_name in EBOOK_FORMATS_WITHOUT_CHILDREN + ['html']:
106             queryset = queryset.filter(children=None)
107
108         queryset = queryset.exclude(**{
109             f'{self.etag_field_name}__in': [
110                 etag, f'{etag}{ETAG_SCHEDULED_SUFFIX}'
111             ]
112         })
113         for obj in queryset:
114             fieldfile = getattr(obj, self.attname)
115             priority = EBOOK_REBUILD_PRIORITY if fieldfile else EBOOK_BUILD_PRIORITY
116             fieldfile.build_delay(priority=priority)
117
118     @classmethod
119     def schedule_all_stale(cls):
120         """Schedules all stale ebooks of all formats to rebuild."""
121         for field in cls.registry:
122             field.schedule_stale()
123
124
125
126 class BuildEbook(Task):
127     librarian2_api = False
128
129     formats = {}
130
131     @classmethod
132     def register(cls, format_name):
133         """A decorator for registering subclasses for particular formats."""
134         def wrapper(builder):
135             cls.formats[format_name] = builder
136             return builder
137         return wrapper
138
139     @classmethod
140     def for_format(cls, format_name):
141         """Returns a celery task suitable for specified format."""
142         return cls.formats.get(format_name, BuildEbookTask)
143
144     @staticmethod
145     def transform(wldoc, fieldfile):
146         """Transforms an librarian.WLDocument into an librarian.OutputFile.
147
148         By default, it just calls relevant wldoc.as_??? method.
149
150         """
151         return getattr(wldoc, "as_%s" % fieldfile.field.format_name)()
152
153     def run(self, obj, field_name):
154         """Just run `build` on FieldFile, can't pass it directly to Celery."""
155         fieldfile = getattr(obj, field_name)
156
157         # Get etag value before actually building the file.
158         etag = fieldfile.field.get_current_etag()
159         task_logger.info("%s -> %s@%s" % (obj.slug, field_name, etag))
160         ret = self.build(getattr(obj, field_name))
161         fieldfile.update_etag(etag)
162         obj.clear_cache()
163         return ret
164
165     def set_file_permissions(self, fieldfile):
166         if fieldfile.instance.preview:
167             fieldfile.set_readable(False)
168
169     def build(self, fieldfile):
170         book = fieldfile.instance
171         out = self.transform(
172             book.wldocument2() if self.librarian2_api else book.wldocument(),
173             fieldfile)
174         fieldfile.save(None, File(open(out.get_filename(), 'rb')), save=False)
175         self.set_file_permissions(fieldfile)
176         if book.pk is not None:
177             book.save(update_fields=[fieldfile.field.attname])
178         if fieldfile.field.format_name in app_settings.FORMAT_ZIPS:
179             remove_zip(app_settings.FORMAT_ZIPS[fieldfile.field.format_name])
180 # Don't decorate BuildEbook, because we want to subclass it.
181 BuildEbookTask = task(BuildEbook, ignore_result=True)
182
183
184 @BuildEbook.register('txt')
185 @task(ignore_result=True)
186 class BuildTxt(BuildEbook):
187     @staticmethod
188     def transform(wldoc, fieldfile):
189         return wldoc.as_text()
190
191
192 @BuildEbook.register('pdf')
193 @task(ignore_result=True)
194 class BuildPdf(BuildEbook):
195     @staticmethod
196     def transform(wldoc, fieldfile):
197         return wldoc.as_pdf(
198             morefloats=settings.LIBRARIAN_PDF_MOREFLOATS, cover=True,
199             base_url=absolute_url(gallery_url(wldoc.book_info.url.slug)), customizations=['notoc'])
200
201     def build(self, fieldfile):
202         BuildEbook.build(self, fieldfile)
203         clear_cache(fieldfile.instance.slug)
204
205
206 @BuildEbook.register('epub')
207 @task(ignore_result=True)
208 class BuildEpub(BuildEbook):
209     librarian2_api = True
210
211     @staticmethod
212     def transform(wldoc, fieldfile):
213         from librarian.builders import EpubBuilder
214         return EpubBuilder(
215                 base_url='file://' + os.path.abspath(gallery_path(wldoc.meta.url.slug)) + '/',
216                 fundraising=settings.EPUB_FUNDRAISING
217             ).build(wldoc)
218
219
220 @BuildEbook.register('mobi')
221 @task(ignore_result=True)
222 class BuildMobi(BuildEbook):
223     librarian2_api = True
224
225     @staticmethod
226     def transform(wldoc, fieldfile):
227         from librarian.builders import MobiBuilder
228         return MobiBuilder(
229                 base_url='file://' + os.path.abspath(gallery_path(wldoc.meta.url.slug)) + '/',
230                 fundraising=settings.EPUB_FUNDRAISING
231             ).build(wldoc)
232
233
234 @BuildEbook.register('html')
235 @task(ignore_result=True)
236 class BuildHtml(BuildEbook):
237     def build(self, fieldfile):
238         from django.core.files.base import ContentFile
239         from slugify import slugify
240         from sortify import sortify
241         from librarian import html
242         from catalogue.models import Fragment, Tag
243
244         book = fieldfile.instance
245
246         html_output = self.transform(book.wldocument(parse_dublincore=False), fieldfile)
247
248         # Delete old fragments, create from scratch if necessary.
249         book.fragments.all().delete()
250
251         if html_output:
252             meta_tags = list(book.tags.filter(
253                 category__in=('author', 'epoch', 'genre', 'kind')))
254
255             lang = book.language
256             lang = LANGUAGES_3TO2.get(lang, lang)
257             if lang not in [ln[0] for ln in settings.LANGUAGES]:
258                 lang = None
259
260             fieldfile.save(None, ContentFile(html_output.get_bytes()), save=False)
261             self.set_file_permissions(fieldfile)
262             type(book).objects.filter(pk=book.pk).update(**{
263                 fieldfile.field.attname: fieldfile
264             })
265
266             # Extract fragments
267             closed_fragments, open_fragments = html.extract_fragments(fieldfile.path)
268             for fragment in closed_fragments.values():
269                 try:
270                     theme_names = [s.strip() for s in fragment.themes.split(',')]
271                 except AttributeError:
272                     continue
273                 themes = []
274                 for theme_name in theme_names:
275                     if not theme_name:
276                         continue
277                     if lang == settings.LANGUAGE_CODE:
278                         # Allow creating themes if book in default language.
279                         tag, created = Tag.objects.get_or_create(
280                             slug=slugify(theme_name),
281                             category='theme'
282                         )
283                         if created:
284                             tag.name = theme_name
285                             setattr(tag, "name_%s" % lang, theme_name)
286                             tag.sort_key = sortify(theme_name.lower())
287                             tag.for_books = True
288                             tag.save()
289                         themes.append(tag)
290                     elif lang is not None:
291                         # Don't create unknown themes in non-default languages.
292                         try:
293                             tag = Tag.objects.get(
294                                 category='theme',
295                                 **{"name_%s" % lang: theme_name}
296                             )
297                         except Tag.DoesNotExist:
298                             pass
299                         else:
300                             themes.append(tag)
301                 if not themes:
302                     continue
303
304                 text = fragment.to_string()
305                 short_text = truncate_html_words(text, 15)
306                 if text == short_text:
307                     short_text = ''
308                 new_fragment = Fragment.objects.create(
309                     anchor=fragment.id,
310                     book=book,
311                     text=text,
312                     short_text=short_text
313                 )
314
315                 new_fragment.save()
316                 new_fragment.tags = set(meta_tags + themes)
317                 for theme in themes:
318                     if not theme.for_books:
319                         theme.for_books = True
320                         theme.save()
321             book.html_built.send(sender=type(self), instance=book)
322             return True
323         return False
324
325     @staticmethod
326     def transform(wldoc, fieldfile):
327         # ugly, but we can't use wldoc.book_info here
328         from librarian import DCNS
329         url_elem = wldoc.edoc.getroot().find('.//' + DCNS('identifier.url'))
330         if url_elem is None:
331             gal_url = ''
332             gal_path = ''
333         else:
334             slug = url_elem.text.rstrip('/').rsplit('/', 1)[1]
335             gal_url = gallery_url(slug=slug)
336             gal_path = gallery_path(slug=slug)
337         return wldoc.as_html(gallery_path=gal_path, gallery_url=gal_url, base_url=absolute_url(gal_url))
338
339
340 class BuildCover(BuildEbook):
341     def set_file_permissions(self, fieldfile):
342         pass
343
344
345 @BuildEbook.register('cover_clean')
346 @task(ignore_result=True)
347 class BuildCoverClean(BuildCover):
348     @classmethod
349     def transform(cls, wldoc, fieldfile):
350         from librarian.cover import WLCover
351         return WLCover(wldoc.book_info, width=240).output_file()
352
353
354 @BuildEbook.register('cover_thumb')
355 @task(ignore_result=True)
356 class BuildCoverThumb(BuildCover):
357     @classmethod
358     def transform(cls, wldoc, fieldfile):
359         from librarian.cover import WLCover
360         return WLCover(wldoc.book_info, height=193).output_file()
361
362
363 @BuildEbook.register('cover_api_thumb')
364 @task(ignore_result=True)
365 class BuildCoverApiThumb(BuildCover):
366     @classmethod
367     def transform(cls, wldoc, fieldfile):
368         from librarian.cover import WLNoBoxCover
369         return WLNoBoxCover(wldoc.book_info, height=500).output_file()
370
371
372 @BuildEbook.register('simple_cover')
373 @task(ignore_result=True)
374 class BuildSimpleCover(BuildCover):
375     @classmethod
376     def transform(cls, wldoc, fieldfile):
377         from librarian.cover import WLNoBoxCover
378         return WLNoBoxCover(wldoc.book_info, height=1000).output_file()
379
380
381 @BuildEbook.register('cover_ebookpoint')
382 @task(ignore_result=True)
383 class BuildCoverEbookpoint(BuildCover):
384     @classmethod
385     def transform(cls, wldoc, fieldfile):
386         from librarian.cover import EbookpointCover
387         return EbookpointCover(wldoc.book_info).output_file()
388
389
390 # not used, but needed for migrations
391 class OverwritingFieldFile(FieldFile):
392     """
393         Deletes the old file before saving the new one.
394     """
395
396     def save(self, name, content, *args, **kwargs):
397         leave = kwargs.pop('leave', None)
398         # delete if there's a file already and there's a new one coming
399         if not leave and self and (not hasattr(content, 'path') or content.path != self.path):
400             self.delete(save=False)
401         return super(OverwritingFieldFile, self).save(name, content, *args, **kwargs)
402
403
404 class OverwritingFileField(models.FileField):
405     attr_class = OverwritingFieldFile
406
407
408 class OverwriteStorage(FileSystemStorage):
409
410     def get_available_name(self, name, max_length=None):
411         self.delete(name)
412         return name