56de8c6dad63f10bc52fe893684897aa74a53cf7
[wolnelektury.git] / src / catalogue / fields.py
1 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
2 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
3 #
4 import os
5 from django.conf import settings
6 from django.core.files import File
7 from django.db import models
8 from django.db.models.fields.files import FieldFile
9 from django.utils.deconstruct import deconstructible
10 from django.utils.translation import gettext_lazy as _
11 from catalogue.constants import LANGUAGES_3TO2, EBOOK_FORMATS_WITH_CHILDREN, EBOOK_FORMATS_WITHOUT_CHILDREN
12 from catalogue.utils import absolute_url, remove_zip, truncate_html_words, gallery_path, gallery_url
13 from waiter.utils import clear_cache
14
15 ETAG_SCHEDULED_SUFFIX = '-scheduled'
16 EBOOK_BUILD_PRIORITY = 0
17 EBOOK_REBUILD_PRIORITY = 9
18
19
20 @deconstructible
21 class UploadToPath(object):
22     def __init__(self, path):
23         self.path = path
24
25     def __call__(self, instance, filename):
26         return self.path % instance.slug
27
28     def __eq__(self, other):
29         return isinstance(other, type(self)) and other.path == self.path
30
31
32 class EbookFieldFile(FieldFile):
33     """Represents contents of an ebook file field."""
34
35     def build(self):
36         """Build the ebook immediately."""
37         etag = self.field.get_current_etag()
38         self.field.build(self)
39         self.update_etag(etag)
40         self.instance.clear_cache()
41
42     def build_delay(self, priority=EBOOK_BUILD_PRIORITY):
43         """Builds the ebook in a delayed task."""
44         from .tasks import build_field
45
46         self.update_etag(
47             "".join([self.field.get_current_etag(), ETAG_SCHEDULED_SUFFIX])
48         )
49         return build_field.apply_async(
50             [self.instance.pk, self.field.attname],
51             priority=priority
52         )
53
54     def set_readable(self, readable):
55         import os
56         permissions = 0o644 if readable else 0o600
57         os.chmod(self.path, permissions)
58
59     def update_etag(self, etag):
60         setattr(self.instance, self.field.etag_field_name, etag)
61         if self.instance.pk:
62             self.instance.save(update_fields=[self.field.etag_field_name])
63
64
65 class EbookField(models.FileField):
66     """Represents an ebook file field, attachable to a model."""
67     attr_class = EbookFieldFile
68     ext = None
69     librarian2_api = False
70     ZIP = None
71
72     def __init__(self, verbose_name=None, with_etag=True, etag_field_name=None, **kwargs):
73         kwargs.setdefault('verbose_name', verbose_name)
74         self.with_etag = with_etag
75         self.etag_field_name = etag_field_name
76         kwargs.setdefault('max_length', 255)
77         kwargs.setdefault('blank', True)
78         kwargs.setdefault('default', '')
79         kwargs.setdefault('upload_to', self.get_upload_to(self.ext))
80
81         super().__init__(**kwargs)
82
83     def deconstruct(self):
84         name, path, args, kwargs = super().deconstruct()
85         if kwargs.get('max_length') == 255:
86             del kwargs['max_length']
87         if kwargs.get('blank') is True:
88             del kwargs['blank']
89         if kwargs.get('default') == '':
90             del kwargs['default']
91         if self.get_upload_to(self.ext) == kwargs.get('upload_to'):
92             del kwargs['upload_to']
93         # with_etag creates a second field, which then deconstructs to manage
94         # its own migrations. So for migrations, etag_field_name is explicitly
95         # set to avoid double creation of the etag field.
96         if self.with_etag:
97             kwargs['etag_field_name'] = self.etag_field_name
98         else:
99             kwargs['with_etag'] = self.with_etag
100
101         return name, path, args, kwargs
102
103     @classmethod
104     def get_upload_to(cls, directory):
105         directory = getattr(cls, 'directory', cls.ext)
106         upload_template = f'book/{directory}/%s.{cls.ext}'
107         return UploadToPath(upload_template)
108
109     def contribute_to_class(self, cls, name):
110         super(EbookField, self).contribute_to_class(cls, name)
111
112         if self.with_etag and not self.etag_field_name:
113             self.etag_field_name = f'{name}_etag'
114             self.etag_field = models.CharField(max_length=255, editable=False, default='', db_index=True)
115             self.etag_field.contribute_to_class(cls, f'{name}_etag')
116
117         def has(model_instance):
118             return bool(getattr(model_instance, self.attname, None))
119         has.__doc__ = None
120         has.__name__ = str("has_%s" % self.attname)
121         has.short_description = self.name
122         has.boolean = True
123
124         setattr(cls, 'has_%s' % self.attname, has)
125
126     def get_current_etag(self):
127         import pkg_resources
128         librarian_version = pkg_resources.get_distribution("librarian").version
129         return librarian_version
130
131     def schedule_stale(self, queryset=None):
132         """Schedule building this format for all the books where etag is stale."""
133         # If there is not ETag field, bail. That's true for xml file field.
134         if not self.with_etag:
135             return
136
137         etag = self.get_current_etag()
138         if queryset is None:
139             queryset = self.model.objects.all()
140
141         if self.format_name in EBOOK_FORMATS_WITHOUT_CHILDREN + ['html']:
142             queryset = queryset.filter(children=None)
143
144         queryset = queryset.exclude(**{
145             f'{self.etag_field_name}__in': [
146                 etag, f'{etag}{ETAG_SCHEDULED_SUFFIX}'
147             ]
148         })
149         for obj in queryset:
150             fieldfile = getattr(obj, self.attname)
151             priority = EBOOK_REBUILD_PRIORITY if fieldfile else EBOOK_BUILD_PRIORITY
152             fieldfile.build_delay(priority=priority)
153
154     @classmethod
155     def schedule_all_stale(cls, model):
156         """Schedules all stale ebooks of all formats to rebuild."""
157         for field in model._meta.fields:
158             if isinstance(field, cls):
159                 field.schedule_stale()
160
161     @staticmethod
162     def transform(wldoc):
163         """Transforms an librarian.WLDocument into an librarian.OutputFile.
164         """
165         raise NotImplemented()
166
167     def set_file_permissions(self, fieldfile):
168         if fieldfile.instance.preview:
169             fieldfile.set_readable(False)
170
171     def build(self, fieldfile):
172         book = fieldfile.instance
173         out = self.transform(
174             book.wldocument2() if self.librarian2_api else book.wldocument(),
175         )
176         fieldfile.save(None, File(open(out.get_filename(), 'rb')), save=False)
177         self.set_file_permissions(fieldfile)
178         if book.pk is not None:
179             book.save(update_fields=[self.attname])
180         if self.ZIP:
181             remove_zip(self.ZIP)
182
183
184 class XmlField(EbookField):
185     ext = 'xml'
186
187     def build(self, fieldfile):
188         pass
189
190
191 class TxtField(EbookField):
192     ext = 'txt'
193
194     @staticmethod
195     def transform(wldoc):
196         return wldoc.as_text()
197
198
199 class Fb2Field(EbookField):
200     ext = 'fb2'
201     ZIP = 'wolnelektury_pl_fb2'
202
203     @staticmethod
204     def transform(wldoc):
205         return wldoc.as_fb2()
206
207
208 class PdfField(EbookField):
209     ext = 'pdf'
210     ZIP = 'wolnelektury_pl_pdf'
211
212     @staticmethod
213     def transform(wldoc):
214         return wldoc.as_pdf(
215             morefloats=settings.LIBRARIAN_PDF_MOREFLOATS, cover=True,
216             base_url=absolute_url(gallery_url(wldoc.book_info.url.slug)), customizations=['notoc'])
217
218     def build(self, fieldfile):
219         super().build(fieldfile)
220         clear_cache(fieldfile.instance.slug)
221
222
223 class EpubField(EbookField):
224     ext = 'epub'
225     librarian2_api = True
226     ZIP = 'wolnelektury_pl_epub'
227
228     @staticmethod
229     def transform(wldoc):
230         from librarian.builders import EpubBuilder
231         return EpubBuilder(
232                 base_url='file://' + os.path.abspath(gallery_path(wldoc.meta.url.slug)) + '/',
233                 fundraising=settings.EPUB_FUNDRAISING
234             ).build(wldoc)
235
236
237 class MobiField(EbookField):
238     ext = 'mobi'
239     librarian2_api = True
240     ZIP = 'wolnelektury_pl_mobi'
241
242     @staticmethod
243     def transform(wldoc):
244         from librarian.builders import MobiBuilder
245         return MobiBuilder(
246                 base_url='file://' + os.path.abspath(gallery_path(wldoc.meta.url.slug)) + '/',
247                 fundraising=settings.EPUB_FUNDRAISING
248             ).build(wldoc)
249
250
251 class HtmlField(EbookField):
252     ext = 'html'
253
254     def build(self, fieldfile):
255         from django.core.files.base import ContentFile
256         from slugify import slugify
257         from sortify import sortify
258         from librarian import html
259         from catalogue.models import Fragment, Tag
260
261         book = fieldfile.instance
262
263         html_output = self.transform(book.wldocument(parse_dublincore=False))
264
265         # Delete old fragments, create from scratch if necessary.
266         book.fragments.all().delete()
267
268         if html_output:
269             meta_tags = list(book.tags.filter(
270                 category__in=('author', 'epoch', 'genre', 'kind')))
271
272             lang = book.language
273             lang = LANGUAGES_3TO2.get(lang, lang)
274             if lang not in [ln[0] for ln in settings.LANGUAGES]:
275                 lang = None
276
277             fieldfile.save(None, ContentFile(html_output.get_bytes()), save=False)
278             self.set_file_permissions(fieldfile)
279             type(book).objects.filter(pk=book.pk).update(**{
280                 fieldfile.field.attname: fieldfile
281             })
282
283             # Extract fragments
284             closed_fragments, open_fragments = html.extract_fragments(fieldfile.path)
285             for fragment in closed_fragments.values():
286                 try:
287                     theme_names = [s.strip() for s in fragment.themes.split(',')]
288                 except AttributeError:
289                     continue
290                 themes = []
291                 for theme_name in theme_names:
292                     if not theme_name:
293                         continue
294                     if lang == settings.LANGUAGE_CODE:
295                         # Allow creating themes if book in default language.
296                         tag, created = Tag.objects.get_or_create(
297                             slug=slugify(theme_name),
298                             category='theme'
299                         )
300                         if created:
301                             tag.name = theme_name
302                             setattr(tag, "name_%s" % lang, theme_name)
303                             tag.sort_key = sortify(theme_name.lower())
304                             tag.for_books = True
305                             tag.save()
306                         themes.append(tag)
307                     elif lang is not None:
308                         # Don't create unknown themes in non-default languages.
309                         try:
310                             tag = Tag.objects.get(
311                                 category='theme',
312                                 **{"name_%s" % lang: theme_name}
313                             )
314                         except Tag.DoesNotExist:
315                             pass
316                         else:
317                             themes.append(tag)
318                 if not themes:
319                     continue
320
321                 text = fragment.to_string()
322                 short_text = truncate_html_words(text, 15)
323                 if text == short_text:
324                     short_text = ''
325                 new_fragment = Fragment.objects.create(
326                     anchor=fragment.id,
327                     book=book,
328                     text=text,
329                     short_text=short_text
330                 )
331
332                 new_fragment.save()
333                 new_fragment.tags = set(meta_tags + themes)
334                 for theme in themes:
335                     if not theme.for_books:
336                         theme.for_books = True
337                         theme.save()
338             book.html_built.send(sender=type(self), instance=book)
339             return True
340         return False
341
342     @staticmethod
343     def transform(wldoc):
344         # ugly, but we can't use wldoc.book_info here
345         from librarian import DCNS
346         url_elem = wldoc.edoc.getroot().find('.//' + DCNS('identifier.url'))
347         if url_elem is None:
348             gal_url = ''
349             gal_path = ''
350         else:
351             slug = url_elem.text.rstrip('/').rsplit('/', 1)[1]
352             gal_url = gallery_url(slug=slug)
353             gal_path = gallery_path(slug=slug)
354         return wldoc.as_html(gallery_path=gal_path, gallery_url=gal_url, base_url=absolute_url(gal_url))
355
356
357 class CoverField(EbookField):
358     ext = 'jpg'
359     directory = 'cover'
360
361     @staticmethod
362     def transform(wldoc):
363         return wldoc.as_cover()
364
365     def set_file_permissions(self, fieldfile):
366         pass
367
368
369 class CoverCleanField(CoverField):
370     directory = 'cover_clean'
371
372     @staticmethod
373     def transform(wldoc):
374         if wldoc.book_info.cover_box_position == 'none':
375             from librarian.cover import WLCover
376             return WLCover(wldoc.book_info, width=240).output_file()
377         from librarian.covers.marquise import MarquiseCover
378         return MarquiseCover(wldoc.book_info, width=240).output_file()
379
380
381 class CoverThumbField(CoverField):
382     directory = 'cover_thumb'
383
384     @staticmethod
385     def transform(wldoc):
386         from librarian.cover import WLCover
387         return WLCover(wldoc.book_info, height=193).output_file()
388
389
390 class CoverApiThumbField(CoverField):
391     directory = 'cover_api_thumb'
392
393     @staticmethod
394     def transform(wldoc):
395         from librarian.cover import WLNoBoxCover
396         return WLNoBoxCover(wldoc.book_info, height=500).output_file()
397
398
399 class SimpleCoverField(CoverField):
400     directory = 'cover_simple'
401
402     @staticmethod
403     def transform(wldoc):
404         from librarian.cover import WLNoBoxCover
405         return WLNoBoxCover(wldoc.book_info, height=1000).output_file()
406
407
408 class CoverEbookpointField(CoverField):
409     directory = 'cover_ebookpoint'
410
411     @staticmethod
412     def transform(wldoc):
413         from librarian.cover import EbookpointCover
414         return EbookpointCover(wldoc.book_info).output_file()