bf17d358130a540500b92ac13ed3d30df61ce4cb
[wolnelektury.git] / src / catalogue / fields.py
1 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
2 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
3 #
4 import os
5 import pkg_resources
6 import random
7 from django.apps import apps
8 from django.conf import settings
9 from django.core.files import File
10 from django.db import models
11 from django.db.models.fields.files import FieldFile
12 from django.utils.deconstruct import deconstructible
13 from django.utils.translation import gettext_lazy as _
14 from catalogue.constants import LANGUAGES_3TO2
15 from catalogue.utils import absolute_url, remove_zip, truncate_html_words, gallery_path, gallery_url
16 from waiter.utils import clear_cache
17
18 ETAG_SCHEDULED_SUFFIX = '-scheduled'
19 EBOOK_BUILD_PRIORITY = 0
20 EBOOK_REBUILD_PRIORITY = 9
21
22
23 @deconstructible
24 class UploadToPath(object):
25     def __init__(self, path):
26         self.path = path
27
28     def __call__(self, instance, filename):
29         return self.path % instance.slug
30
31     def __eq__(self, other):
32         return isinstance(other, type(self)) and other.path == self.path
33
34
35 class EbookFieldFile(FieldFile):
36     """Represents contents of an ebook file field."""
37
38     def build(self):
39         """Build the ebook immediately."""
40         etag = self.field.get_current_etag()
41         self.field.build(self)
42         self.update_etag(etag)
43         self.instance.clear_cache()
44
45     def build_delay(self, priority=EBOOK_BUILD_PRIORITY):
46         """Builds the ebook in a delayed task."""
47         from .tasks import build_field
48
49         self.update_etag(
50             "".join([self.field.get_current_etag(), ETAG_SCHEDULED_SUFFIX])
51         )
52         return build_field.apply_async(
53             [self.instance.pk, self.field.attname],
54             priority=priority
55         )
56
57     def set_readable(self, readable):
58         import os
59         permissions = 0o644 if readable else 0o600
60         os.chmod(self.path, permissions)
61
62     def update_etag(self, etag):
63         setattr(self.instance, self.field.etag_field_name, etag)
64         if self.instance.pk:
65             self.instance.save(update_fields=[self.field.etag_field_name])
66
67
68 class EbookField(models.FileField):
69     """Represents an ebook file field, attachable to a model."""
70     attr_class = EbookFieldFile
71     ext = None
72     for_parents = True
73     librarian2_api = False
74     ZIP = None
75
76     def __init__(self, verbose_name=None, with_etag=True, etag_field_name=None, **kwargs):
77         kwargs.setdefault('verbose_name', verbose_name)
78         self.with_etag = with_etag
79         self.etag_field_name = etag_field_name
80         kwargs.setdefault('max_length', 255)
81         kwargs.setdefault('blank', True)
82         kwargs.setdefault('default', '')
83         kwargs.setdefault('upload_to', self.get_upload_to(self.ext))
84
85         super().__init__(**kwargs)
86
87     def deconstruct(self):
88         name, path, args, kwargs = super().deconstruct()
89         if kwargs.get('max_length') == 255:
90             del kwargs['max_length']
91         if kwargs.get('blank') is True:
92             del kwargs['blank']
93         if kwargs.get('default') == '':
94             del kwargs['default']
95         if self.get_upload_to(self.ext) == kwargs.get('upload_to'):
96             del kwargs['upload_to']
97         # with_etag creates a second field, which then deconstructs to manage
98         # its own migrations. So for migrations, etag_field_name is explicitly
99         # set to avoid double creation of the etag field.
100         if self.with_etag:
101             kwargs['etag_field_name'] = self.etag_field_name
102         else:
103             kwargs['with_etag'] = self.with_etag
104
105         return name, path, args, kwargs
106
107     @classmethod
108     def get_upload_to(cls, directory):
109         directory = getattr(cls, 'directory', cls.ext)
110         upload_template = f'book/{directory}/%s.{cls.ext}'
111         return UploadToPath(upload_template)
112
113     def contribute_to_class(self, cls, name):
114         super(EbookField, self).contribute_to_class(cls, name)
115
116         if self.with_etag and not self.etag_field_name:
117             self.etag_field_name = f'{name}_etag'
118             self.etag_field = models.CharField(max_length=255, editable=False, default='', db_index=True)
119             self.etag_field.contribute_to_class(cls, f'{name}_etag')
120
121         def has(model_instance):
122             return bool(getattr(model_instance, self.attname, None))
123         has.__doc__ = None
124         has.__name__ = str("has_%s" % self.attname)
125         has.short_description = self.name
126         has.boolean = True
127
128         setattr(cls, 'has_%s' % self.attname, has)
129
130     def get_current_etag(self):
131         MediaInsertSet = apps.get_model('annoy', 'MediaInsertSet')
132         librarian_version = pkg_resources.get_distribution("librarian").version
133         etag = librarian_version
134         mis = MediaInsertSet.get_for_format(self.ext)
135         if mis is not None:
136             etag += '_' + mis.etag
137         return etag
138
139     def find_stale(self, limit):
140         """Find some books where this format is stale."""
141         # If there is not ETag field, bail. That's true for xml file field.
142         if not self.with_etag:
143             return []
144
145         etag = self.get_current_etag()
146
147         queryset = self.model.objects.all()
148         if not self.for_parents:
149             queryset = queryset.filter(children=None)
150
151         queryset = queryset.exclude(**{
152             f'{self.etag_field_name}__in': [
153                 etag, f'{etag}{ETAG_SCHEDULED_SUFFIX}'
154            ]
155         })
156
157         queryset = queryset.order_by('?')[:limit]
158         return queryset
159
160     @classmethod
161     def find_all_stale(cls, model, limit):
162         """Schedules all stale ebooks of all formats to rebuild."""
163         found = []
164         for field in model._meta.fields:
165             if isinstance(field, cls):
166                 for instance in field.find_stale(limit):
167                     found.append((
168                         field.name,
169                         instance
170                     ))
171         random.shuffle(found)
172         found = found[:limit]
173         return found
174
175     @staticmethod
176     def transform(wldoc):
177         """Transforms an librarian.WLDocument into an librarian.OutputFile.
178         """
179         raise NotImplemented()
180
181     def set_file_permissions(self, fieldfile):
182         if fieldfile.instance.preview:
183             fieldfile.set_readable(False)
184
185     def build(self, fieldfile):
186         book = fieldfile.instance
187         out = self.transform(
188             book.wldocument2() if self.librarian2_api else book.wldocument(),
189         )
190         with open(out.get_filename(), 'rb') as f:
191             fieldfile.save(None, File(f), save=False)
192         self.set_file_permissions(fieldfile)
193         if book.pk is not None:
194             book.save(update_fields=[self.attname])
195         if self.ZIP:
196             remove_zip(self.ZIP)
197
198
199 class XmlField(EbookField):
200     ext = 'xml'
201
202     def build(self, fieldfile):
203         pass
204
205
206 class TxtField(EbookField):
207     ext = 'txt'
208     for_parents = False
209
210     @staticmethod
211     def transform(wldoc):
212         return wldoc.as_text()
213
214
215 class Fb2Field(EbookField):
216     ext = 'fb2'
217     for_parents = False
218     ZIP = 'wolnelektury_pl_fb2'
219
220     @staticmethod
221     def transform(wldoc):
222         return wldoc.as_fb2()
223
224
225 class PdfField(EbookField):
226     ext = 'pdf'
227     ZIP = 'wolnelektury_pl_pdf'
228
229     @staticmethod
230     def transform(wldoc):
231         return wldoc.as_pdf(
232             morefloats=settings.LIBRARIAN_PDF_MOREFLOATS, cover=True,
233             base_url=absolute_url(gallery_url(wldoc.book_info.url.slug)), customizations=['notoc'])
234
235     def build(self, fieldfile):
236         super().build(fieldfile)
237         clear_cache(fieldfile.instance.slug)
238
239
240 class EpubField(EbookField):
241     ext = 'epub'
242     librarian2_api = True
243     ZIP = 'wolnelektury_pl_epub'
244
245     @staticmethod
246     def transform(wldoc):
247         from librarian.builders import EpubBuilder
248         MediaInsertSet = apps.get_model('annoy', 'MediaInsertSet')
249         return EpubBuilder(
250                 base_url='file://' + os.path.abspath(gallery_path(wldoc.meta.url.slug)) + '/',
251                 fundraising=MediaInsertSet.get_texts_for('epub')
252             ).build(wldoc)
253
254
255 class MobiField(EbookField):
256     ext = 'mobi'
257     librarian2_api = True
258     ZIP = 'wolnelektury_pl_mobi'
259
260     @staticmethod
261     def transform(wldoc):
262         from librarian.builders import MobiBuilder
263         MediaInsertSet = apps.get_model('annoy', 'MediaInsertSet')
264         return MobiBuilder(
265                 base_url='file://' + os.path.abspath(gallery_path(wldoc.meta.url.slug)) + '/',
266                 fundraising=MediaInsertSet.get_texts_for('mobi')
267             ).build(wldoc)
268
269
270 class HtmlField(EbookField):
271     ext = 'html'
272     for_parents = False
273
274     def build(self, fieldfile):
275         from django.core.files.base import ContentFile
276         from slugify import slugify
277         from sortify import sortify
278         from librarian import html
279         from catalogue.models import Fragment, Tag
280
281         book = fieldfile.instance
282
283         html_output = self.transform(book.wldocument(parse_dublincore=False))
284
285         # Delete old fragments, create from scratch if necessary.
286         book.fragments.all().delete()
287
288         if html_output:
289             meta_tags = list(book.tags.filter(
290                 category__in=('author', 'epoch', 'genre', 'kind')))
291
292             lang = book.language
293             lang = LANGUAGES_3TO2.get(lang, lang)
294             if lang not in [ln[0] for ln in settings.LANGUAGES]:
295                 lang = None
296
297             fieldfile.save(None, ContentFile(html_output.get_bytes()), save=False)
298             self.set_file_permissions(fieldfile)
299             type(book).objects.filter(pk=book.pk).update(**{
300                 fieldfile.field.attname: fieldfile
301             })
302
303             # Extract fragments
304             closed_fragments, open_fragments = html.extract_fragments(fieldfile.path)
305             for fragment in closed_fragments.values():
306                 try:
307                     theme_names = [s.strip() for s in fragment.themes.split(',')]
308                 except AttributeError:
309                     continue
310                 themes = []
311                 for theme_name in theme_names:
312                     if not theme_name:
313                         continue
314                     if lang == settings.LANGUAGE_CODE:
315                         # Allow creating themes if book in default language.
316                         tag, created = Tag.objects.get_or_create(
317                             slug=slugify(theme_name),
318                             category='theme'
319                         )
320                         if created:
321                             tag.name = theme_name
322                             setattr(tag, "name_%s" % lang, theme_name)
323                             tag.sort_key = sortify(theme_name.lower())
324                             tag.for_books = True
325                             tag.save()
326                         themes.append(tag)
327                     elif lang is not None:
328                         # Don't create unknown themes in non-default languages.
329                         try:
330                             tag = Tag.objects.get(
331                                 category='theme',
332                                 **{"name_%s" % lang: theme_name}
333                             )
334                         except Tag.DoesNotExist:
335                             pass
336                         else:
337                             themes.append(tag)
338                 if not themes:
339                     continue
340
341                 text = fragment.to_string()
342                 short_text = truncate_html_words(text, 15)
343                 if text == short_text:
344                     short_text = ''
345                 new_fragment = Fragment.objects.create(
346                     anchor=fragment.id,
347                     book=book,
348                     text=text,
349                     short_text=short_text
350                 )
351
352                 new_fragment.save()
353                 new_fragment.tags = set(meta_tags + themes)
354                 for theme in themes:
355                     if not theme.for_books:
356                         theme.for_books = True
357                         theme.save()
358             book.html_built.send(sender=type(self), instance=book)
359             return True
360         return False
361
362     @staticmethod
363     def transform(wldoc):
364         # ugly, but we can't use wldoc.book_info here
365         from librarian import DCNS
366         url_elem = wldoc.edoc.getroot().find('.//' + DCNS('identifier.url'))
367         if url_elem is None:
368             gal_url = ''
369             gal_path = ''
370         else:
371             slug = url_elem.text.rstrip('/').rsplit('/', 1)[1]
372             gal_url = gallery_url(slug=slug)
373             gal_path = gallery_path(slug=slug)
374         return wldoc.as_html(gallery_path=gal_path, gallery_url=gal_url, base_url=absolute_url(gal_url))
375
376
377 class CoverField(EbookField):
378     ext = 'jpg'
379     directory = 'cover'
380
381     @staticmethod
382     def transform(wldoc):
383         return wldoc.as_cover()
384
385     def set_file_permissions(self, fieldfile):
386         pass
387
388
389 class CoverCleanField(CoverField):
390     directory = 'cover_clean'
391
392     @staticmethod
393     def transform(wldoc):
394         from librarian.covers.marquise import MarquiseCover
395         return MarquiseCover(wldoc.book_info, width=360).output_file()
396
397
398 class CoverThumbField(CoverField):
399     directory = 'cover_thumb'
400
401     @staticmethod
402     def transform(wldoc):
403         from librarian.cover import WLCover
404         return WLCover(wldoc.book_info, height=193).output_file()
405
406
407 class CoverApiThumbField(CoverField):
408     directory = 'cover_api_thumb'
409
410     @staticmethod
411     def transform(wldoc):
412         from librarian.cover import WLNoBoxCover
413         return WLNoBoxCover(wldoc.book_info, height=500).output_file()
414
415
416 class SimpleCoverField(CoverField):
417     directory = 'cover_simple'
418
419     @staticmethod
420     def transform(wldoc):
421         from librarian.cover import WLNoBoxCover
422         return WLNoBoxCover(wldoc.book_info, height=1000).output_file()
423
424
425 class CoverEbookpointField(CoverField):
426     directory = 'cover_ebookpoint'
427
428     @staticmethod
429     def transform(wldoc):
430         from librarian.cover import EbookpointCover
431         return EbookpointCover(wldoc.book_info).output_file()