fix
[wolnelektury.git] / src / catalogue / fields.py
1 # This file is part of Wolne Lektury, licensed under GNU Affero GPLv3 or later.
2 # Copyright © Fundacja Wolne Lektury. See NOTICE for more information.
3 #
4 import io
5 import os
6 import pkg_resources
7 import random
8 import time
9 from urllib.request import urlopen
10 from django.apps import apps
11 from django.conf import settings
12 from django.core.files import File
13 from django.db import models
14 from django.db.models.fields.files import FieldFile
15 from django.utils.deconstruct import deconstructible
16 from librarian.cover import make_cover
17 from catalogue.constants import LANGUAGES_3TO2
18 from catalogue.utils import absolute_url, remove_zip, truncate_html_words, gallery_path, gallery_url
19 from waiter.utils import clear_cache
20
21 ETAG_SCHEDULED_SUFFIX = '-scheduled'
22 EBOOK_BUILD_PRIORITY = 0
23 EBOOK_REBUILD_PRIORITY = 9
24
25
26 @deconstructible
27 class UploadToPath(object):
28     def __init__(self, path):
29         self.path = path
30
31     def __call__(self, instance, filename):
32         return self.path % instance.slug
33
34     def __eq__(self, other):
35         return isinstance(other, type(self)) and other.path == self.path
36
37
38 def get_make_cover(book):
39     extra = book.get_extra_info_json()
40     cover_logo = extra.get('logo_mono', extra.get('logo'))
41     if cover_logo:
42         while True:
43             try:
44                 cover_logo = io.BytesIO(urlopen(cover_logo, timeout=3).read())
45             except:
46                 time.sleep(2)
47             else:
48                 break
49     
50     def mc(*args, **kwargs):
51         if cover_logo:
52             kwargs['cover_logo'] = cover_logo
53         return make_cover(*args, **kwargs)
54     return mc
55     
56
57 class EbookFieldFile(FieldFile):
58     """Represents contents of an ebook file field."""
59
60     def build(self):
61         """Build the ebook immediately."""
62         etag = self.field.get_current_etag()
63         self.field.build(self)
64         self.update_etag(etag)
65         self.instance.clear_cache()
66
67     def build_delay(self, priority=EBOOK_BUILD_PRIORITY):
68         """Builds the ebook in a delayed task."""
69         from .tasks import build_field
70
71         self.update_etag(
72             "".join([self.field.get_current_etag(), ETAG_SCHEDULED_SUFFIX])
73         )
74         return build_field.apply_async(
75             [self.instance.pk, self.field.attname],
76             priority=priority
77         )
78
79     def set_readable(self, readable):
80         import os
81         permissions = 0o644 if readable else 0o600
82         os.chmod(self.path, permissions)
83
84     def update_etag(self, etag):
85         setattr(self.instance, self.field.etag_field_name, etag)
86         if self.instance.pk:
87             self.instance.save(update_fields=[self.field.etag_field_name])
88
89
90 class EbookField(models.FileField):
91     """Represents an ebook file field, attachable to a model."""
92     attr_class = EbookFieldFile
93     ext = None
94     for_parents = True
95     librarian2_api = False
96     ZIP = None
97
98     def __init__(self, verbose_name=None, with_etag=True, etag_field_name=None, **kwargs):
99         kwargs.setdefault('verbose_name', verbose_name)
100         self.with_etag = with_etag
101         self.etag_field_name = etag_field_name
102         kwargs.setdefault('max_length', 255)
103         kwargs.setdefault('blank', True)
104         kwargs.setdefault('default', '')
105         kwargs.setdefault('upload_to', self.get_upload_to(self.ext))
106
107         super().__init__(**kwargs)
108
109     def deconstruct(self):
110         name, path, args, kwargs = super().deconstruct()
111         if kwargs.get('max_length') == 255:
112             del kwargs['max_length']
113         if kwargs.get('blank') is True:
114             del kwargs['blank']
115         if kwargs.get('default') == '':
116             del kwargs['default']
117         if self.get_upload_to(self.ext) == kwargs.get('upload_to'):
118             del kwargs['upload_to']
119         # with_etag creates a second field, which then deconstructs to manage
120         # its own migrations. So for migrations, etag_field_name is explicitly
121         # set to avoid double creation of the etag field.
122         if self.with_etag:
123             kwargs['etag_field_name'] = self.etag_field_name
124         else:
125             kwargs['with_etag'] = self.with_etag
126
127         return name, path, args, kwargs
128
129     @classmethod
130     def get_upload_to(cls, directory):
131         directory = getattr(cls, 'directory', cls.ext)
132         upload_template = f'book/{directory}/%s.{cls.ext}'
133         return UploadToPath(upload_template)
134
135     def contribute_to_class(self, cls, name):
136         super(EbookField, self).contribute_to_class(cls, name)
137
138         if self.with_etag and not self.etag_field_name:
139             self.etag_field_name = f'{name}_etag'
140             self.etag_field = models.CharField(max_length=255, editable=False, default='', db_index=True)
141             self.etag_field.contribute_to_class(cls, f'{name}_etag')
142
143         def has(model_instance):
144             return bool(getattr(model_instance, self.attname, None))
145         has.__doc__ = None
146         has.__name__ = str("has_%s" % self.attname)
147         has.short_description = self.name
148         has.boolean = True
149
150         setattr(cls, 'has_%s' % self.attname, has)
151
152     def get_current_etag(self):
153         MediaInsertSet = apps.get_model('annoy', 'MediaInsertSet')
154         librarian_version = pkg_resources.get_distribution("librarian").version
155         etag = librarian_version
156         mis = MediaInsertSet.get_for_format(self.ext)
157         if mis is not None:
158             etag += '_' + mis.etag
159         return etag
160
161     def find_stale(self, limit):
162         """Find some books where this format is stale."""
163         # If there is not ETag field, bail. That's true for xml file field.
164         if not self.with_etag:
165             return []
166
167         etag = self.get_current_etag()
168
169         queryset = self.model.objects.all()
170         if not self.for_parents:
171             queryset = queryset.filter(children=None)
172
173         queryset = queryset.exclude(**{
174             f'{self.etag_field_name}__in': [
175                 etag, f'{etag}{ETAG_SCHEDULED_SUFFIX}'
176            ]
177         })
178
179         queryset = queryset.order_by('?')[:limit]
180         return queryset
181
182     @classmethod
183     def find_all_stale(cls, model, limit):
184         """Schedules all stale ebooks of all formats to rebuild."""
185         found = []
186         for field in model._meta.fields:
187             if isinstance(field, cls):
188                 for instance in field.find_stale(limit):
189                     found.append((
190                         field.name,
191                         instance
192                     ))
193         random.shuffle(found)
194         found = found[:limit]
195         return found
196
197     @staticmethod
198     def transform(wldoc, book):
199         """Transforms an librarian.WLDocument into an librarian.OutputFile.
200         """
201         raise NotImplemented()
202
203     def set_file_permissions(self, fieldfile):
204         if fieldfile.instance.preview:
205             fieldfile.set_readable(False)
206
207     def build(self, fieldfile):
208         book = fieldfile.instance
209         out = self.transform(
210             book.wldocument2() if self.librarian2_api else book.wldocument(),
211             book,
212         )
213         with open(out.get_filename(), 'rb') as f:
214             fieldfile.save(None, File(f), save=False)
215         self.set_file_permissions(fieldfile)
216         if book.pk is not None:
217             book.save(update_fields=[self.attname])
218         if self.ZIP:
219             remove_zip(self.ZIP)
220
221
222 class XmlField(EbookField):
223     ext = 'xml'
224
225     def build(self, fieldfile):
226         pass
227
228
229 class TxtField(EbookField):
230     ext = 'txt'
231     for_parents = False
232
233     @staticmethod
234     def transform(wldoc, book):
235         return wldoc.as_text()
236
237
238 class Fb2Field(EbookField):
239     ext = 'fb2'
240     for_parents = False
241     ZIP = 'wolnelektury_pl_fb2'
242
243     @staticmethod
244     def transform(wldoc, book):
245         return wldoc.as_fb2()
246
247
248 class PdfField(EbookField):
249     ext = 'pdf'
250     ZIP = 'wolnelektury_pl_pdf'
251
252     @staticmethod
253     def transform(wldoc, book):
254         return wldoc.as_pdf(
255             morefloats=settings.LIBRARIAN_PDF_MOREFLOATS,
256             cover=get_make_cover(book),
257             base_url=absolute_url(gallery_url(wldoc.book_info.url.slug)), customizations=['notoc'])
258
259     def build(self, fieldfile):
260         super().build(fieldfile)
261         clear_cache(fieldfile.instance.slug)
262
263
264 class EpubField(EbookField):
265     ext = 'epub'
266     librarian2_api = True
267     ZIP = 'wolnelektury_pl_epub'
268
269     @staticmethod
270     def transform(wldoc, book):
271         from librarian.builders import EpubBuilder
272         MediaInsertSet = apps.get_model('annoy', 'MediaInsertSet')
273         return EpubBuilder(
274                 base_url='file://' + os.path.abspath(gallery_path(wldoc.meta.url.slug)) + '/',
275                 fundraising=MediaInsertSet.get_texts_for('epub'),
276                 cover=get_make_cover(book),
277             ).build(wldoc)
278
279
280 class MobiField(EbookField):
281     ext = 'mobi'
282     librarian2_api = True
283     ZIP = 'wolnelektury_pl_mobi'
284
285     @staticmethod
286     def transform(wldoc, book):
287         from librarian.builders import MobiBuilder
288         MediaInsertSet = apps.get_model('annoy', 'MediaInsertSet')
289         return MobiBuilder(
290                 base_url='file://' + os.path.abspath(gallery_path(wldoc.meta.url.slug)) + '/',
291                 fundraising=MediaInsertSet.get_texts_for('mobi'),
292                 cover=get_make_cover(book),
293             ).build(wldoc)
294
295
296 class HtmlField(EbookField):
297     ext = 'html'
298     for_parents = False
299
300     def build(self, fieldfile):
301         from django.core.files.base import ContentFile
302         from slugify import slugify
303         from sortify import sortify
304         from librarian import html
305         from catalogue.models import Fragment, Tag
306
307         book = fieldfile.instance
308
309         html_output = self.transform(book.wldocument(parse_dublincore=False), book)
310
311         # Delete old fragments, create from scratch if necessary.
312         book.fragments.all().delete()
313
314         if html_output:
315             meta_tags = list(book.tags.filter(
316                 category__in=('author', 'epoch', 'genre', 'kind')))
317
318             lang = book.language
319             lang = LANGUAGES_3TO2.get(lang, lang)
320             if lang not in [ln[0] for ln in settings.LANGUAGES]:
321                 lang = None
322
323             fieldfile.save(None, ContentFile(html_output.get_bytes()), save=False)
324             self.set_file_permissions(fieldfile)
325             type(book).objects.filter(pk=book.pk).update(**{
326                 fieldfile.field.attname: fieldfile
327             })
328
329             # Extract fragments
330             closed_fragments, open_fragments = html.extract_fragments(fieldfile.path)
331             for fragment in closed_fragments.values():
332                 try:
333                     theme_names = [s.strip() for s in fragment.themes.split(',')]
334                 except AttributeError:
335                     continue
336                 themes = []
337                 for theme_name in theme_names:
338                     if not theme_name:
339                         continue
340                     if lang == settings.LANGUAGE_CODE:
341                         # Allow creating themes if book in default language.
342                         tag, created = Tag.objects.get_or_create(
343                             slug=slugify(theme_name),
344                             category='theme'
345                         )
346                         if created:
347                             tag.name = theme_name
348                             setattr(tag, "name_%s" % lang, theme_name)
349                             tag.sort_key = sortify(theme_name.lower())
350                             tag.for_books = True
351                             tag.save()
352                         themes.append(tag)
353                     elif lang is not None:
354                         # Don't create unknown themes in non-default languages.
355                         try:
356                             tag = Tag.objects.get(
357                                 category='theme',
358                                 **{"name_%s" % lang: theme_name}
359                             )
360                         except Tag.DoesNotExist:
361                             pass
362                         else:
363                             themes.append(tag)
364                 if not themes:
365                     continue
366
367                 text = fragment.to_string()
368                 short_text = truncate_html_words(text, 15)
369                 if text == short_text:
370                     short_text = ''
371                 new_fragment = Fragment.objects.create(
372                     anchor=fragment.id,
373                     book=book,
374                     text=text,
375                     short_text=short_text
376                 )
377
378                 new_fragment.save()
379                 new_fragment.tags = set(meta_tags + themes)
380                 for theme in themes:
381                     if not theme.for_books:
382                         theme.for_books = True
383                         theme.save()
384             book.html_built.send(sender=type(self), instance=book)
385             return True
386         return False
387
388     @staticmethod
389     def transform(wldoc, book):
390         # ugly, but we can't use wldoc.book_info here
391         from librarian import DCNS
392         url_elem = wldoc.edoc.getroot().find('.//' + DCNS('identifier.url'))
393         if url_elem is None:
394             gal_url = ''
395             gal_path = ''
396         else:
397             slug = url_elem.text.rstrip('/').rsplit('/', 1)[1]
398             gal_url = gallery_url(slug=slug)
399             gal_path = gallery_path(slug=slug)
400         return wldoc.as_html(gallery_path=gal_path, gallery_url=gal_url, base_url=absolute_url(gal_url))
401
402
403 class CoverField(EbookField):
404     ext = 'jpg'
405     directory = 'cover'
406
407     @staticmethod
408     def transform(wldoc, book):
409         return get_make_cover(book)(wldoc.book_info, width=360).output_file()
410
411     def set_file_permissions(self, fieldfile):
412         pass
413
414
415 class CoverCleanField(CoverField):
416     directory = 'cover_clean'
417
418     @staticmethod
419     def transform(wldoc, book):
420         return get_make_cover(book)(wldoc.book_info, width=360).output_file()
421
422
423 class CoverThumbField(CoverField):
424     directory = 'cover_thumb'
425
426     @staticmethod
427     def transform(wldoc, book):
428         from librarian.cover import WLCover
429         return WLCover(wldoc.book_info, height=193).output_file()
430
431
432 class CoverApiThumbField(CoverField):
433     directory = 'cover_api_thumb'
434
435     @staticmethod
436     def transform(wldoc, book):
437         from librarian.cover import WLNoBoxCover
438         return WLNoBoxCover(wldoc.book_info, height=500).output_file()
439
440
441 class SimpleCoverField(CoverField):
442     directory = 'cover_simple'
443
444     @staticmethod
445     def transform(wldoc, book):
446         from librarian.cover import WLNoBoxCover
447         return WLNoBoxCover(wldoc.book_info, height=1000).output_file()
448
449
450 class CoverEbookpointField(CoverField):
451     directory = 'cover_ebookpoint'
452
453     @staticmethod
454     def transform(wldoc, book):
455         from librarian.cover import EbookpointCover
456         return EbookpointCover(wldoc.book_info).output_file()