fix
[wolnelektury.git] / src / catalogue / fields.py
1 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
2 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
3 #
4 from django.conf import settings
5 from django.core.files import File
6 from django.core.files.storage import FileSystemStorage
7 from django.db import models
8 from django.db.models.fields.files import FieldFile
9 from catalogue import app_settings
10 from catalogue.constants import LANGUAGES_3TO2
11 from catalogue.utils import remove_zip, truncate_html_words, gallery_path, gallery_url
12 from celery.task import Task, task
13 from celery.utils.log import get_task_logger
14 from waiter.utils import clear_cache
15
16 task_logger = get_task_logger(__name__)
17
18
19 class EbookFieldFile(FieldFile):
20     """Represents contents of an ebook file field."""
21
22     def build(self):
23         """Build the ebook immediately."""
24         return self.field.builder.build(self)
25
26     def build_delay(self):
27         """Builds the ebook in a delayed task."""
28         return self.field.builder.delay(self.instance, self.field.attname)
29
30     def get_url(self):
31         return self.instance.media_url(self.field.attname.split('_')[0])
32
33     def set_readable(self, readable):
34         import os
35         permissions = 0o644 if readable else 0o600
36         os.chmod(self.path, permissions)
37
38
39 class EbookField(models.FileField):
40     """Represents an ebook file field, attachable to a model."""
41     attr_class = EbookFieldFile
42
43     def __init__(self, format_name, *args, **kwargs):
44         super(EbookField, self).__init__(*args, **kwargs)
45         self.format_name = format_name
46
47     def deconstruct(self):
48         name, path, args, kwargs = super(EbookField, self).deconstruct()
49         args.insert(0, self.format_name)
50         return name, path, args, kwargs
51
52     @property
53     def builder(self):
54         """Finds a celery task suitable for the format of the field."""
55         return BuildEbook.for_format(self.format_name)
56
57     def contribute_to_class(self, cls, name):
58         super(EbookField, self).contribute_to_class(cls, name)
59
60         def has(model_instance):
61             return bool(getattr(model_instance, self.attname, None))
62         has.__doc__ = None
63         has.__name__ = str("has_%s" % self.attname)
64         has.short_description = self.name
65         has.boolean = True
66         setattr(cls, 'has_%s' % self.attname, has)
67
68
69 class BuildEbook(Task):
70     formats = {}
71
72     @classmethod
73     def register(cls, format_name):
74         """A decorator for registering subclasses for particular formats."""
75         def wrapper(builder):
76             cls.formats[format_name] = builder
77             return builder
78         return wrapper
79
80     @classmethod
81     def for_format(cls, format_name):
82         """Returns a celery task suitable for specified format."""
83         return cls.formats.get(format_name, BuildEbookTask)
84
85     @staticmethod
86     def transform(wldoc, fieldfile):
87         """Transforms an librarian.WLDocument into an librarian.OutputFile.
88
89         By default, it just calls relevant wldoc.as_??? method.
90
91         """
92         return getattr(wldoc, "as_%s" % fieldfile.field.format_name)()
93
94     def run(self, obj, field_name):
95         """Just run `build` on FieldFile, can't pass it directly to Celery."""
96         task_logger.info("%s -> %s" % (obj.slug, field_name))
97         ret = self.build(getattr(obj, field_name))
98         obj.clear_cache()
99         return ret
100
101     def set_file_permissions(self, fieldfile):
102         if fieldfile.instance.preview:
103             fieldfile.set_readable(False)
104
105     def build(self, fieldfile):
106         book = fieldfile.instance
107         out = self.transform(book.wldocument(), fieldfile)
108         fieldfile.save(None, File(open(out.get_filename(), 'rb')), save=False)
109         self.set_file_permissions(fieldfile)
110         if book.pk is not None:
111             books = type(book).objects.filter(pk=book.pk)
112             books.update(**{
113                 fieldfile.field.attname: fieldfile
114             })
115             for book in books:
116                 book.save()  # just to trigger post-save
117         if fieldfile.field.format_name in app_settings.FORMAT_ZIPS:
118             remove_zip(app_settings.FORMAT_ZIPS[fieldfile.field.format_name])
119 # Don't decorate BuildEbook, because we want to subclass it.
120 BuildEbookTask = task(BuildEbook, ignore_result=True)
121
122
123 @BuildEbook.register('txt')
124 @task(ignore_result=True)
125 class BuildTxt(BuildEbook):
126     @staticmethod
127     def transform(wldoc, fieldfile):
128         return wldoc.as_text()
129
130
131 @BuildEbook.register('pdf')
132 @task(ignore_result=True)
133 class BuildPdf(BuildEbook):
134     @staticmethod
135     def transform(wldoc, fieldfile):
136         return wldoc.as_pdf(morefloats=settings.LIBRARIAN_PDF_MOREFLOATS, cover=True,
137                             ilustr_path=gallery_path(wldoc.book_info.url.slug), customizations=['notoc'])
138
139     def build(self, fieldfile):
140         BuildEbook.build(self, fieldfile)
141         clear_cache(fieldfile.instance.slug)
142
143
144 @BuildEbook.register('epub')
145 @task(ignore_result=True)
146 class BuildEpub(BuildEbook):
147     @staticmethod
148     def transform(wldoc, fieldfile):
149         return wldoc.as_epub(cover=True, ilustr_path=gallery_path(wldoc.book_info.url.slug))
150
151
152 @BuildEbook.register('mobi')
153 @task(ignore_result=True)
154 class BuildMobi(BuildEbook):
155     @staticmethod
156     def transform(wldoc, fieldfile):
157         return wldoc.as_mobi(cover=True, ilustr_path=gallery_path(wldoc.book_info.url.slug))
158
159
160 @BuildEbook.register('html')
161 @task(ignore_result=True)
162 class BuildHtml(BuildEbook):
163     def build(self, fieldfile):
164         from django.core.files.base import ContentFile
165         from slugify import slugify
166         from sortify import sortify
167         from librarian import html
168         from catalogue.models import Fragment, Tag
169
170         book = fieldfile.instance
171
172         html_output = self.transform(book.wldocument(parse_dublincore=False), fieldfile)
173
174         # Delete old fragments, create from scratch if necessary.
175         book.fragments.all().delete()
176
177         if html_output:
178             meta_tags = list(book.tags.filter(
179                 category__in=('author', 'epoch', 'genre', 'kind')))
180
181             lang = book.language
182             lang = LANGUAGES_3TO2.get(lang, lang)
183             if lang not in [ln[0] for ln in settings.LANGUAGES]:
184                 lang = None
185
186             fieldfile.save(None, ContentFile(html_output.get_bytes()), save=False)
187             self.set_file_permissions(fieldfile)
188             type(book).objects.filter(pk=book.pk).update(**{
189                 fieldfile.field.attname: fieldfile
190             })
191
192             # Extract fragments
193             closed_fragments, open_fragments = html.extract_fragments(fieldfile.path)
194             for fragment in closed_fragments.values():
195                 try:
196                     theme_names = [s.strip() for s in fragment.themes.split(',')]
197                 except AttributeError:
198                     continue
199                 themes = []
200                 for theme_name in theme_names:
201                     if not theme_name:
202                         continue
203                     if lang == settings.LANGUAGE_CODE:
204                         # Allow creating themes if book in default language.
205                         tag, created = Tag.objects.get_or_create(
206                                             slug=slugify(theme_name),
207                                             category='theme')
208                         if created:
209                             tag.name = theme_name
210                             setattr(tag, "name_%s" % lang, theme_name)
211                             tag.sort_key = sortify(theme_name.lower())
212                             tag.for_books = True
213                             tag.save()
214                         themes.append(tag)
215                     elif lang is not None:
216                         # Don't create unknown themes in non-default languages.
217                         try:
218                             tag = Tag.objects.get(category='theme', **{"name_%s" % lang: theme_name})
219                         except Tag.DoesNotExist:
220                             pass
221                         else:
222                             themes.append(tag)
223                 if not themes:
224                     continue
225
226                 text = fragment.to_string()
227                 short_text = truncate_html_words(text, 15)
228                 if text == short_text:
229                     short_text = ''
230                 new_fragment = Fragment.objects.create(anchor=fragment.id, book=book, text=text, short_text=short_text)
231
232                 new_fragment.save()
233                 new_fragment.tags = set(meta_tags + themes)
234                 for theme in themes:
235                     if not theme.for_books:
236                         theme.for_books = True
237                         theme.save()
238             book.html_built.send(sender=type(self), instance=book)
239             return True
240         return False
241
242     @staticmethod
243     def transform(wldoc, fieldfile):
244         # ugly, but we can't use wldoc.book_info here
245         from librarian import DCNS
246         url_elem = wldoc.edoc.getroot().find('.//' + DCNS('identifier.url'))
247         if url_elem is None:
248             gallery = ''
249         else:
250             gallery = gallery_url(slug=url_elem.text.rsplit('/', 1)[1])
251         return wldoc.as_html(options={'gallery': "'%s'" % gallery})
252
253
254 class BuildCover(BuildEbook):
255     def set_file_permissions(self, fieldfile):
256         pass
257
258
259 @BuildEbook.register('cover_thumb')
260 @task(ignore_result=True)
261 class BuildCoverThumb(BuildCover):
262     @classmethod
263     def transform(cls, wldoc, fieldfile):
264         from librarian.cover import WLCover
265         return WLCover(wldoc.book_info, height=193).output_file()
266
267
268 @BuildEbook.register('cover_api_thumb')
269 @task(ignore_result=True)
270 class BuildCoverApiThumb(BuildCover):
271     @classmethod
272     def transform(cls, wldoc, fieldfile):
273         from librarian.cover import WLNoBoxCover
274         return WLNoBoxCover(wldoc.book_info, height=500).output_file()
275
276
277 @BuildEbook.register('simple_cover')
278 @task(ignore_result=True)
279 class BuildSimpleCover(BuildCover):
280     @classmethod
281     def transform(cls, wldoc, fieldfile):
282         from librarian.cover import WLNoBoxCover
283         return WLNoBoxCover(wldoc.book_info, height=1000).output_file()
284
285
286 # not used, but needed for migrations
287 class OverwritingFieldFile(FieldFile):
288     """
289         Deletes the old file before saving the new one.
290     """
291
292     def save(self, name, content, *args, **kwargs):
293         leave = kwargs.pop('leave', None)
294         # delete if there's a file already and there's a new one coming
295         if not leave and self and (not hasattr(content, 'path') or content.path != self.path):
296             self.delete(save=False)
297         return super(OverwritingFieldFile, self).save(name, content, *args, **kwargs)
298
299
300 class OverwritingFileField(models.FileField):
301     attr_class = OverwritingFieldFile
302
303
304 class OverwriteStorage(FileSystemStorage):
305
306     def get_available_name(self, name, max_length=None):
307         self.delete(name)
308         return name