1 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
2 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
4 from django.conf import settings
5 from django.core.files import File
6 from django.core.files.storage import FileSystemStorage
7 from django.db import models
8 from django.db.models.fields.files import FieldFile
9 from catalogue import app_settings
10 from catalogue.constants import LANGUAGES_3TO2
11 from catalogue.utils import remove_zip, truncate_html_words, gallery_path, gallery_url
12 from celery.task import Task, task
13 from celery.utils.log import get_task_logger
14 from waiter.utils import clear_cache
16 task_logger = get_task_logger(__name__)
19 class EbookFieldFile(FieldFile):
20 """Represents contents of an ebook file field."""
23 """Build the ebook immediately."""
24 return self.field.builder.build(self)
26 def build_delay(self):
27 """Builds the ebook in a delayed task."""
28 return self.field.builder.delay(self.instance, self.field.attname)
31 return self.instance.media_url(self.field.attname.split('_')[0])
33 def set_readable(self, readable):
35 permissions = 0o644 if readable else 0o600
36 os.chmod(self.path, permissions)
39 class EbookField(models.FileField):
40 """Represents an ebook file field, attachable to a model."""
41 attr_class = EbookFieldFile
43 def __init__(self, format_name, *args, **kwargs):
44 super(EbookField, self).__init__(*args, **kwargs)
45 self.format_name = format_name
47 def deconstruct(self):
48 name, path, args, kwargs = super(EbookField, self).deconstruct()
49 args.insert(0, self.format_name)
50 return name, path, args, kwargs
54 """Finds a celery task suitable for the format of the field."""
55 return BuildEbook.for_format(self.format_name)
57 def contribute_to_class(self, cls, name):
58 super(EbookField, self).contribute_to_class(cls, name)
60 def has(model_instance):
61 return bool(getattr(model_instance, self.attname, None))
63 has.__name__ = str("has_%s" % self.attname)
64 has.short_description = self.name
66 setattr(cls, 'has_%s' % self.attname, has)
69 class BuildEbook(Task):
73 def register(cls, format_name):
74 """A decorator for registering subclasses for particular formats."""
76 cls.formats[format_name] = builder
81 def for_format(cls, format_name):
82 """Returns a celery task suitable for specified format."""
83 return cls.formats.get(format_name, BuildEbookTask)
86 def transform(wldoc, fieldfile):
87 """Transforms an librarian.WLDocument into an librarian.OutputFile.
89 By default, it just calls relevant wldoc.as_??? method.
92 return getattr(wldoc, "as_%s" % fieldfile.field.format_name)()
94 def run(self, obj, field_name):
95 """Just run `build` on FieldFile, can't pass it directly to Celery."""
96 task_logger.info("%s -> %s" % (obj.slug, field_name))
97 ret = self.build(getattr(obj, field_name))
101 def set_file_permissions(self, fieldfile):
102 if fieldfile.instance.preview:
103 fieldfile.set_readable(False)
105 def build(self, fieldfile):
106 book = fieldfile.instance
107 out = self.transform(book.wldocument(), fieldfile)
108 fieldfile.save(None, File(open(out.get_filename(), 'rb')), save=False)
109 self.set_file_permissions(fieldfile)
110 if book.pk is not None:
111 books = type(book).objects.filter(pk=book.pk)
113 fieldfile.field.attname: fieldfile
116 book.save() # just to trigger post-save
117 if fieldfile.field.format_name in app_settings.FORMAT_ZIPS:
118 remove_zip(app_settings.FORMAT_ZIPS[fieldfile.field.format_name])
119 # Don't decorate BuildEbook, because we want to subclass it.
120 BuildEbookTask = task(BuildEbook, ignore_result=True)
123 @BuildEbook.register('txt')
124 @task(ignore_result=True)
125 class BuildTxt(BuildEbook):
127 def transform(wldoc, fieldfile):
128 return wldoc.as_text()
131 @BuildEbook.register('pdf')
132 @task(ignore_result=True)
133 class BuildPdf(BuildEbook):
135 def transform(wldoc, fieldfile):
136 return wldoc.as_pdf(morefloats=settings.LIBRARIAN_PDF_MOREFLOATS, cover=True,
137 ilustr_path=gallery_path(wldoc.book_info.url.slug), customizations=['notoc'])
139 def build(self, fieldfile):
140 BuildEbook.build(self, fieldfile)
141 clear_cache(fieldfile.instance.slug)
144 @BuildEbook.register('epub')
145 @task(ignore_result=True)
146 class BuildEpub(BuildEbook):
148 def transform(wldoc, fieldfile):
149 return wldoc.as_epub(cover=True, ilustr_path=gallery_path(wldoc.book_info.url.slug))
152 @BuildEbook.register('mobi')
153 @task(ignore_result=True)
154 class BuildMobi(BuildEbook):
156 def transform(wldoc, fieldfile):
157 return wldoc.as_mobi(cover=True, ilustr_path=gallery_path(wldoc.book_info.url.slug))
160 @BuildEbook.register('html')
161 @task(ignore_result=True)
162 class BuildHtml(BuildEbook):
163 def build(self, fieldfile):
164 from django.core.files.base import ContentFile
165 from slugify import slugify
166 from sortify import sortify
167 from librarian import html
168 from catalogue.models import Fragment, Tag
170 book = fieldfile.instance
172 html_output = self.transform(book.wldocument(parse_dublincore=False), fieldfile)
174 # Delete old fragments, create from scratch if necessary.
175 book.fragments.all().delete()
178 meta_tags = list(book.tags.filter(
179 category__in=('author', 'epoch', 'genre', 'kind')))
182 lang = LANGUAGES_3TO2.get(lang, lang)
183 if lang not in [ln[0] for ln in settings.LANGUAGES]:
186 fieldfile.save(None, ContentFile(html_output.get_bytes()), save=False)
187 self.set_file_permissions(fieldfile)
188 type(book).objects.filter(pk=book.pk).update(**{
189 fieldfile.field.attname: fieldfile
193 closed_fragments, open_fragments = html.extract_fragments(fieldfile.path)
194 for fragment in closed_fragments.values():
196 theme_names = [s.strip() for s in fragment.themes.split(',')]
197 except AttributeError:
200 for theme_name in theme_names:
203 if lang == settings.LANGUAGE_CODE:
204 # Allow creating themes if book in default language.
205 tag, created = Tag.objects.get_or_create(
206 slug=slugify(theme_name),
209 tag.name = theme_name
210 setattr(tag, "name_%s" % lang, theme_name)
211 tag.sort_key = sortify(theme_name.lower())
215 elif lang is not None:
216 # Don't create unknown themes in non-default languages.
218 tag = Tag.objects.get(category='theme', **{"name_%s" % lang: theme_name})
219 except Tag.DoesNotExist:
226 text = fragment.to_string()
227 short_text = truncate_html_words(text, 15)
228 if text == short_text:
230 new_fragment = Fragment.objects.create(anchor=fragment.id, book=book, text=text, short_text=short_text)
233 new_fragment.tags = set(meta_tags + themes)
235 if not theme.for_books:
236 theme.for_books = True
238 book.html_built.send(sender=type(self), instance=book)
243 def transform(wldoc, fieldfile):
244 # ugly, but we can't use wldoc.book_info here
245 from librarian import DCNS
246 url_elem = wldoc.edoc.getroot().find('.//' + DCNS('identifier.url'))
250 gallery = gallery_url(slug=url_elem.text.rsplit('/', 1)[1])
251 return wldoc.as_html(options={'gallery': "'%s'" % gallery})
254 class BuildCover(BuildEbook):
255 def set_file_permissions(self, fieldfile):
259 @BuildEbook.register('cover_thumb')
260 @task(ignore_result=True)
261 class BuildCoverThumb(BuildCover):
263 def transform(cls, wldoc, fieldfile):
264 from librarian.cover import WLCover
265 return WLCover(wldoc.book_info, height=193).output_file()
268 @BuildEbook.register('cover_api_thumb')
269 @task(ignore_result=True)
270 class BuildCoverApiThumb(BuildCover):
272 def transform(cls, wldoc, fieldfile):
273 from librarian.cover import WLNoBoxCover
274 return WLNoBoxCover(wldoc.book_info, height=500).output_file()
277 @BuildEbook.register('simple_cover')
278 @task(ignore_result=True)
279 class BuildSimpleCover(BuildCover):
281 def transform(cls, wldoc, fieldfile):
282 from librarian.cover import WLNoBoxCover
283 return WLNoBoxCover(wldoc.book_info, height=1000).output_file()
286 # not used, but needed for migrations
287 class OverwritingFieldFile(FieldFile):
289 Deletes the old file before saving the new one.
292 def save(self, name, content, *args, **kwargs):
293 leave = kwargs.pop('leave', None)
294 # delete if there's a file already and there's a new one coming
295 if not leave and self and (not hasattr(content, 'path') or content.path != self.path):
296 self.delete(save=False)
297 return super(OverwritingFieldFile, self).save(name, content, *args, **kwargs)
300 class OverwritingFileField(models.FileField):
301 attr_class = OverwritingFieldFile
304 class OverwriteStorage(FileSystemStorage):
306 def get_available_name(self, name, max_length=None):