Fixes
[redakcja.git] / src / documents / models / book.py
1 # This file is part of FNP-Redakcja, licensed under GNU Affero GPLv3 or later.
2 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
3 #
4 from django.apps import apps
5 from django.core.files.base import ContentFile
6 from django.contrib.sites.models import Site
7 from django.db import connection, models, transaction
8 from django.template.loader import render_to_string
9 from django.urls import reverse
10 from django.utils.translation import gettext_lazy as _
11 from django.conf import settings
12 from slugify import slugify
13 from librarian.cover import make_cover
14 from librarian.dcparser import BookInfo
15
16 import apiclient
17 from documents.helpers import cached_in_field, GalleryMerger
18 from documents.models import BookPublishRecord, ChunkPublishRecord, Project
19 from documents.signals import post_publish
20 from documents.xml_tools import compile_text, split_xml
21 from cover.models import Image
22 from io import BytesIO
23 import os
24 import shutil
25 import re
26 from urllib.parse import urljoin
27
28
29 class Book(models.Model):
30     """ A document edited on the wiki """
31
32     title = models.CharField(_('title'), max_length=255, db_index=True)
33     slug = models.SlugField(_('slug'), max_length=128, unique=True, db_index=True)
34     public = models.BooleanField(_('public'), default=True, db_index=True)
35     gallery = models.CharField(_('scan gallery name'), max_length=255, blank=True)
36     project = models.ForeignKey(Project, models.SET_NULL, null=True, blank=True)
37
38     parent = models.ForeignKey('self', models.SET_NULL, null=True, blank=True, verbose_name=_('parent'), related_name="children", editable=False)
39     parent_number = models.IntegerField(_('parent number'), null=True, blank=True, db_index=True, editable=False)
40
41     # Cache
42     _single = models.BooleanField(editable=False, null=True, db_index=True)
43     _new_publishable = models.BooleanField(editable=False, null=True)
44     _published = models.BooleanField(editable=False, null=True)
45     _on_track = models.IntegerField(null=True, blank=True, db_index=True, editable=False)
46     dc_cover_image = models.ForeignKey(Image, blank=True, null=True,
47         db_index=True, on_delete=models.SET_NULL, editable=False)
48     dc = models.JSONField(null=True, editable=False)
49     cover = models.FileField(blank=True, upload_to='documents/cover')
50
51     dc_slug = models.CharField(
52         max_length=2048,
53         null=True, blank=True,
54         editable=False,
55     )
56     catalogue_book = models.ForeignKey(
57         'catalogue.Book',
58         models.PROTECT,
59         null=True, blank=True,
60         editable=False,
61         related_name='document_books',
62         related_query_name='document_book',
63     )
64
65     class NoTextError(BaseException):
66         pass
67
68     class Meta:
69         app_label = 'documents'
70         ordering = ['title', 'slug']
71         verbose_name = _('book')
72         verbose_name_plural = _('books')
73
74     @classmethod
75     def get_visible_for(cls, user):
76         qs = cls.objects.all()
77         if not user.is_authenticated:
78             qs = qs.filter(public=True)
79         return qs
80
81     @staticmethod
82     def q_dc(field, field_plural, value, prefix=''):
83         if connection.features.supports_json_field_contains:
84             return models.Q(**{f'{prefix}dc__{field_plural}__contains': value})
85         else:
86             return models.Q(**{f'{prefix}dc__{field}': value})
87             
88     
89     # Representing
90     # ============
91
92     def __iter__(self):
93         return iter(self.chunk_set.all())
94
95     def __getitem__(self, chunk):
96         return self.chunk_set.all()[chunk]
97
98     def __len__(self):
99         return self.chunk_set.count()
100
101     def __bool__(self):
102         """
103             Necessary so that __len__ isn't used for bool evaluation.
104         """
105         return True
106
107     def __str__(self):
108         return self.title
109
110     def get_absolute_url(self):
111         return reverse("documents_book", args=[self.slug])
112
113     def correct_about(self):
114         return "http://%s%s" % (
115             Site.objects.get_current().domain,
116             self.get_absolute_url()
117         )
118
119     def gallery_path(self):
120         return os.path.join(settings.MEDIA_ROOT, settings.IMAGE_DIR, self.gallery)
121
122     def gallery_url(self):
123         return '%s%s%s/' % (settings.MEDIA_URL, settings.IMAGE_DIR, self.gallery)
124
125     # Creating & manipulating
126     # =======================
127
128     def accessible(self, request):
129         return self.public or request.user.is_authenticated
130
131     @classmethod
132     @transaction.atomic
133     def create(cls, creator, text, *args, **kwargs):
134         b = cls.objects.create(*args, **kwargs)
135         b.chunk_set.all().update(creator=creator)
136         b[0].commit(text, author=creator)
137         return b
138
139     def add(self, *args, **kwargs):
140         """Add a new chunk at the end."""
141         return self.chunk_set.reverse()[0].split(*args, **kwargs)
142
143     @classmethod
144     @transaction.atomic
145     def import_xml_text(cls, text=u'', previous_book=None,
146                 commit_args=None, **kwargs):
147         """Imports a book from XML, splitting it into chunks as necessary."""
148         texts = split_xml(text)
149         if previous_book:
150             instance = previous_book
151         else:
152             instance = cls(**kwargs)
153             instance.save()
154
155         # if there are more parts, set the rest to empty strings
156         book_len = len(instance)
157         for i in range(book_len - len(texts)):
158             texts.append((u'pusta część %d' % (i + 1), u''))
159
160         i = 0
161         for i, (title, text) in enumerate(texts):
162             if not title:
163                 title = u'część %d' % (i + 1)
164
165             slug = slugify(title)
166
167             if i < book_len:
168                 chunk = instance[i]
169                 chunk.slug = slug[:50]
170                 chunk.title = title[:255]
171                 chunk.save()
172             else:
173                 chunk = instance.add(slug, title)
174
175             chunk.commit(text, **commit_args)
176
177         return instance
178
179     def make_chunk_slug(self, proposed):
180         """ 
181             Finds a chunk slug not yet used in the book.
182         """
183         slugs = set(c.slug for c in self)
184         i = 1
185         new_slug = proposed[:50]
186         while new_slug in slugs:
187             new_slug = "%s_%d" % (proposed[:45], i)
188             i += 1
189         return new_slug
190
191     @transaction.atomic
192     def append(self, other, slugs=None, titles=None):
193         """Add all chunks of another book to self."""
194         assert self != other
195
196         number = self[len(self) - 1].number + 1
197         len_other = len(other)
198         single = len_other == 1
199
200         if slugs is not None:
201             assert len(slugs) == len_other
202         if titles is not None:
203             assert len(titles) == len_other
204             if slugs is None:
205                 slugs = [slugify(t) for t in titles]
206
207         for i, chunk in enumerate(other):
208             # move chunk to new book
209             chunk.book = self
210             chunk.number = number
211
212             if titles is None:
213                 # try some title guessing
214                 if other.title.startswith(self.title):
215                     other_title_part = other.title[len(self.title):].lstrip(' /')
216                 else:
217                     other_title_part = other.title
218
219                 if single:
220                     # special treatment for appending one-parters:
221                     # just use the guessed title and original book slug
222                     chunk.title = other_title_part
223                     if other.slug.startswith(self.slug):
224                         chunk.slug = other.slug[len(self.slug):].lstrip('-_')
225                     else:
226                         chunk.slug = other.slug
227                 else:
228                     chunk.title = ("%s, %s" % (other_title_part, chunk.title))[:255]
229             else:
230                 chunk.slug = slugs[i]
231                 chunk.title = titles[i]
232
233             chunk.slug = self.make_chunk_slug(chunk.slug)
234             chunk.save()
235             number += 1
236         assert not other.chunk_set.exists()
237
238         gm = GalleryMerger(self.gallery, other.gallery)
239         self.gallery = gm.merge()
240
241         # and move the gallery starts
242         if gm.was_merged:
243                 for chunk in self[len(self) - len_other:]:
244                         old_start = chunk.gallery_start or 1
245                         chunk.gallery_start = old_start + gm.dest_size - gm.num_deleted
246                         chunk.save()
247
248         other.delete()
249
250
251     @transaction.atomic
252     def prepend_history(self, other):
253         """Prepend history from all the other book's chunks to own."""
254         assert self != other
255
256         for i in range(len(self), len(other)):
257             title = u"pusta część %d" % i
258             chunk = self.add(slugify(title), title)
259             chunk.commit('')
260
261         for i in range(len(other)):
262             self[i].prepend_history(other[0])
263
264         assert not other.chunk_set.exists()
265         other.delete()
266
267     def split(self):
268         """Splits all the chunks into separate books."""
269         self.title
270         for chunk in self:
271             book = Book.objects.create(title=chunk.title, slug=chunk.slug,
272                     public=self.public, gallery=self.gallery)
273             book[0].delete()
274             chunk.book = book
275             chunk.number = 1
276             chunk.save()
277         assert not self.chunk_set.exists()
278         self.delete()
279
280     # State & cache
281     # =============
282
283     def last_published(self):
284         try:
285             return self.publish_log.all()[0].timestamp
286         except IndexError:
287             return None
288
289     def assert_publishable(self):
290         assert self.chunk_set.exists(), _('No chunks in the book.')
291         try:
292             changes = self.get_current_changes(publishable=True)
293         except self.NoTextError:
294             raise AssertionError(_('Not all chunks have approved revisions.'))
295
296         from librarian import NoDublinCore, ParseError, ValidationError
297
298         try:
299             bi = self.wldocument(changes=changes, strict=True).book_info
300         except ParseError as e:
301             raise AssertionError(_('Invalid XML') + ': ' + str(e))
302         except NoDublinCore:
303             raise AssertionError(_('No Dublin Core found.'))
304         except ValidationError as e:
305             raise AssertionError(_('Invalid Dublin Core') + ': ' + str(e))
306
307         valid_about = self.correct_about()
308         assert bi.about == valid_about, _("rdf:about is not") + " " + valid_about
309
310         srcs = self.relative_ilustr_srcs(changes=changes)
311         assert not srcs, 'Ilustracje bez pełnego adresu: ' + ', '.join(
312             src or '(pusty)' for src in srcs)
313
314     def relative_ilustr_srcs(self, changes=None):
315         """Returns `src` of illustrations which aren't absolute URLs.
316
317         Those only work relative to the gallery, so they break
318         outside of it. Checks the current text, unless `changes` are given.
319         """
320         from lxml import etree
321         try:
322             tree = etree.fromstring(self.materialize(changes=changes).encode('utf-8'))
323         except (self.NoTextError, etree.XMLSyntaxError):
324             return []
325         return [
326             elem.get('src', '') for elem in tree.iter('ilustr')
327             if not elem.get('src', '').startswith(('http://', 'https://'))
328         ]
329
330     def publishable_error(self):
331         try:
332             return self.assert_publishable()
333         except AssertionError as e:
334             return e
335         else:
336             return None
337
338     def hidden(self):
339         return self.slug.startswith('.')
340
341     def is_new_publishable(self):
342         """Checks if book is ready for publishing.
343
344         Returns True if there is a publishable version newer than the one
345         already published.
346
347         """
348         new_publishable = False
349         if not self.chunk_set.exists():
350             return False
351         for chunk in self:
352             change = chunk.publishable()
353             if not change:
354                 return False
355             if not new_publishable and not change.publish_log.exists():
356                 new_publishable = True
357         return new_publishable
358     new_publishable = cached_in_field('_new_publishable')(is_new_publishable)
359
360     def is_published(self):
361         return self.publish_log.exists()
362     published = cached_in_field('_published')(is_published)
363
364     def get_on_track(self):
365         if self.published:
366             return -1
367         stages = [ch.stage.ordering if ch.stage is not None else 0
368                     for ch in self]
369         if not len(stages):
370             return 0
371         return min(stages)
372     on_track = cached_in_field('_on_track')(get_on_track)
373
374     def is_single(self):
375         return len(self) == 1
376     single = cached_in_field('_single')(is_single)
377
378     def book_info(self, publishable=True):
379         try:
380             book_xml = self.materialize(publishable=publishable)
381         except self.NoTextError:
382             pass
383         else:
384             from librarian.dcparser import BookInfo
385             from librarian import NoDublinCore, ParseError, ValidationError
386             try:
387                 return BookInfo.from_bytes(book_xml.encode('utf-8'))
388             except (self.NoTextError, ParseError, NoDublinCore, ValidationError):
389                 return None
390
391     def refresh_dc_cache(self):
392         update = {
393             'dc_slug': None,
394             'dc_cover_image': None,
395         }
396
397         info = self.book_info()
398         if info is not None:
399             update['dc_slug'] = info.url.slug
400             if info.cover_source:
401                 try:
402                     image = Image.objects.get(pk=int(info.cover_source.rstrip('/').rsplit('/', 1)[-1]))
403                 except:
404                     pass
405                 else:
406                     if info.cover_source.rstrip('/') == image.get_full_url().rstrip('/'):
407                         update['dc_cover_image'] = image
408             update['dc'] = info.to_dict()
409         Book.objects.filter(pk=self.pk).update(**update)
410
411     def touch(self):
412         update = {
413             "_new_publishable": self.is_new_publishable(),
414             "_published": self.is_published(),
415             "_single": self.is_single(),
416             "_on_track": self.get_on_track(),
417         }
418         Book.objects.filter(pk=self.pk).update(**update)
419         self.refresh_dc_cache()
420         self.build_cover()
421
422     def build_cover(self):
423         width, height = 212, 300
424         try:
425             xml = self.materialize(publishable=True).encode('utf-8')
426             info = BookInfo.from_bytes(xml)
427             kwargs = {}
428             if self.project is not None:
429                 if self.project.logo_mono or self.project.logo:
430                     kwargs['cover_logo'] = (self.project.logo_mono or self.project.logo).path
431             cover = make_cover(info, width=width, height=height, **kwargs)
432             out = BytesIO()
433             ext = cover.ext()
434             cover.save(out)
435             self.cover.save(f'{self.slug}.{ext}', out, save=False)
436             type(self).objects.filter(pk=self.pk).update(cover=self.cover)
437         except:
438             type(self).objects.filter(pk=self.pk).update(cover='')
439
440     # Materializing & publishing
441     # ==========================
442
443     def get_current_changes(self, publishable=True):
444         """
445             Returns a list containing one Change for every Chunk in the Book.
446             Takes the most recent revision (publishable, if set).
447             Throws an error, if a proper revision is unavailable for a Chunk.
448         """
449         if publishable:
450             changes = [chunk.publishable() for chunk in self]
451         else:
452             changes = [chunk.head for chunk in self if chunk.head is not None]
453         if None in changes:
454             raise self.NoTextError('Some chunks have no available text.')
455         return changes
456
457     def materialize(self, publishable=False, changes=None):
458         """ 
459             Get full text of the document compiled from chunks.
460             Takes the current versions of all texts
461             or versions most recently tagged for publishing,
462             or a specified iterable changes.
463         """
464         if changes is None:
465             changes = self.get_current_changes(publishable)
466         return compile_text(change.materialize() for change in changes)
467
468     def wldocument(self, publishable=True, changes=None, 
469                    parse_dublincore=True, strict=False, librarian2=False):
470         from documents.ebook_utils import RedakcjaDocProvider
471         from librarian.parser import WLDocument
472         from librarian.document import WLDocument as WLDocument2
473
474         provider = RedakcjaDocProvider(publishable=publishable)
475         xml = self.materialize(publishable=publishable, changes=changes).encode('utf-8')
476         
477         if librarian2:
478             return WLDocument2(
479                 BytesIO(xml),
480                 provider=provider)
481         return WLDocument.from_bytes(
482                 xml,
483                 provider=provider,
484                 parse_dublincore=parse_dublincore,
485                 strict=strict)
486
487     def publish(self, user, fake=False, host=None, days=0, beta=False, hidden=False):
488         """
489             Publishes a book on behalf of a (local) user.
490         """
491         self.assert_publishable()
492         changes = self.get_current_changes(publishable=True)
493         if not fake:
494             book_xml = self.materialize(changes=changes)
495             data = {"book_xml": book_xml, "days": days, "hidden": hidden}
496             if self.catalogue_book is not None:
497                 cb = self.catalogue_book
498                 isbn = cb.isbn_set.filter(form='mp3').first()
499                 if isbn is not None:
500                     data['isbn_mp3'] = isbn.get_code()
501             if self.project is not None:
502                 if self.project.logo:
503                     data['logo'] = urljoin(
504                         'https://' + Site.objects.get_current().domain,
505                         self.project.logo.url,
506                     )
507                 if self.project.logo_mono:
508                     data['logo_mono'] = urljoin(
509                         'https://' + Site.objects.get_current().domain,
510                         self.project.logo_mono.url,
511                     )
512                 if self.project.logo_alt:
513                     data['logo_alt'] = self.project.logo_alt
514                 if not self.project.can_sell:
515                     data['can_sell'] = False
516             if host:
517                 data['gallery_url'] = host + self.gallery_url()
518             apiclient.api_call(user, "books/", data, beta=beta)
519         if not beta:
520             # record the publish
521             br = BookPublishRecord.objects.create(book=self, user=user)
522             for c in changes:
523                 ChunkPublishRecord.objects.create(book_record=br, change=c)
524             if not self.public and days == 0:
525                 self.public = True
526                 self.save()
527             if self.public and days > 0:
528                 self.public = False
529                 self.save()
530             post_publish.send(sender=br)
531
532     def latex_dir(self):
533         doc = self.wldocument()
534         return doc.latex_dir(cover=True, ilustr_path=self.gallery_path())