3811bbac1d4ea826490878f59852568b6bd72c4a
[wolnelektury.git] / src / catalogue / models / book.py
1 # -*- coding: utf-8 -*-
2 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
3 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
4 #
5 from collections import OrderedDict
6 from random import randint
7 import os.path
8 import re
9 import urllib
10 from django.conf import settings
11 from django.db import connection, models, transaction
12 from django.db.models import permalink
13 import django.dispatch
14 from django.contrib.contenttypes.fields import GenericRelation
15 from django.core.urlresolvers import reverse
16 from django.utils.translation import ugettext_lazy as _
17 import jsonfield
18 from fnpdjango.storage import BofhFileSystemStorage
19 from ssify import flush_ssi_includes
20 from newtagging import managers
21 from catalogue import constants
22 from catalogue.fields import EbookField
23 from catalogue.models import Tag, Fragment, BookMedia
24 from catalogue.utils import create_zip, gallery_url, gallery_path
25 from catalogue import app_settings
26 from catalogue import tasks
27 from wolnelektury.utils import makedirs
28
29 bofh_storage = BofhFileSystemStorage()
30
31
32 def _make_upload_to(path):
33     def _upload_to(i, n):
34         return path % i.slug
35     return _upload_to
36
37
38 _cover_upload_to = _make_upload_to('book/cover/%s.jpg')
39 _cover_thumb_upload_to = _make_upload_to('book/cover_thumb/%s.jpg')
40
41
42 def _ebook_upload_to(upload_path):
43     return _make_upload_to(upload_path)
44
45
46 class Book(models.Model):
47     """Represents a book imported from WL-XML."""
48     title = models.CharField(_('title'), max_length=32767)
49     sort_key = models.CharField(_('sort key'), max_length=120, db_index=True, editable=False)
50     sort_key_author = models.CharField(
51         _('sort key by author'), max_length=120, db_index=True, editable=False, default=u'')
52     slug = models.SlugField(_('slug'), max_length=120, db_index=True, unique=True)
53     common_slug = models.SlugField(_('slug'), max_length=120, db_index=True)
54     language = models.CharField(_('language code'), max_length=3, db_index=True, default=app_settings.DEFAULT_LANGUAGE)
55     description = models.TextField(_('description'), blank=True)
56     created_at = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True)
57     changed_at = models.DateTimeField(_('creation date'), auto_now=True, db_index=True)
58     parent_number = models.IntegerField(_('parent number'), default=0)
59     extra_info = jsonfield.JSONField(_('extra information'), default={})
60     gazeta_link = models.CharField(blank=True, max_length=240)
61     wiki_link = models.CharField(blank=True, max_length=240)
62
63     # files generated during publication
64     cover = EbookField(
65         'cover', _('cover'),
66         null=True, blank=True,
67         upload_to=_cover_upload_to,
68         storage=bofh_storage, max_length=255)
69     # Cleaner version of cover for thumbs
70     cover_thumb = EbookField(
71         'cover_thumb', _('cover thumbnail'),
72         null=True, blank=True,
73         upload_to=_cover_thumb_upload_to,
74         max_length=255)
75     ebook_formats = constants.EBOOK_FORMATS
76     formats = ebook_formats + ['html', 'xml']
77
78     parent = models.ForeignKey('self', blank=True, null=True, related_name='children')
79     ancestor = models.ManyToManyField('self', blank=True, editable=False, related_name='descendant', symmetrical=False)
80
81     objects = models.Manager()
82     tagged = managers.ModelTaggedItemManager(Tag)
83     tags = managers.TagDescriptor(Tag)
84     tag_relations = GenericRelation(Tag.intermediary_table_model)
85
86     html_built = django.dispatch.Signal()
87     published = django.dispatch.Signal()
88
89     short_html_url_name = 'catalogue_book_short'
90
91     class AlreadyExists(Exception):
92         pass
93
94     class Meta:
95         ordering = ('sort_key',)
96         verbose_name = _('book')
97         verbose_name_plural = _('books')
98         app_label = 'catalogue'
99
100     def __unicode__(self):
101         return self.title
102
103     def get_initial(self):
104         try:
105             return re.search(r'\w', self.title, re.U).group(0)
106         except AttributeError:
107             return ''
108
109     def authors(self):
110         return self.tags.filter(category='author')
111
112     def author_unicode(self):
113         return ", ".join(self.authors().values_list('name', flat=True))
114
115     def save(self, force_insert=False, force_update=False, **kwargs):
116         from sortify import sortify
117
118         self.sort_key = sortify(self.title)[:120]
119         self.title = unicode(self.title)  # ???
120
121         try:
122             author = self.authors().first().sort_key
123         except AttributeError:
124             author = u''
125         self.sort_key_author = author
126
127         ret = super(Book, self).save(force_insert, force_update, **kwargs)
128
129         return ret
130
131     @permalink
132     def get_absolute_url(self):
133         return 'catalogue.views.book_detail', [self.slug]
134
135     @staticmethod
136     @permalink
137     def create_url(slug):
138         return 'catalogue.views.book_detail', [slug]
139
140     def gallery_path(self):
141         return gallery_path(self.slug)
142
143     def gallery_url(self):
144         return gallery_url(self.slug)
145
146     @property
147     def name(self):
148         return self.title
149
150     def language_code(self):
151         return constants.LANGUAGES_3TO2.get(self.language, self.language)
152
153     def language_name(self):
154         return dict(settings.LANGUAGES).get(self.language_code(), "")
155
156     def has_media(self, type_):
157         if type_ in Book.formats:
158             return bool(getattr(self, "%s_file" % type_))
159         else:
160             return self.media.filter(type=type_).exists()
161
162     def get_media(self, type_):
163         if self.has_media(type_):
164             if type_ in Book.formats:
165                 return getattr(self, "%s_file" % type_)
166             else:
167                 return self.media.filter(type=type_)
168         else:
169             return None
170
171     def get_mp3(self):
172         return self.get_media("mp3")
173
174     def get_odt(self):
175         return self.get_media("odt")
176
177     def get_ogg(self):
178         return self.get_media("ogg")
179
180     def get_daisy(self):
181         return self.get_media("daisy")
182
183     def has_description(self):
184         return len(self.description) > 0
185     has_description.short_description = _('description')
186     has_description.boolean = True
187
188     # ugly ugly ugly
189     def has_mp3_file(self):
190         return bool(self.has_media("mp3"))
191     has_mp3_file.short_description = 'MP3'
192     has_mp3_file.boolean = True
193
194     def has_ogg_file(self):
195         return bool(self.has_media("ogg"))
196     has_ogg_file.short_description = 'OGG'
197     has_ogg_file.boolean = True
198
199     def has_daisy_file(self):
200         return bool(self.has_media("daisy"))
201     has_daisy_file.short_description = 'DAISY'
202     has_daisy_file.boolean = True
203
204     def wldocument(self, parse_dublincore=True, inherit=True):
205         from catalogue.import_utils import ORMDocProvider
206         from librarian.parser import WLDocument
207
208         if inherit and self.parent:
209             meta_fallbacks = self.parent.cover_info()
210         else:
211             meta_fallbacks = None
212
213         return WLDocument.from_file(
214             self.xml_file.path,
215             provider=ORMDocProvider(self),
216             parse_dublincore=parse_dublincore,
217             meta_fallbacks=meta_fallbacks)
218
219     @staticmethod
220     def zip_format(format_):
221         def pretty_file_name(book):
222             return "%s/%s.%s" % (
223                 book.extra_info['author'],
224                 book.slug,
225                 format_)
226
227         field_name = "%s_file" % format_
228         books = Book.objects.filter(parent=None).exclude(**{field_name: ""})
229         paths = [(pretty_file_name(b), getattr(b, field_name).path) for b in books.iterator()]
230         return create_zip(paths, app_settings.FORMAT_ZIPS[format_])
231
232     def zip_audiobooks(self, format_):
233         bm = BookMedia.objects.filter(book=self, type=format_)
234         paths = map(lambda bm: (None, bm.file.path), bm)
235         return create_zip(paths, "%s_%s" % (self.slug, format_))
236
237     def search_index(self, book_info=None, index=None, index_tags=True, commit=True):
238         if index is None:
239             from search.index import Index
240             index = Index()
241         try:
242             index.index_book(self, book_info)
243             if index_tags:
244                 index.index_tags()
245             if commit:
246                 index.index.commit()
247         except Exception, e:
248             index.index.rollback()
249             raise e
250
251     def download_pictures(self, remote_gallery_url):
252         gallery_path = self.gallery_path()
253         # delete previous files, so we don't include old files in ebooks
254         if os.path.isdir(gallery_path):
255             for filename in os.listdir(gallery_path):
256                 file_path = os.path.join(gallery_path, filename)
257                 os.unlink(file_path)
258         ilustr_elements = list(self.wldocument().edoc.findall('//ilustr'))
259         if ilustr_elements:
260             makedirs(gallery_path)
261             for ilustr in ilustr_elements:
262                 ilustr_src = ilustr.get('src')
263                 ilustr_path = os.path.join(gallery_path, ilustr_src)
264                 urllib.urlretrieve('%s/%s' % (remote_gallery_url, ilustr_src), ilustr_path)
265
266     @classmethod
267     def from_xml_file(cls, xml_file, **kwargs):
268         from django.core.files import File
269         from librarian import dcparser
270
271         # use librarian to parse meta-data
272         book_info = dcparser.parse(xml_file)
273
274         if not isinstance(xml_file, File):
275             xml_file = File(open(xml_file))
276
277         try:
278             return cls.from_text_and_meta(xml_file, book_info, **kwargs)
279         finally:
280             xml_file.close()
281
282     @classmethod
283     def from_text_and_meta(cls, raw_file, book_info, overwrite=False, dont_build=None, search_index=True,
284                            search_index_tags=True, remote_gallery_url=None):
285         if dont_build is None:
286             dont_build = set()
287         dont_build = set.union(set(dont_build), set(app_settings.DONT_BUILD))
288
289         # check for parts before we do anything
290         children = []
291         if hasattr(book_info, 'parts'):
292             for part_url in book_info.parts:
293                 try:
294                     children.append(Book.objects.get(slug=part_url.slug))
295                 except Book.DoesNotExist:
296                     raise Book.DoesNotExist(_('Book "%s" does not exist.') % part_url.slug)
297
298         # Read book metadata
299         book_slug = book_info.url.slug
300         if re.search(r'[^a-z0-9-]', book_slug):
301             raise ValueError('Invalid characters in slug')
302         book, created = Book.objects.get_or_create(slug=book_slug)
303
304         if created:
305             book_shelves = []
306             old_cover = None
307         else:
308             if not overwrite:
309                 raise Book.AlreadyExists(_('Book %s already exists') % book_slug)
310             # Save shelves for this book
311             book_shelves = list(book.tags.filter(category='set'))
312             old_cover = book.cover_info()
313
314         # Save XML file
315         book.xml_file.save('%s.xml' % book.slug, raw_file, save=False)
316
317         book.language = book_info.language
318         book.title = book_info.title
319         if book_info.variant_of:
320             book.common_slug = book_info.variant_of.slug
321         else:
322             book.common_slug = book.slug
323         book.extra_info = book_info.to_dict()
324         book.save()
325
326         meta_tags = Tag.tags_from_info(book_info)
327
328         book.tags = set(meta_tags + book_shelves)
329
330         cover_changed = old_cover != book.cover_info()
331         obsolete_children = set(b for b in book.children.all()
332                                 if b not in children)
333         notify_cover_changed = []
334         for n, child_book in enumerate(children):
335             new_child = child_book.parent != book
336             child_book.parent = book
337             child_book.parent_number = n
338             child_book.save()
339             if new_child or cover_changed:
340                 notify_cover_changed.append(child_book)
341         # Disown unfaithful children and let them cope on their own.
342         for child in obsolete_children:
343             child.parent = None
344             child.parent_number = 0
345             child.save()
346             if old_cover:
347                 notify_cover_changed.append(child)
348
349         cls.repopulate_ancestors()
350         tasks.update_counters.delay()
351
352         if remote_gallery_url:
353             book.download_pictures(remote_gallery_url)
354
355         # No saves beyond this point.
356
357         # Build cover.
358         if 'cover' not in dont_build:
359             book.cover.build_delay()
360             book.cover_thumb.build_delay()
361
362         # Build HTML and ebooks.
363         book.html_file.build_delay()
364         if not children:
365             for format_ in constants.EBOOK_FORMATS_WITHOUT_CHILDREN:
366                 if format_ not in dont_build:
367                     getattr(book, '%s_file' % format_).build_delay()
368         for format_ in constants.EBOOK_FORMATS_WITH_CHILDREN:
369             if format_ not in dont_build:
370                 getattr(book, '%s_file' % format_).build_delay()
371
372         if not settings.NO_SEARCH_INDEX and search_index:
373             tasks.index_book.delay(book.id, book_info=book_info, index_tags=search_index_tags)
374
375         for child in notify_cover_changed:
376             child.parent_cover_changed()
377
378         cls.published.send(sender=cls, instance=book)
379         return book
380
381     @classmethod
382     @transaction.atomic
383     def repopulate_ancestors(cls):
384         """Fixes the ancestry cache."""
385         # TODO: table names
386         cursor = connection.cursor()
387         if connection.vendor == 'postgres':
388             cursor.execute("TRUNCATE catalogue_book_ancestor")
389             cursor.execute("""
390                 WITH RECURSIVE ancestry AS (
391                     SELECT book.id, book.parent_id
392                     FROM catalogue_book AS book
393                     WHERE book.parent_id IS NOT NULL
394                     UNION
395                     SELECT ancestor.id, book.parent_id
396                     FROM ancestry AS ancestor, catalogue_book AS book
397                     WHERE ancestor.parent_id = book.id
398                         AND book.parent_id IS NOT NULL
399                     )
400                 INSERT INTO catalogue_book_ancestor
401                     (from_book_id, to_book_id)
402                     SELECT id, parent_id
403                     FROM ancestry
404                     ORDER BY id;
405                 """)
406         else:
407             cursor.execute("DELETE FROM catalogue_book_ancestor")
408             for b in cls.objects.exclude(parent=None):
409                 parent = b.parent
410                 while parent is not None:
411                     b.ancestor.add(parent)
412                     parent = parent.parent
413
414     def flush_includes(self, languages=True):
415         if not languages:
416             return
417         if languages is True:
418             languages = [lc for (lc, _ln) in settings.LANGUAGES]
419         flush_ssi_includes([
420             template % (self.pk, lang)
421             for template in [
422                 '/katalog/b/%d/mini.%s.html',
423                 '/katalog/b/%d/mini_nolink.%s.html',
424                 '/katalog/b/%d/short.%s.html',
425                 '/katalog/b/%d/wide.%s.html',
426                 '/api/include/book/%d.%s.json',
427                 '/api/include/book/%d.%s.xml',
428                 ]
429             for lang in languages
430             ])
431
432     def cover_info(self, inherit=True):
433         """Returns a dictionary to serve as fallback for BookInfo.
434
435         For now, the only thing inherited is the cover image.
436         """
437         need = False
438         info = {}
439         for field in ('cover_url', 'cover_by', 'cover_source'):
440             val = self.extra_info.get(field)
441             if val:
442                 info[field] = val
443             else:
444                 need = True
445         if inherit and need and self.parent is not None:
446             parent_info = self.parent.cover_info()
447             parent_info.update(info)
448             info = parent_info
449         return info
450
451     def related_themes(self):
452         return Tag.objects.usage_for_queryset(
453             Fragment.objects.filter(models.Q(book=self) | models.Q(book__ancestor=self)),
454             counts=True).filter(category='theme')
455
456     def parent_cover_changed(self):
457         """Called when parent book's cover image is changed."""
458         if not self.cover_info(inherit=False):
459             if 'cover' not in app_settings.DONT_BUILD:
460                 self.cover.build_delay()
461                 self.cover_thumb.build_delay()
462             for format_ in constants.EBOOK_FORMATS_WITH_COVERS:
463                 if format_ not in app_settings.DONT_BUILD:
464                     getattr(self, '%s_file' % format_).build_delay()
465             for child in self.children.all():
466                 child.parent_cover_changed()
467
468     def other_versions(self):
469         """Find other versions (i.e. in other languages) of the book."""
470         return type(self).objects.filter(common_slug=self.common_slug).exclude(pk=self.pk)
471
472     def parents(self):
473         books = []
474         parent = self.parent
475         while parent is not None:
476             books.insert(0, parent)
477             parent = parent.parent
478         return books
479
480     def pretty_title(self, html_links=False):
481         names = [(tag.name, tag.get_absolute_url()) for tag in self.authors().only('name', 'category', 'slug')]
482         books = self.parents() + [self]
483         names.extend([(b.title, b.get_absolute_url()) for b in books])
484
485         if html_links:
486             names = ['<a href="%s">%s</a>' % (tag[1], tag[0]) for tag in names]
487         else:
488             names = [tag[0] for tag in names]
489         return ', '.join(names)
490
491     @classmethod
492     def tagged_top_level(cls, tags):
493         """ Returns top-level books tagged with `tags`.
494
495         It only returns those books which don't have ancestors which are
496         also tagged with those tags.
497
498         """
499         objects = cls.tagged.with_all(tags)
500         return objects.exclude(ancestor__in=objects)
501
502     @classmethod
503     def book_list(cls, book_filter=None):
504         """Generates a hierarchical listing of all books.
505
506         Books are optionally filtered with a test function.
507
508         """
509
510         books_by_parent = {}
511         books = cls.objects.order_by('parent_number', 'sort_key').only('title', 'parent', 'slug')
512         if book_filter:
513             books = books.filter(book_filter).distinct()
514
515             book_ids = set(b['pk'] for b in books.values("pk").iterator())
516             for book in books.iterator():
517                 parent = book.parent_id
518                 if parent not in book_ids:
519                     parent = None
520                 books_by_parent.setdefault(parent, []).append(book)
521         else:
522             for book in books.iterator():
523                 books_by_parent.setdefault(book.parent_id, []).append(book)
524
525         orphans = []
526         books_by_author = OrderedDict()
527         for tag in Tag.objects.filter(category='author').iterator():
528             books_by_author[tag] = []
529
530         for book in books_by_parent.get(None, ()):
531             authors = list(book.authors().only('pk'))
532             if authors:
533                 for author in authors:
534                     books_by_author[author].append(book)
535             else:
536                 orphans.append(book)
537
538         return books_by_author, orphans, books_by_parent
539
540     _audiences_pl = {
541         "SP": (1, u"szkoła podstawowa"),
542         "SP1": (1, u"szkoła podstawowa"),
543         "SP2": (1, u"szkoła podstawowa"),
544         "P": (1, u"szkoła podstawowa"),
545         "G": (2, u"gimnazjum"),
546         "L": (3, u"liceum"),
547         "LP": (3, u"liceum"),
548     }
549
550     def audiences_pl(self):
551         audiences = self.extra_info.get('audiences', [])
552         audiences = sorted(set([self._audiences_pl.get(a, (99, a)) for a in audiences]))
553         return [a[1] for a in audiences]
554
555     def stage_note(self):
556         stage = self.extra_info.get('stage')
557         if stage and stage < '0.4':
558             return (_('This work needs modernisation'),
559                     reverse('infopage', args=['wymagajace-uwspolczesnienia']))
560         else:
561             return None, None
562
563     def choose_fragment(self):
564         fragments = self.fragments.order_by()
565         fragments_count = fragments.count()
566         if not fragments_count and self.children.exists():
567             fragments = Fragment.objects.filter(book__ancestor=self).order_by()
568             fragments_count = fragments.count()
569         if fragments_count:
570             return fragments[randint(0, fragments_count - 1)]
571         elif self.parent:
572             return self.parent.choose_fragment()
573         else:
574             return None
575
576
577 def add_file_fields():
578     for format_ in Book.formats:
579         field_name = "%s_file" % format_
580         # This weird globals() assignment makes Django migrations comfortable.
581         _upload_to = _ebook_upload_to('book/%s/%%s.%s' % (format_, format_))
582         _upload_to.__name__ = '_%s_upload_to' % format_
583         globals()[_upload_to.__name__] = _upload_to
584
585         EbookField(
586             format_, _("%s file" % format_.upper()),
587             upload_to=_upload_to,
588             storage=bofh_storage,
589             max_length=255,
590             blank=True,
591             default=''
592         ).contribute_to_class(Book, field_name)
593
594 add_file_fields()