cover images in filter API
[wolnelektury.git] / src / catalogue / models / book.py
1 # -*- coding: utf-8 -*-
2 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
3 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
4 #
5 from collections import OrderedDict
6 from random import randint
7 import os.path
8 import re
9 import urllib
10 from django.conf import settings
11 from django.db import connection, models, transaction
12 from django.db.models import permalink
13 import django.dispatch
14 from django.contrib.contenttypes.fields import GenericRelation
15 from django.core.urlresolvers import reverse
16 from django.utils.translation import ugettext_lazy as _, get_language
17 import jsonfield
18 from fnpdjango.storage import BofhFileSystemStorage
19 from ssify import flush_ssi_includes
20 from newtagging import managers
21 from catalogue import constants
22 from catalogue.fields import EbookField
23 from catalogue.models import Tag, Fragment, BookMedia
24 from catalogue.utils import create_zip, gallery_url, gallery_path
25 from catalogue.models.tag import prefetched_relations
26 from catalogue import app_settings
27 from catalogue import tasks
28 from wolnelektury.utils import makedirs
29
30 bofh_storage = BofhFileSystemStorage()
31
32
33 def _make_upload_to(path):
34     def _upload_to(i, n):
35         return path % i.slug
36     return _upload_to
37
38
39 _cover_upload_to = _make_upload_to('book/cover/%s.jpg')
40 _cover_thumb_upload_to = _make_upload_to('book/cover_thumb/%s.jpg')
41
42
43 def _ebook_upload_to(upload_path):
44     return _make_upload_to(upload_path)
45
46
47 class Book(models.Model):
48     """Represents a book imported from WL-XML."""
49     title = models.CharField(_('title'), max_length=32767)
50     sort_key = models.CharField(_('sort key'), max_length=120, db_index=True, editable=False)
51     sort_key_author = models.CharField(
52         _('sort key by author'), max_length=120, db_index=True, editable=False, default=u'')
53     slug = models.SlugField(_('slug'), max_length=120, db_index=True, unique=True)
54     common_slug = models.SlugField(_('slug'), max_length=120, db_index=True)
55     language = models.CharField(_('language code'), max_length=3, db_index=True, default=app_settings.DEFAULT_LANGUAGE)
56     description = models.TextField(_('description'), blank=True)
57     created_at = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True)
58     changed_at = models.DateTimeField(_('creation date'), auto_now=True, db_index=True)
59     parent_number = models.IntegerField(_('parent number'), default=0)
60     extra_info = jsonfield.JSONField(_('extra information'), default={})
61     gazeta_link = models.CharField(blank=True, max_length=240)
62     wiki_link = models.CharField(blank=True, max_length=240)
63     print_on_demand = models.BooleanField(_('print on demand'), default=False)
64     recommended = models.BooleanField(_('recommended'), default=False)
65
66     # files generated during publication
67     cover = EbookField(
68         'cover', _('cover'),
69         null=True, blank=True,
70         upload_to=_cover_upload_to,
71         storage=bofh_storage, max_length=255)
72     # Cleaner version of cover for thumbs
73     cover_thumb = EbookField(
74         'cover_thumb', _('cover thumbnail'),
75         null=True, blank=True,
76         upload_to=_cover_thumb_upload_to,
77         max_length=255)
78     ebook_formats = constants.EBOOK_FORMATS
79     formats = ebook_formats + ['html', 'xml']
80
81     parent = models.ForeignKey('self', blank=True, null=True, related_name='children')
82     ancestor = models.ManyToManyField('self', blank=True, editable=False, related_name='descendant', symmetrical=False)
83
84     cached_author = models.CharField(blank=True, max_length=240, db_index=True)
85     has_audience = models.BooleanField(default=False)
86
87     objects = models.Manager()
88     tagged = managers.ModelTaggedItemManager(Tag)
89     tags = managers.TagDescriptor(Tag)
90     tag_relations = GenericRelation(Tag.intermediary_table_model)
91
92     html_built = django.dispatch.Signal()
93     published = django.dispatch.Signal()
94
95     short_html_url_name = 'catalogue_book_short'
96
97     class AlreadyExists(Exception):
98         pass
99
100     class Meta:
101         ordering = ('sort_key_author', 'sort_key')
102         verbose_name = _('book')
103         verbose_name_plural = _('books')
104         app_label = 'catalogue'
105
106     def __unicode__(self):
107         return self.title
108
109     def get_initial(self):
110         try:
111             return re.search(r'\w', self.title, re.U).group(0)
112         except AttributeError:
113             return ''
114
115     def authors(self):
116         return self.tags.filter(category='author')
117
118     def tag_unicode(self, category):
119         relations = prefetched_relations(self, category)
120         if relations:
121             return ', '.join(rel.tag.name for rel in relations)
122         else:
123             return ', '.join(self.tags.filter(category=category).values_list('name', flat=True))
124
125     def author_unicode(self):
126         return self.cached_author
127
128     def translator(self):
129         translators = self.extra_info.get('translators')
130         if not translators:
131             return None
132         if len(translators) > 3:
133             translators = translators[:2]
134             others = ' i inni'
135         else:
136             others = ''
137         return ', '.join(u'\xa0'.join(reversed(translator.split(', ', 1))) for translator in translators) + others
138
139     def cover_source(self):
140         return self.extra_info.get('cover_source', self.parent.cover_source() if self.parent else '')
141
142     def save(self, force_insert=False, force_update=False, **kwargs):
143         from sortify import sortify
144
145         self.sort_key = sortify(self.title)[:120]
146         self.title = unicode(self.title)  # ???
147
148         try:
149             author = self.authors().first().sort_key
150         except AttributeError:
151             author = u''
152         self.sort_key_author = author
153
154         self.cached_author = self.tag_unicode('author')
155         self.has_audience = 'audience' in self.extra_info
156
157         ret = super(Book, self).save(force_insert, force_update, **kwargs)
158
159         return ret
160
161     @permalink
162     def get_absolute_url(self):
163         return 'catalogue.views.book_detail', [self.slug]
164
165     @staticmethod
166     @permalink
167     def create_url(slug):
168         return 'catalogue.views.book_detail', [slug]
169
170     def gallery_path(self):
171         return gallery_path(self.slug)
172
173     def gallery_url(self):
174         return gallery_url(self.slug)
175
176     @property
177     def name(self):
178         return self.title
179
180     def language_code(self):
181         return constants.LANGUAGES_3TO2.get(self.language, self.language)
182
183     def language_name(self):
184         return dict(settings.LANGUAGES).get(self.language_code(), "")
185
186     def is_foreign(self):
187         return self.language_code() != settings.LANGUAGE_CODE
188
189     def has_media(self, type_):
190         if type_ in Book.formats:
191             return bool(getattr(self, "%s_file" % type_))
192         else:
193             return self.media.filter(type=type_).exists()
194
195     def get_media(self, type_):
196         if self.has_media(type_):
197             if type_ in Book.formats:
198                 return getattr(self, "%s_file" % type_)
199             else:
200                 return self.media.filter(type=type_)
201         else:
202             return None
203
204     def get_mp3(self):
205         return self.get_media("mp3")
206
207     def get_odt(self):
208         return self.get_media("odt")
209
210     def get_ogg(self):
211         return self.get_media("ogg")
212
213     def get_daisy(self):
214         return self.get_media("daisy")
215
216     def has_description(self):
217         return len(self.description) > 0
218     has_description.short_description = _('description')
219     has_description.boolean = True
220
221     # ugly ugly ugly
222     def has_mp3_file(self):
223         return bool(self.has_media("mp3"))
224     has_mp3_file.short_description = 'MP3'
225     has_mp3_file.boolean = True
226
227     def has_ogg_file(self):
228         return bool(self.has_media("ogg"))
229     has_ogg_file.short_description = 'OGG'
230     has_ogg_file.boolean = True
231
232     def has_daisy_file(self):
233         return bool(self.has_media("daisy"))
234     has_daisy_file.short_description = 'DAISY'
235     has_daisy_file.boolean = True
236
237     def wldocument(self, parse_dublincore=True, inherit=True):
238         from catalogue.import_utils import ORMDocProvider
239         from librarian.parser import WLDocument
240
241         if inherit and self.parent:
242             meta_fallbacks = self.parent.cover_info()
243         else:
244             meta_fallbacks = None
245
246         return WLDocument.from_file(
247             self.xml_file.path,
248             provider=ORMDocProvider(self),
249             parse_dublincore=parse_dublincore,
250             meta_fallbacks=meta_fallbacks)
251
252     @staticmethod
253     def zip_format(format_):
254         def pretty_file_name(book):
255             return "%s/%s.%s" % (
256                 book.extra_info['author'],
257                 book.slug,
258                 format_)
259
260         field_name = "%s_file" % format_
261         books = Book.objects.filter(parent=None).exclude(**{field_name: ""})
262         paths = [(pretty_file_name(b), getattr(b, field_name).path) for b in books.iterator()]
263         return create_zip(paths, app_settings.FORMAT_ZIPS[format_])
264
265     def zip_audiobooks(self, format_):
266         bm = BookMedia.objects.filter(book=self, type=format_)
267         paths = map(lambda bm: (None, bm.file.path), bm)
268         return create_zip(paths, "%s_%s" % (self.slug, format_))
269
270     def search_index(self, book_info=None, index=None, index_tags=True, commit=True):
271         if index is None:
272             from search.index import Index
273             index = Index()
274         try:
275             index.index_book(self, book_info)
276             if index_tags:
277                 index.index_tags()
278             if commit:
279                 index.index.commit()
280         except Exception, e:
281             index.index.rollback()
282             raise e
283
284     def download_pictures(self, remote_gallery_url):
285         gallery_path = self.gallery_path()
286         # delete previous files, so we don't include old files in ebooks
287         if os.path.isdir(gallery_path):
288             for filename in os.listdir(gallery_path):
289                 file_path = os.path.join(gallery_path, filename)
290                 os.unlink(file_path)
291         ilustr_elements = list(self.wldocument().edoc.findall('//ilustr'))
292         if ilustr_elements:
293             makedirs(gallery_path)
294             for ilustr in ilustr_elements:
295                 ilustr_src = ilustr.get('src')
296                 ilustr_path = os.path.join(gallery_path, ilustr_src)
297                 urllib.urlretrieve('%s/%s' % (remote_gallery_url, ilustr_src), ilustr_path)
298
299     @classmethod
300     def from_xml_file(cls, xml_file, **kwargs):
301         from django.core.files import File
302         from librarian import dcparser
303
304         # use librarian to parse meta-data
305         book_info = dcparser.parse(xml_file)
306
307         if not isinstance(xml_file, File):
308             xml_file = File(open(xml_file))
309
310         try:
311             return cls.from_text_and_meta(xml_file, book_info, **kwargs)
312         finally:
313             xml_file.close()
314
315     @classmethod
316     def from_text_and_meta(cls, raw_file, book_info, overwrite=False, dont_build=None, search_index=True,
317                            search_index_tags=True, remote_gallery_url=None):
318         if dont_build is None:
319             dont_build = set()
320         dont_build = set.union(set(dont_build), set(app_settings.DONT_BUILD))
321
322         # check for parts before we do anything
323         children = []
324         if hasattr(book_info, 'parts'):
325             for part_url in book_info.parts:
326                 try:
327                     children.append(Book.objects.get(slug=part_url.slug))
328                 except Book.DoesNotExist:
329                     raise Book.DoesNotExist(_('Book "%s" does not exist.') % part_url.slug)
330
331         # Read book metadata
332         book_slug = book_info.url.slug
333         if re.search(r'[^a-z0-9-]', book_slug):
334             raise ValueError('Invalid characters in slug')
335         book, created = Book.objects.get_or_create(slug=book_slug)
336
337         if created:
338             book_shelves = []
339             old_cover = None
340         else:
341             if not overwrite:
342                 raise Book.AlreadyExists(_('Book %s already exists') % book_slug)
343             # Save shelves for this book
344             book_shelves = list(book.tags.filter(category='set'))
345             old_cover = book.cover_info()
346
347         # Save XML file
348         book.xml_file.save('%s.xml' % book.slug, raw_file, save=False)
349
350         book.language = book_info.language
351         book.title = book_info.title
352         if book_info.variant_of:
353             book.common_slug = book_info.variant_of.slug
354         else:
355             book.common_slug = book.slug
356         book.extra_info = book_info.to_dict()
357         book.save()
358
359         meta_tags = Tag.tags_from_info(book_info)
360
361         book.tags = set(meta_tags + book_shelves)
362
363         cover_changed = old_cover != book.cover_info()
364         obsolete_children = set(b for b in book.children.all()
365                                 if b not in children)
366         notify_cover_changed = []
367         for n, child_book in enumerate(children):
368             new_child = child_book.parent != book
369             child_book.parent = book
370             child_book.parent_number = n
371             child_book.save()
372             if new_child or cover_changed:
373                 notify_cover_changed.append(child_book)
374         # Disown unfaithful children and let them cope on their own.
375         for child in obsolete_children:
376             child.parent = None
377             child.parent_number = 0
378             child.save()
379             if old_cover:
380                 notify_cover_changed.append(child)
381
382         cls.repopulate_ancestors()
383         tasks.update_counters.delay()
384
385         if remote_gallery_url:
386             book.download_pictures(remote_gallery_url)
387
388         # No saves beyond this point.
389
390         # Build cover.
391         if 'cover' not in dont_build:
392             book.cover.build_delay()
393             book.cover_thumb.build_delay()
394
395         # Build HTML and ebooks.
396         book.html_file.build_delay()
397         if not children:
398             for format_ in constants.EBOOK_FORMATS_WITHOUT_CHILDREN:
399                 if format_ not in dont_build:
400                     getattr(book, '%s_file' % format_).build_delay()
401         for format_ in constants.EBOOK_FORMATS_WITH_CHILDREN:
402             if format_ not in dont_build:
403                 getattr(book, '%s_file' % format_).build_delay()
404
405         if not settings.NO_SEARCH_INDEX and search_index:
406             tasks.index_book.delay(book.id, book_info=book_info, index_tags=search_index_tags)
407
408         for child in notify_cover_changed:
409             child.parent_cover_changed()
410
411         book.save()  # update sort_key_author
412         cls.published.send(sender=cls, instance=book)
413         return book
414
415     @classmethod
416     @transaction.atomic
417     def repopulate_ancestors(cls):
418         """Fixes the ancestry cache."""
419         # TODO: table names
420         cursor = connection.cursor()
421         if connection.vendor == 'postgres':
422             cursor.execute("TRUNCATE catalogue_book_ancestor")
423             cursor.execute("""
424                 WITH RECURSIVE ancestry AS (
425                     SELECT book.id, book.parent_id
426                     FROM catalogue_book AS book
427                     WHERE book.parent_id IS NOT NULL
428                     UNION
429                     SELECT ancestor.id, book.parent_id
430                     FROM ancestry AS ancestor, catalogue_book AS book
431                     WHERE ancestor.parent_id = book.id
432                         AND book.parent_id IS NOT NULL
433                     )
434                 INSERT INTO catalogue_book_ancestor
435                     (from_book_id, to_book_id)
436                     SELECT id, parent_id
437                     FROM ancestry
438                     ORDER BY id;
439                 """)
440         else:
441             cursor.execute("DELETE FROM catalogue_book_ancestor")
442             for b in cls.objects.exclude(parent=None):
443                 parent = b.parent
444                 while parent is not None:
445                     b.ancestor.add(parent)
446                     parent = parent.parent
447
448     def flush_includes(self, languages=True):
449         if not languages:
450             return
451         if languages is True:
452             languages = [lc for (lc, _ln) in settings.LANGUAGES]
453         flush_ssi_includes([
454             template % (self.pk, lang)
455             for template in [
456                 '/katalog/b/%d/mini.%s.html',
457                 '/katalog/b/%d/mini_nolink.%s.html',
458                 '/katalog/b/%d/short.%s.html',
459                 '/katalog/b/%d/wide.%s.html',
460                 '/api/include/book/%d.%s.json',
461                 '/api/include/book/%d.%s.xml',
462                 ]
463             for lang in languages
464             ])
465
466     def cover_info(self, inherit=True):
467         """Returns a dictionary to serve as fallback for BookInfo.
468
469         For now, the only thing inherited is the cover image.
470         """
471         need = False
472         info = {}
473         for field in ('cover_url', 'cover_by', 'cover_source'):
474             val = self.extra_info.get(field)
475             if val:
476                 info[field] = val
477             else:
478                 need = True
479         if inherit and need and self.parent is not None:
480             parent_info = self.parent.cover_info()
481             parent_info.update(info)
482             info = parent_info
483         return info
484
485     def related_themes(self):
486         return Tag.objects.usage_for_queryset(
487             Fragment.objects.filter(models.Q(book=self) | models.Q(book__ancestor=self)),
488             counts=True).filter(category='theme')
489
490     def parent_cover_changed(self):
491         """Called when parent book's cover image is changed."""
492         if not self.cover_info(inherit=False):
493             if 'cover' not in app_settings.DONT_BUILD:
494                 self.cover.build_delay()
495                 self.cover_thumb.build_delay()
496             for format_ in constants.EBOOK_FORMATS_WITH_COVERS:
497                 if format_ not in app_settings.DONT_BUILD:
498                     getattr(self, '%s_file' % format_).build_delay()
499             for child in self.children.all():
500                 child.parent_cover_changed()
501
502     def other_versions(self):
503         """Find other versions (i.e. in other languages) of the book."""
504         return type(self).objects.filter(common_slug=self.common_slug).exclude(pk=self.pk)
505
506     def parents(self):
507         books = []
508         parent = self.parent
509         while parent is not None:
510             books.insert(0, parent)
511             parent = parent.parent
512         return books
513
514     def pretty_title(self, html_links=False):
515         names = [(tag.name, tag.get_absolute_url()) for tag in self.authors().only('name', 'category', 'slug')]
516         books = self.parents() + [self]
517         names.extend([(b.title, b.get_absolute_url()) for b in books])
518
519         if html_links:
520             names = ['<a href="%s">%s</a>' % (tag[1], tag[0]) for tag in names]
521         else:
522             names = [tag[0] for tag in names]
523         return ', '.join(names)
524
525     def publisher(self):
526         publisher = self.extra_info['publisher']
527         if isinstance(publisher, basestring):
528             return publisher
529         elif isinstance(publisher, list):
530             return ', '.join(publisher)
531
532     @classmethod
533     def tagged_top_level(cls, tags):
534         """ Returns top-level books tagged with `tags`.
535
536         It only returns those books which don't have ancestors which are
537         also tagged with those tags.
538
539         """
540         objects = cls.tagged.with_all(tags)
541         return objects.exclude(ancestor__in=objects)
542
543     @classmethod
544     def book_list(cls, book_filter=None):
545         """Generates a hierarchical listing of all books.
546
547         Books are optionally filtered with a test function.
548
549         """
550
551         books_by_parent = {}
552         books = cls.objects.order_by('parent_number', 'sort_key').only('title', 'parent', 'slug')
553         if book_filter:
554             books = books.filter(book_filter).distinct()
555
556             book_ids = set(b['pk'] for b in books.values("pk").iterator())
557             for book in books.iterator():
558                 parent = book.parent_id
559                 if parent not in book_ids:
560                     parent = None
561                 books_by_parent.setdefault(parent, []).append(book)
562         else:
563             for book in books.iterator():
564                 books_by_parent.setdefault(book.parent_id, []).append(book)
565
566         orphans = []
567         books_by_author = OrderedDict()
568         for tag in Tag.objects.filter(category='author').iterator():
569             books_by_author[tag] = []
570
571         for book in books_by_parent.get(None, ()):
572             authors = list(book.authors().only('pk'))
573             if authors:
574                 for author in authors:
575                     books_by_author[author].append(book)
576             else:
577                 orphans.append(book)
578
579         return books_by_author, orphans, books_by_parent
580
581     _audiences_pl = {
582         "SP": (1, u"szkoła podstawowa"),
583         "SP1": (1, u"szkoła podstawowa"),
584         "SP2": (1, u"szkoła podstawowa"),
585         "SP3": (1, u"szkoła podstawowa"),
586         "P": (1, u"szkoła podstawowa"),
587         "G": (2, u"gimnazjum"),
588         "L": (3, u"liceum"),
589         "LP": (3, u"liceum"),
590     }
591
592     def audiences_pl(self):
593         audiences = self.extra_info.get('audiences', [])
594         audiences = sorted(set([self._audiences_pl.get(a, (99, a)) for a in audiences]))
595         return [a[1] for a in audiences]
596
597     def stage_note(self):
598         stage = self.extra_info.get('stage')
599         if stage and stage < '0.4':
600             return (_('This work needs modernisation'),
601                     reverse('infopage', args=['wymagajace-uwspolczesnienia']))
602         else:
603             return None, None
604
605     def choose_fragment(self):
606         fragments = self.fragments.order_by()
607         fragments_count = fragments.count()
608         if not fragments_count and self.children.exists():
609             fragments = Fragment.objects.filter(book__ancestor=self).order_by()
610             fragments_count = fragments.count()
611         if fragments_count:
612             return fragments[randint(0, fragments_count - 1)]
613         elif self.parent:
614             return self.parent.choose_fragment()
615         else:
616             return None
617
618     def update_popularity(self):
619         count = self.tags.filter(category='set').values('user').order_by('user').distinct().count()
620         try:
621             pop = self.popularity
622             pop.count = count
623             pop.save()
624         except BookPopularity.DoesNotExist:
625             BookPopularity.objects.create(book=self, count=count)
626
627     def ridero_link(self):
628         return 'https://ridero.eu/%s/books/wl_%s/' % (get_language(), self.slug.replace('-', '_'))
629
630
631 def add_file_fields():
632     for format_ in Book.formats:
633         field_name = "%s_file" % format_
634         # This weird globals() assignment makes Django migrations comfortable.
635         _upload_to = _ebook_upload_to('book/%s/%%s.%s' % (format_, format_))
636         _upload_to.__name__ = '_%s_upload_to' % format_
637         globals()[_upload_to.__name__] = _upload_to
638
639         EbookField(
640             format_, _("%s file" % format_.upper()),
641             upload_to=_upload_to,
642             storage=bofh_storage,
643             max_length=255,
644             blank=True,
645             default=''
646         ).contribute_to_class(Book, field_name)
647
648 add_file_fields()
649
650
651 class BookPopularity(models.Model):
652     book = models.OneToOneField(Book, related_name='popularity')
653     count = models.IntegerField(default=0)