translator in book box
[wolnelektury.git] / src / catalogue / models / book.py
1 # -*- coding: utf-8 -*-
2 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
3 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
4 #
5 from collections import OrderedDict
6 from random import randint
7 import os.path
8 import re
9 import urllib
10 from django.conf import settings
11 from django.db import connection, models, transaction
12 from django.db.models import permalink
13 import django.dispatch
14 from django.contrib.contenttypes.fields import GenericRelation
15 from django.core.urlresolvers import reverse
16 from django.utils.translation import ugettext_lazy as _, get_language
17 import jsonfield
18 from fnpdjango.storage import BofhFileSystemStorage
19 from ssify import flush_ssi_includes
20 from newtagging import managers
21 from catalogue import constants
22 from catalogue.fields import EbookField
23 from catalogue.models import Tag, Fragment, BookMedia
24 from catalogue.utils import create_zip, gallery_url, gallery_path
25 from catalogue.models.tag import prefetched_relations
26 from catalogue import app_settings
27 from catalogue import tasks
28 from wolnelektury.utils import makedirs
29
30 bofh_storage = BofhFileSystemStorage()
31
32
33 def _make_upload_to(path):
34     def _upload_to(i, n):
35         return path % i.slug
36     return _upload_to
37
38
39 _cover_upload_to = _make_upload_to('book/cover/%s.jpg')
40 _cover_thumb_upload_to = _make_upload_to('book/cover_thumb/%s.jpg')
41
42
43 def _ebook_upload_to(upload_path):
44     return _make_upload_to(upload_path)
45
46
47 class Book(models.Model):
48     """Represents a book imported from WL-XML."""
49     title = models.CharField(_('title'), max_length=32767)
50     sort_key = models.CharField(_('sort key'), max_length=120, db_index=True, editable=False)
51     sort_key_author = models.CharField(
52         _('sort key by author'), max_length=120, db_index=True, editable=False, default=u'')
53     slug = models.SlugField(_('slug'), max_length=120, db_index=True, unique=True)
54     common_slug = models.SlugField(_('slug'), max_length=120, db_index=True)
55     language = models.CharField(_('language code'), max_length=3, db_index=True, default=app_settings.DEFAULT_LANGUAGE)
56     description = models.TextField(_('description'), blank=True)
57     created_at = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True)
58     changed_at = models.DateTimeField(_('creation date'), auto_now=True, db_index=True)
59     parent_number = models.IntegerField(_('parent number'), default=0)
60     extra_info = jsonfield.JSONField(_('extra information'), default={})
61     gazeta_link = models.CharField(blank=True, max_length=240)
62     wiki_link = models.CharField(blank=True, max_length=240)
63     print_on_demand = models.BooleanField(_('print on demand'), default=False)
64
65     # files generated during publication
66     cover = EbookField(
67         'cover', _('cover'),
68         null=True, blank=True,
69         upload_to=_cover_upload_to,
70         storage=bofh_storage, max_length=255)
71     # Cleaner version of cover for thumbs
72     cover_thumb = EbookField(
73         'cover_thumb', _('cover thumbnail'),
74         null=True, blank=True,
75         upload_to=_cover_thumb_upload_to,
76         max_length=255)
77     ebook_formats = constants.EBOOK_FORMATS
78     formats = ebook_formats + ['html', 'xml']
79
80     parent = models.ForeignKey('self', blank=True, null=True, related_name='children')
81     ancestor = models.ManyToManyField('self', blank=True, editable=False, related_name='descendant', symmetrical=False)
82
83     objects = models.Manager()
84     tagged = managers.ModelTaggedItemManager(Tag)
85     tags = managers.TagDescriptor(Tag)
86     tag_relations = GenericRelation(Tag.intermediary_table_model)
87
88     html_built = django.dispatch.Signal()
89     published = django.dispatch.Signal()
90
91     short_html_url_name = 'catalogue_book_short'
92
93     class AlreadyExists(Exception):
94         pass
95
96     class Meta:
97         ordering = ('sort_key_author', 'sort_key')
98         verbose_name = _('book')
99         verbose_name_plural = _('books')
100         app_label = 'catalogue'
101
102     def __unicode__(self):
103         return self.title
104
105     def get_initial(self):
106         try:
107             return re.search(r'\w', self.title, re.U).group(0)
108         except AttributeError:
109             return ''
110
111     def authors(self):
112         return self.tags.filter(category='author')
113
114     def tag_unicode(self, category):
115         relations = prefetched_relations(self, category)
116         if relations:
117             return ', '.join(rel.tag.name for rel in relations)
118         else:
119             return ', '.join(self.tags.filter(category=category).values_list('name', flat=True))
120
121     def author_unicode(self):
122         return self.tag_unicode('author')
123
124     def translator(self):
125         translators = self.extra_info.get('translators')
126         if not translators:
127             return None
128         if len(translators) > 3:
129             translators = translators[:2]
130             others = ' i inni'
131         else:
132             others = ''
133         return ', '.join(u'\xa0'.join(reversed(translator.split(', ', 1))) for translator in translators) + others
134
135     def save(self, force_insert=False, force_update=False, **kwargs):
136         from sortify import sortify
137
138         self.sort_key = sortify(self.title)[:120]
139         self.title = unicode(self.title)  # ???
140
141         try:
142             author = self.authors().first().sort_key
143         except AttributeError:
144             author = u''
145         self.sort_key_author = author
146
147         ret = super(Book, self).save(force_insert, force_update, **kwargs)
148
149         return ret
150
151     @permalink
152     def get_absolute_url(self):
153         return 'catalogue.views.book_detail', [self.slug]
154
155     @staticmethod
156     @permalink
157     def create_url(slug):
158         return 'catalogue.views.book_detail', [slug]
159
160     def gallery_path(self):
161         return gallery_path(self.slug)
162
163     def gallery_url(self):
164         return gallery_url(self.slug)
165
166     @property
167     def name(self):
168         return self.title
169
170     def language_code(self):
171         return constants.LANGUAGES_3TO2.get(self.language, self.language)
172
173     def language_name(self):
174         return dict(settings.LANGUAGES).get(self.language_code(), "")
175
176     def is_foreign(self):
177         return self.language_code() != settings.LANGUAGE_CODE
178
179     def has_media(self, type_):
180         if type_ in Book.formats:
181             return bool(getattr(self, "%s_file" % type_))
182         else:
183             return self.media.filter(type=type_).exists()
184
185     def get_media(self, type_):
186         if self.has_media(type_):
187             if type_ in Book.formats:
188                 return getattr(self, "%s_file" % type_)
189             else:
190                 return self.media.filter(type=type_)
191         else:
192             return None
193
194     def get_mp3(self):
195         return self.get_media("mp3")
196
197     def get_odt(self):
198         return self.get_media("odt")
199
200     def get_ogg(self):
201         return self.get_media("ogg")
202
203     def get_daisy(self):
204         return self.get_media("daisy")
205
206     def has_description(self):
207         return len(self.description) > 0
208     has_description.short_description = _('description')
209     has_description.boolean = True
210
211     # ugly ugly ugly
212     def has_mp3_file(self):
213         return bool(self.has_media("mp3"))
214     has_mp3_file.short_description = 'MP3'
215     has_mp3_file.boolean = True
216
217     def has_ogg_file(self):
218         return bool(self.has_media("ogg"))
219     has_ogg_file.short_description = 'OGG'
220     has_ogg_file.boolean = True
221
222     def has_daisy_file(self):
223         return bool(self.has_media("daisy"))
224     has_daisy_file.short_description = 'DAISY'
225     has_daisy_file.boolean = True
226
227     def wldocument(self, parse_dublincore=True, inherit=True):
228         from catalogue.import_utils import ORMDocProvider
229         from librarian.parser import WLDocument
230
231         if inherit and self.parent:
232             meta_fallbacks = self.parent.cover_info()
233         else:
234             meta_fallbacks = None
235
236         return WLDocument.from_file(
237             self.xml_file.path,
238             provider=ORMDocProvider(self),
239             parse_dublincore=parse_dublincore,
240             meta_fallbacks=meta_fallbacks)
241
242     @staticmethod
243     def zip_format(format_):
244         def pretty_file_name(book):
245             return "%s/%s.%s" % (
246                 book.extra_info['author'],
247                 book.slug,
248                 format_)
249
250         field_name = "%s_file" % format_
251         books = Book.objects.filter(parent=None).exclude(**{field_name: ""})
252         paths = [(pretty_file_name(b), getattr(b, field_name).path) for b in books.iterator()]
253         return create_zip(paths, app_settings.FORMAT_ZIPS[format_])
254
255     def zip_audiobooks(self, format_):
256         bm = BookMedia.objects.filter(book=self, type=format_)
257         paths = map(lambda bm: (None, bm.file.path), bm)
258         return create_zip(paths, "%s_%s" % (self.slug, format_))
259
260     def search_index(self, book_info=None, index=None, index_tags=True, commit=True):
261         if index is None:
262             from search.index import Index
263             index = Index()
264         try:
265             index.index_book(self, book_info)
266             if index_tags:
267                 index.index_tags()
268             if commit:
269                 index.index.commit()
270         except Exception, e:
271             index.index.rollback()
272             raise e
273
274     def download_pictures(self, remote_gallery_url):
275         gallery_path = self.gallery_path()
276         # delete previous files, so we don't include old files in ebooks
277         if os.path.isdir(gallery_path):
278             for filename in os.listdir(gallery_path):
279                 file_path = os.path.join(gallery_path, filename)
280                 os.unlink(file_path)
281         ilustr_elements = list(self.wldocument().edoc.findall('//ilustr'))
282         if ilustr_elements:
283             makedirs(gallery_path)
284             for ilustr in ilustr_elements:
285                 ilustr_src = ilustr.get('src')
286                 ilustr_path = os.path.join(gallery_path, ilustr_src)
287                 urllib.urlretrieve('%s/%s' % (remote_gallery_url, ilustr_src), ilustr_path)
288
289     @classmethod
290     def from_xml_file(cls, xml_file, **kwargs):
291         from django.core.files import File
292         from librarian import dcparser
293
294         # use librarian to parse meta-data
295         book_info = dcparser.parse(xml_file)
296
297         if not isinstance(xml_file, File):
298             xml_file = File(open(xml_file))
299
300         try:
301             return cls.from_text_and_meta(xml_file, book_info, **kwargs)
302         finally:
303             xml_file.close()
304
305     @classmethod
306     def from_text_and_meta(cls, raw_file, book_info, overwrite=False, dont_build=None, search_index=True,
307                            search_index_tags=True, remote_gallery_url=None):
308         if dont_build is None:
309             dont_build = set()
310         dont_build = set.union(set(dont_build), set(app_settings.DONT_BUILD))
311
312         # check for parts before we do anything
313         children = []
314         if hasattr(book_info, 'parts'):
315             for part_url in book_info.parts:
316                 try:
317                     children.append(Book.objects.get(slug=part_url.slug))
318                 except Book.DoesNotExist:
319                     raise Book.DoesNotExist(_('Book "%s" does not exist.') % part_url.slug)
320
321         # Read book metadata
322         book_slug = book_info.url.slug
323         if re.search(r'[^a-z0-9-]', book_slug):
324             raise ValueError('Invalid characters in slug')
325         book, created = Book.objects.get_or_create(slug=book_slug)
326
327         if created:
328             book_shelves = []
329             old_cover = None
330         else:
331             if not overwrite:
332                 raise Book.AlreadyExists(_('Book %s already exists') % book_slug)
333             # Save shelves for this book
334             book_shelves = list(book.tags.filter(category='set'))
335             old_cover = book.cover_info()
336
337         # Save XML file
338         book.xml_file.save('%s.xml' % book.slug, raw_file, save=False)
339
340         book.language = book_info.language
341         book.title = book_info.title
342         if book_info.variant_of:
343             book.common_slug = book_info.variant_of.slug
344         else:
345             book.common_slug = book.slug
346         book.extra_info = book_info.to_dict()
347         book.save()
348
349         meta_tags = Tag.tags_from_info(book_info)
350
351         book.tags = set(meta_tags + book_shelves)
352
353         cover_changed = old_cover != book.cover_info()
354         obsolete_children = set(b for b in book.children.all()
355                                 if b not in children)
356         notify_cover_changed = []
357         for n, child_book in enumerate(children):
358             new_child = child_book.parent != book
359             child_book.parent = book
360             child_book.parent_number = n
361             child_book.save()
362             if new_child or cover_changed:
363                 notify_cover_changed.append(child_book)
364         # Disown unfaithful children and let them cope on their own.
365         for child in obsolete_children:
366             child.parent = None
367             child.parent_number = 0
368             child.save()
369             if old_cover:
370                 notify_cover_changed.append(child)
371
372         cls.repopulate_ancestors()
373         tasks.update_counters.delay()
374
375         if remote_gallery_url:
376             book.download_pictures(remote_gallery_url)
377
378         # No saves beyond this point.
379
380         # Build cover.
381         if 'cover' not in dont_build:
382             book.cover.build_delay()
383             book.cover_thumb.build_delay()
384
385         # Build HTML and ebooks.
386         book.html_file.build_delay()
387         if not children:
388             for format_ in constants.EBOOK_FORMATS_WITHOUT_CHILDREN:
389                 if format_ not in dont_build:
390                     getattr(book, '%s_file' % format_).build_delay()
391         for format_ in constants.EBOOK_FORMATS_WITH_CHILDREN:
392             if format_ not in dont_build:
393                 getattr(book, '%s_file' % format_).build_delay()
394
395         if not settings.NO_SEARCH_INDEX and search_index:
396             tasks.index_book.delay(book.id, book_info=book_info, index_tags=search_index_tags)
397
398         for child in notify_cover_changed:
399             child.parent_cover_changed()
400
401         book.save()  # update sort_key_author
402         cls.published.send(sender=cls, instance=book)
403         return book
404
405     @classmethod
406     @transaction.atomic
407     def repopulate_ancestors(cls):
408         """Fixes the ancestry cache."""
409         # TODO: table names
410         cursor = connection.cursor()
411         if connection.vendor == 'postgres':
412             cursor.execute("TRUNCATE catalogue_book_ancestor")
413             cursor.execute("""
414                 WITH RECURSIVE ancestry AS (
415                     SELECT book.id, book.parent_id
416                     FROM catalogue_book AS book
417                     WHERE book.parent_id IS NOT NULL
418                     UNION
419                     SELECT ancestor.id, book.parent_id
420                     FROM ancestry AS ancestor, catalogue_book AS book
421                     WHERE ancestor.parent_id = book.id
422                         AND book.parent_id IS NOT NULL
423                     )
424                 INSERT INTO catalogue_book_ancestor
425                     (from_book_id, to_book_id)
426                     SELECT id, parent_id
427                     FROM ancestry
428                     ORDER BY id;
429                 """)
430         else:
431             cursor.execute("DELETE FROM catalogue_book_ancestor")
432             for b in cls.objects.exclude(parent=None):
433                 parent = b.parent
434                 while parent is not None:
435                     b.ancestor.add(parent)
436                     parent = parent.parent
437
438     def flush_includes(self, languages=True):
439         if not languages:
440             return
441         if languages is True:
442             languages = [lc for (lc, _ln) in settings.LANGUAGES]
443         flush_ssi_includes([
444             template % (self.pk, lang)
445             for template in [
446                 '/katalog/b/%d/mini.%s.html',
447                 '/katalog/b/%d/mini_nolink.%s.html',
448                 '/katalog/b/%d/short.%s.html',
449                 '/katalog/b/%d/wide.%s.html',
450                 '/api/include/book/%d.%s.json',
451                 '/api/include/book/%d.%s.xml',
452                 ]
453             for lang in languages
454             ])
455
456     def cover_info(self, inherit=True):
457         """Returns a dictionary to serve as fallback for BookInfo.
458
459         For now, the only thing inherited is the cover image.
460         """
461         need = False
462         info = {}
463         for field in ('cover_url', 'cover_by', 'cover_source'):
464             val = self.extra_info.get(field)
465             if val:
466                 info[field] = val
467             else:
468                 need = True
469         if inherit and need and self.parent is not None:
470             parent_info = self.parent.cover_info()
471             parent_info.update(info)
472             info = parent_info
473         return info
474
475     def related_themes(self):
476         return Tag.objects.usage_for_queryset(
477             Fragment.objects.filter(models.Q(book=self) | models.Q(book__ancestor=self)),
478             counts=True).filter(category='theme')
479
480     def parent_cover_changed(self):
481         """Called when parent book's cover image is changed."""
482         if not self.cover_info(inherit=False):
483             if 'cover' not in app_settings.DONT_BUILD:
484                 self.cover.build_delay()
485                 self.cover_thumb.build_delay()
486             for format_ in constants.EBOOK_FORMATS_WITH_COVERS:
487                 if format_ not in app_settings.DONT_BUILD:
488                     getattr(self, '%s_file' % format_).build_delay()
489             for child in self.children.all():
490                 child.parent_cover_changed()
491
492     def other_versions(self):
493         """Find other versions (i.e. in other languages) of the book."""
494         return type(self).objects.filter(common_slug=self.common_slug).exclude(pk=self.pk)
495
496     def parents(self):
497         books = []
498         parent = self.parent
499         while parent is not None:
500             books.insert(0, parent)
501             parent = parent.parent
502         return books
503
504     def pretty_title(self, html_links=False):
505         names = [(tag.name, tag.get_absolute_url()) for tag in self.authors().only('name', 'category', 'slug')]
506         books = self.parents() + [self]
507         names.extend([(b.title, b.get_absolute_url()) for b in books])
508
509         if html_links:
510             names = ['<a href="%s">%s</a>' % (tag[1], tag[0]) for tag in names]
511         else:
512             names = [tag[0] for tag in names]
513         return ', '.join(names)
514
515     def publisher(self):
516         publisher = self.extra_info['publisher']
517         if isinstance(publisher, basestring):
518             return publisher
519         elif isinstance(publisher, list):
520             return ', '.join(publisher)
521
522     @classmethod
523     def tagged_top_level(cls, tags):
524         """ Returns top-level books tagged with `tags`.
525
526         It only returns those books which don't have ancestors which are
527         also tagged with those tags.
528
529         """
530         objects = cls.tagged.with_all(tags)
531         return objects.exclude(ancestor__in=objects)
532
533     @classmethod
534     def book_list(cls, book_filter=None):
535         """Generates a hierarchical listing of all books.
536
537         Books are optionally filtered with a test function.
538
539         """
540
541         books_by_parent = {}
542         books = cls.objects.order_by('parent_number', 'sort_key').only('title', 'parent', 'slug')
543         if book_filter:
544             books = books.filter(book_filter).distinct()
545
546             book_ids = set(b['pk'] for b in books.values("pk").iterator())
547             for book in books.iterator():
548                 parent = book.parent_id
549                 if parent not in book_ids:
550                     parent = None
551                 books_by_parent.setdefault(parent, []).append(book)
552         else:
553             for book in books.iterator():
554                 books_by_parent.setdefault(book.parent_id, []).append(book)
555
556         orphans = []
557         books_by_author = OrderedDict()
558         for tag in Tag.objects.filter(category='author').iterator():
559             books_by_author[tag] = []
560
561         for book in books_by_parent.get(None, ()):
562             authors = list(book.authors().only('pk'))
563             if authors:
564                 for author in authors:
565                     books_by_author[author].append(book)
566             else:
567                 orphans.append(book)
568
569         return books_by_author, orphans, books_by_parent
570
571     _audiences_pl = {
572         "SP": (1, u"szkoła podstawowa"),
573         "SP1": (1, u"szkoła podstawowa"),
574         "SP2": (1, u"szkoła podstawowa"),
575         "P": (1, u"szkoła podstawowa"),
576         "G": (2, u"gimnazjum"),
577         "L": (3, u"liceum"),
578         "LP": (3, u"liceum"),
579     }
580
581     def audiences_pl(self):
582         audiences = self.extra_info.get('audiences', [])
583         audiences = sorted(set([self._audiences_pl.get(a, (99, a)) for a in audiences]))
584         return [a[1] for a in audiences]
585
586     def stage_note(self):
587         stage = self.extra_info.get('stage')
588         if stage and stage < '0.4':
589             return (_('This work needs modernisation'),
590                     reverse('infopage', args=['wymagajace-uwspolczesnienia']))
591         else:
592             return None, None
593
594     def choose_fragment(self):
595         fragments = self.fragments.order_by()
596         fragments_count = fragments.count()
597         if not fragments_count and self.children.exists():
598             fragments = Fragment.objects.filter(book__ancestor=self).order_by()
599             fragments_count = fragments.count()
600         if fragments_count:
601             return fragments[randint(0, fragments_count - 1)]
602         elif self.parent:
603             return self.parent.choose_fragment()
604         else:
605             return None
606
607     def update_popularity(self):
608         count = self.tags.filter(category='set').values('user').order_by('user').distinct().count()
609         try:
610             pop = self.popularity
611             pop.count = count
612             pop.save()
613         except BookPopularity.DoesNotExist:
614             BookPopularity.objects.create(book=self, count=count)
615
616     def ridero_link(self):
617         return 'https://ridero.eu/%s/books/wl_%s/' % (get_language(), self.slug.replace('-', '_'))
618
619
620 def add_file_fields():
621     for format_ in Book.formats:
622         field_name = "%s_file" % format_
623         # This weird globals() assignment makes Django migrations comfortable.
624         _upload_to = _ebook_upload_to('book/%s/%%s.%s' % (format_, format_))
625         _upload_to.__name__ = '_%s_upload_to' % format_
626         globals()[_upload_to.__name__] = _upload_to
627
628         EbookField(
629             format_, _("%s file" % format_.upper()),
630             upload_to=_upload_to,
631             storage=bofh_storage,
632             max_length=255,
633             blank=True,
634             default=''
635         ).contribute_to_class(Book, field_name)
636
637 add_file_fields()
638
639
640 class BookPopularity(models.Model):
641     book = models.OneToOneField(Book, related_name='popularity')
642     count = models.IntegerField(default=0)