1 # -*- coding: utf-8 -*-
2 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
3 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
5 from collections import OrderedDict
6 from random import randint
10 from django.conf import settings
11 from django.db import connection, models, transaction
12 from django.db.models import permalink
13 import django.dispatch
14 from django.contrib.contenttypes.fields import GenericRelation
15 from django.core.urlresolvers import reverse
16 from django.utils.translation import ugettext_lazy as _, get_language
17 from django.utils.deconstruct import deconstructible
19 from fnpdjango.storage import BofhFileSystemStorage
20 from ssify import flush_ssi_includes
21 from newtagging import managers
22 from catalogue import constants
23 from catalogue.fields import EbookField
24 from catalogue.models import Tag, Fragment, BookMedia
25 from catalogue.utils import create_zip, gallery_url, gallery_path, split_tags
26 from catalogue.models.tag import prefetched_relations
27 from catalogue import app_settings
28 from catalogue import tasks
29 from wolnelektury.utils import makedirs
31 bofh_storage = BofhFileSystemStorage()
35 class UploadToPath(object):
36 def __init__(self, path):
39 def __call__(self, instance, filename):
40 return self.path % instance.slug
43 _cover_upload_to = UploadToPath('book/cover/%s.jpg')
44 _cover_thumb_upload_to = UploadToPath('book/cover_thumb/%s.jpg')
45 _cover_api_thumb_upload_to = UploadToPath('book/cover_api_thumb/%s.jpg')
46 _simple_cover_upload_to = UploadToPath('book/cover_simple/%s.jpg')
49 def _ebook_upload_to(upload_path):
50 return UploadToPath(upload_path)
53 class Book(models.Model):
54 """Represents a book imported from WL-XML."""
55 title = models.CharField(_('title'), max_length=32767)
56 sort_key = models.CharField(_('sort key'), max_length=120, db_index=True, editable=False)
57 sort_key_author = models.CharField(
58 _('sort key by author'), max_length=120, db_index=True, editable=False, default=u'')
59 slug = models.SlugField(_('slug'), max_length=120, db_index=True, unique=True)
60 common_slug = models.SlugField(_('slug'), max_length=120, db_index=True)
61 language = models.CharField(_('language code'), max_length=3, db_index=True, default=app_settings.DEFAULT_LANGUAGE)
62 description = models.TextField(_('description'), blank=True)
63 created_at = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True)
64 changed_at = models.DateTimeField(_('change date'), auto_now=True, db_index=True)
65 parent_number = models.IntegerField(_('parent number'), default=0)
66 extra_info = jsonfield.JSONField(_('extra information'), default={})
67 gazeta_link = models.CharField(blank=True, max_length=240)
68 wiki_link = models.CharField(blank=True, max_length=240)
69 print_on_demand = models.BooleanField(_('print on demand'), default=False)
70 recommended = models.BooleanField(_('recommended'), default=False)
72 # files generated during publication
75 null=True, blank=True,
76 upload_to=_cover_upload_to,
77 storage=bofh_storage, max_length=255)
78 # Cleaner version of cover for thumbs
79 cover_thumb = EbookField(
80 'cover_thumb', _('cover thumbnail'),
81 null=True, blank=True,
82 upload_to=_cover_thumb_upload_to,
84 cover_api_thumb = EbookField(
85 'cover_api_thumb', _('cover thumbnail for mobile app'),
86 null=True, blank=True,
87 upload_to=_cover_api_thumb_upload_to,
89 simple_cover = EbookField(
90 'simple_cover', _('cover for mobile app'),
91 null=True, blank=True,
92 upload_to=_simple_cover_upload_to,
94 ebook_formats = constants.EBOOK_FORMATS
95 formats = ebook_formats + ['html', 'xml']
97 parent = models.ForeignKey('self', blank=True, null=True, related_name='children')
98 ancestor = models.ManyToManyField('self', blank=True, editable=False, related_name='descendant', symmetrical=False)
100 cached_author = models.CharField(blank=True, max_length=240, db_index=True)
101 has_audience = models.BooleanField(default=False)
103 objects = models.Manager()
104 tagged = managers.ModelTaggedItemManager(Tag)
105 tags = managers.TagDescriptor(Tag)
106 tag_relations = GenericRelation(Tag.intermediary_table_model)
108 html_built = django.dispatch.Signal()
109 published = django.dispatch.Signal()
111 short_html_url_name = 'catalogue_book_short'
113 class AlreadyExists(Exception):
117 ordering = ('sort_key_author', 'sort_key')
118 verbose_name = _('book')
119 verbose_name_plural = _('books')
120 app_label = 'catalogue'
122 def __unicode__(self):
125 def get_initial(self):
127 return re.search(r'\w', self.title, re.U).group(0)
128 except AttributeError:
132 return self.tags.filter(category='author')
134 def tag_unicode(self, category):
135 relations = prefetched_relations(self, category)
137 return ', '.join(rel.tag.name for rel in relations)
139 return ', '.join(self.tags.filter(category=category).values_list('name', flat=True))
141 def tags_by_category(self):
142 return split_tags(self.tags.exclude(category__in=('set', 'theme')))
144 def author_unicode(self):
145 return self.cached_author
147 def translator(self):
148 translators = self.extra_info.get('translators')
151 if len(translators) > 3:
152 translators = translators[:2]
156 return ', '.join(u'\xa0'.join(reversed(translator.split(', ', 1))) for translator in translators) + others
158 def cover_source(self):
159 return self.extra_info.get('cover_source', self.parent.cover_source() if self.parent else '')
161 def save(self, force_insert=False, force_update=False, **kwargs):
162 from sortify import sortify
164 self.sort_key = sortify(self.title)[:120]
165 self.title = unicode(self.title) # ???
168 author = self.authors().first().sort_key
169 except AttributeError:
171 self.sort_key_author = author
173 self.cached_author = self.tag_unicode('author')
174 self.has_audience = 'audience' in self.extra_info
176 ret = super(Book, self).save(force_insert, force_update, **kwargs)
181 def get_absolute_url(self):
182 return 'catalogue.views.book_detail', [self.slug]
186 def create_url(slug):
187 return 'catalogue.views.book_detail', [slug]
189 def gallery_path(self):
190 return gallery_path(self.slug)
192 def gallery_url(self):
193 return gallery_url(self.slug)
199 def language_code(self):
200 return constants.LANGUAGES_3TO2.get(self.language, self.language)
202 def language_name(self):
203 return dict(settings.LANGUAGES).get(self.language_code(), "")
205 def is_foreign(self):
206 return self.language_code() != settings.LANGUAGE_CODE
208 def has_media(self, type_):
209 if type_ in Book.formats:
210 return bool(getattr(self, "%s_file" % type_))
212 return self.media.filter(type=type_).exists()
215 return self.has_media('mp3')
217 def get_media(self, type_):
218 if self.has_media(type_):
219 if type_ in Book.formats:
220 return getattr(self, "%s_file" % type_)
222 return self.media.filter(type=type_)
227 return self.get_media("mp3")
230 return self.get_media("odt")
233 return self.get_media("ogg")
236 return self.get_media("daisy")
238 def has_description(self):
239 return len(self.description) > 0
240 has_description.short_description = _('description')
241 has_description.boolean = True
244 def has_mp3_file(self):
245 return bool(self.has_media("mp3"))
246 has_mp3_file.short_description = 'MP3'
247 has_mp3_file.boolean = True
249 def has_ogg_file(self):
250 return bool(self.has_media("ogg"))
251 has_ogg_file.short_description = 'OGG'
252 has_ogg_file.boolean = True
254 def has_daisy_file(self):
255 return bool(self.has_media("daisy"))
256 has_daisy_file.short_description = 'DAISY'
257 has_daisy_file.boolean = True
259 def get_audiobooks(self):
261 for m in self.media.filter(type='ogg').order_by().iterator():
262 ogg_files[m.name] = m
266 for mp3 in self.media.filter(type='mp3').iterator():
267 # ogg files are always from the same project
268 meta = mp3.extra_info
269 project = meta.get('project')
272 project = u'CzytamySłuchając'
274 projects.add((project, meta.get('funded_by', '')))
278 ogg = ogg_files.get(mp3.name)
281 audiobooks.append(media)
283 projects = sorted(projects)
284 return audiobooks, projects
286 def wldocument(self, parse_dublincore=True, inherit=True):
287 from catalogue.import_utils import ORMDocProvider
288 from librarian.parser import WLDocument
290 if inherit and self.parent:
291 meta_fallbacks = self.parent.cover_info()
293 meta_fallbacks = None
295 return WLDocument.from_file(
297 provider=ORMDocProvider(self),
298 parse_dublincore=parse_dublincore,
299 meta_fallbacks=meta_fallbacks)
302 def zip_format(format_):
303 def pretty_file_name(book):
304 return "%s/%s.%s" % (
305 book.extra_info['author'],
309 field_name = "%s_file" % format_
310 books = Book.objects.filter(parent=None).exclude(**{field_name: ""})
311 paths = [(pretty_file_name(b), getattr(b, field_name).path) for b in books.iterator()]
312 return create_zip(paths, app_settings.FORMAT_ZIPS[format_])
314 def zip_audiobooks(self, format_):
315 bm = BookMedia.objects.filter(book=self, type=format_)
316 paths = map(lambda bm: (None, bm.file.path), bm)
317 return create_zip(paths, "%s_%s" % (self.slug, format_))
319 def search_index(self, book_info=None, index=None, index_tags=True, commit=True):
321 from search.index import Index
324 index.index_book(self, book_info)
330 index.index.rollback()
333 def download_pictures(self, remote_gallery_url):
334 gallery_path = self.gallery_path()
335 # delete previous files, so we don't include old files in ebooks
336 if os.path.isdir(gallery_path):
337 for filename in os.listdir(gallery_path):
338 file_path = os.path.join(gallery_path, filename)
340 ilustr_elements = list(self.wldocument().edoc.findall('//ilustr'))
342 makedirs(gallery_path)
343 for ilustr in ilustr_elements:
344 ilustr_src = ilustr.get('src')
345 ilustr_path = os.path.join(gallery_path, ilustr_src)
346 urllib.urlretrieve('%s/%s' % (remote_gallery_url, ilustr_src), ilustr_path)
349 def from_xml_file(cls, xml_file, **kwargs):
350 from django.core.files import File
351 from librarian import dcparser
353 # use librarian to parse meta-data
354 book_info = dcparser.parse(xml_file)
356 if not isinstance(xml_file, File):
357 xml_file = File(open(xml_file))
360 return cls.from_text_and_meta(xml_file, book_info, **kwargs)
365 def from_text_and_meta(cls, raw_file, book_info, overwrite=False, dont_build=None, search_index=True,
366 search_index_tags=True, remote_gallery_url=None):
367 if dont_build is None:
369 dont_build = set.union(set(dont_build), set(app_settings.DONT_BUILD))
371 # check for parts before we do anything
373 if hasattr(book_info, 'parts'):
374 for part_url in book_info.parts:
376 children.append(Book.objects.get(slug=part_url.slug))
377 except Book.DoesNotExist:
378 raise Book.DoesNotExist(_('Book "%s" does not exist.') % part_url.slug)
381 book_slug = book_info.url.slug
382 if re.search(r'[^a-z0-9-]', book_slug):
383 raise ValueError('Invalid characters in slug')
384 book, created = Book.objects.get_or_create(slug=book_slug)
391 raise Book.AlreadyExists(_('Book %s already exists') % book_slug)
392 # Save shelves for this book
393 book_shelves = list(book.tags.filter(category='set'))
394 old_cover = book.cover_info()
397 book.xml_file.save('%s.xml' % book.slug, raw_file, save=False)
399 book.language = book_info.language
400 book.title = book_info.title
401 if book_info.variant_of:
402 book.common_slug = book_info.variant_of.slug
404 book.common_slug = book.slug
405 book.extra_info = book_info.to_dict()
408 meta_tags = Tag.tags_from_info(book_info)
410 for tag in meta_tags:
411 if not tag.for_books:
415 book.tags = set(meta_tags + book_shelves)
417 cover_changed = old_cover != book.cover_info()
418 obsolete_children = set(b for b in book.children.all()
419 if b not in children)
420 notify_cover_changed = []
421 for n, child_book in enumerate(children):
422 new_child = child_book.parent != book
423 child_book.parent = book
424 child_book.parent_number = n
426 if new_child or cover_changed:
427 notify_cover_changed.append(child_book)
428 # Disown unfaithful children and let them cope on their own.
429 for child in obsolete_children:
431 child.parent_number = 0
434 notify_cover_changed.append(child)
436 cls.repopulate_ancestors()
437 tasks.update_counters.delay()
439 if remote_gallery_url:
440 book.download_pictures(remote_gallery_url)
442 # No saves beyond this point.
445 if 'cover' not in dont_build:
446 book.cover.build_delay()
447 book.cover_thumb.build_delay()
448 book.cover_api_thumb.build_delay()
449 book.simple_cover.build_delay()
451 # Build HTML and ebooks.
452 book.html_file.build_delay()
454 for format_ in constants.EBOOK_FORMATS_WITHOUT_CHILDREN:
455 if format_ not in dont_build:
456 getattr(book, '%s_file' % format_).build_delay()
457 for format_ in constants.EBOOK_FORMATS_WITH_CHILDREN:
458 if format_ not in dont_build:
459 getattr(book, '%s_file' % format_).build_delay()
461 if not settings.NO_SEARCH_INDEX and search_index:
462 tasks.index_book.delay(book.id, book_info=book_info, index_tags=search_index_tags)
464 for child in notify_cover_changed:
465 child.parent_cover_changed()
467 book.save() # update sort_key_author
468 book.update_popularity()
469 cls.published.send(sender=cls, instance=book)
474 def repopulate_ancestors(cls):
475 """Fixes the ancestry cache."""
477 cursor = connection.cursor()
478 if connection.vendor == 'postgres':
479 cursor.execute("TRUNCATE catalogue_book_ancestor")
481 WITH RECURSIVE ancestry AS (
482 SELECT book.id, book.parent_id
483 FROM catalogue_book AS book
484 WHERE book.parent_id IS NOT NULL
486 SELECT ancestor.id, book.parent_id
487 FROM ancestry AS ancestor, catalogue_book AS book
488 WHERE ancestor.parent_id = book.id
489 AND book.parent_id IS NOT NULL
491 INSERT INTO catalogue_book_ancestor
492 (from_book_id, to_book_id)
498 cursor.execute("DELETE FROM catalogue_book_ancestor")
499 for b in cls.objects.exclude(parent=None):
501 while parent is not None:
502 b.ancestor.add(parent)
503 parent = parent.parent
505 def flush_includes(self, languages=True):
508 if languages is True:
509 languages = [lc for (lc, _ln) in settings.LANGUAGES]
511 template % (self.pk, lang)
513 '/katalog/b/%d/mini.%s.html',
514 '/katalog/b/%d/mini_nolink.%s.html',
515 '/katalog/b/%d/short.%s.html',
516 '/katalog/b/%d/wide.%s.html',
517 '/api/include/book/%d.%s.json',
518 '/api/include/book/%d.%s.xml',
520 for lang in languages
523 def cover_info(self, inherit=True):
524 """Returns a dictionary to serve as fallback for BookInfo.
526 For now, the only thing inherited is the cover image.
530 for field in ('cover_url', 'cover_by', 'cover_source'):
531 val = self.extra_info.get(field)
536 if inherit and need and self.parent is not None:
537 parent_info = self.parent.cover_info()
538 parent_info.update(info)
542 def related_themes(self):
543 return Tag.objects.usage_for_queryset(
544 Fragment.objects.filter(models.Q(book=self) | models.Q(book__ancestor=self)),
545 counts=True).filter(category='theme')
547 def parent_cover_changed(self):
548 """Called when parent book's cover image is changed."""
549 if not self.cover_info(inherit=False):
550 if 'cover' not in app_settings.DONT_BUILD:
551 self.cover.build_delay()
552 self.cover_thumb.build_delay()
553 self.cover_api_thumb.build_delay()
554 self.simple_cover.build_delay()
555 for format_ in constants.EBOOK_FORMATS_WITH_COVERS:
556 if format_ not in app_settings.DONT_BUILD:
557 getattr(self, '%s_file' % format_).build_delay()
558 for child in self.children.all():
559 child.parent_cover_changed()
561 def other_versions(self):
562 """Find other versions (i.e. in other languages) of the book."""
563 return type(self).objects.filter(common_slug=self.common_slug).exclude(pk=self.pk)
568 while parent is not None:
569 books.insert(0, parent)
570 parent = parent.parent
573 def pretty_title(self, html_links=False):
574 names = [(tag.name, tag.get_absolute_url()) for tag in self.authors().only('name', 'category', 'slug')]
575 books = self.parents() + [self]
576 names.extend([(b.title, b.get_absolute_url()) for b in books])
579 names = ['<a href="%s">%s</a>' % (tag[1], tag[0]) for tag in names]
581 names = [tag[0] for tag in names]
582 return ', '.join(names)
585 publisher = self.extra_info['publisher']
586 if isinstance(publisher, basestring):
588 elif isinstance(publisher, list):
589 return ', '.join(publisher)
592 def tagged_top_level(cls, tags):
593 """ Returns top-level books tagged with `tags`.
595 It only returns those books which don't have ancestors which are
596 also tagged with those tags.
599 objects = cls.tagged.with_all(tags)
600 return objects.exclude(ancestor__in=objects)
603 def book_list(cls, book_filter=None):
604 """Generates a hierarchical listing of all books.
606 Books are optionally filtered with a test function.
611 books = cls.objects.order_by('parent_number', 'sort_key').only('title', 'parent', 'slug')
613 books = books.filter(book_filter).distinct()
615 book_ids = set(b['pk'] for b in books.values("pk").iterator())
616 for book in books.iterator():
617 parent = book.parent_id
618 if parent not in book_ids:
620 books_by_parent.setdefault(parent, []).append(book)
622 for book in books.iterator():
623 books_by_parent.setdefault(book.parent_id, []).append(book)
626 books_by_author = OrderedDict()
627 for tag in Tag.objects.filter(category='author').iterator():
628 books_by_author[tag] = []
630 for book in books_by_parent.get(None, ()):
631 authors = list(book.authors().only('pk'))
633 for author in authors:
634 books_by_author[author].append(book)
638 return books_by_author, orphans, books_by_parent
641 "SP": (1, u"szkoła podstawowa"),
642 "SP1": (1, u"szkoła podstawowa"),
643 "SP2": (1, u"szkoła podstawowa"),
644 "SP3": (1, u"szkoła podstawowa"),
645 "P": (1, u"szkoła podstawowa"),
646 "G": (2, u"gimnazjum"),
648 "LP": (3, u"liceum"),
651 def audiences_pl(self):
652 audiences = self.extra_info.get('audiences', [])
653 audiences = sorted(set([self._audiences_pl.get(a, (99, a)) for a in audiences]))
654 return [a[1] for a in audiences]
656 def stage_note(self):
657 stage = self.extra_info.get('stage')
658 if stage and stage < '0.4':
659 return (_('This work needs modernisation'),
660 reverse('infopage', args=['wymagajace-uwspolczesnienia']))
664 def choose_fragment(self):
665 fragments = self.fragments.order_by()
666 fragments_count = fragments.count()
667 if not fragments_count and self.children.exists():
668 fragments = Fragment.objects.filter(book__ancestor=self).order_by()
669 fragments_count = fragments.count()
671 return fragments[randint(0, fragments_count - 1)]
673 return self.parent.choose_fragment()
677 def fragment_data(self):
678 fragment = self.choose_fragment()
680 return {'title': fragment.book.pretty_title(), 'html': fragment.get_short_text()}
684 def update_popularity(self):
685 count = self.tags.filter(category='set').values('user').order_by('user').distinct().count()
687 pop = self.popularity
690 except BookPopularity.DoesNotExist:
691 BookPopularity.objects.create(book=self, count=count)
693 def ridero_link(self):
694 return 'https://ridero.eu/%s/books/wl_%s/' % (get_language(), self.slug.replace('-', '_'))
697 def add_file_fields():
698 for format_ in Book.formats:
699 field_name = "%s_file" % format_
700 # This weird globals() assignment makes Django migrations comfortable.
701 _upload_to = _ebook_upload_to('book/%s/%%s.%s' % (format_, format_))
702 _upload_to.__name__ = '_%s_upload_to' % format_
703 globals()[_upload_to.__name__] = _upload_to
706 format_, _("%s file" % format_.upper()),
707 upload_to=_upload_to,
708 storage=bofh_storage,
712 ).contribute_to_class(Book, field_name)
717 class BookPopularity(models.Model):
718 book = models.OneToOneField(Book, related_name='popularity')
719 count = models.IntegerField(default=0, db_index=True)