1 # -*- coding: utf-8 -*-
2 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
3 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
5 from collections import OrderedDict
6 from random import randint
10 from django.conf import settings
11 from django.db import connection, models, transaction
12 from django.db.models import permalink
13 import django.dispatch
14 from django.contrib.contenttypes.fields import GenericRelation
15 from django.core.urlresolvers import reverse
16 from django.utils.translation import ugettext_lazy as _, get_language
17 from django.utils.deconstruct import deconstructible
19 from fnpdjango.storage import BofhFileSystemStorage
20 from ssify import flush_ssi_includes
21 from newtagging import managers
22 from catalogue import constants
23 from catalogue.fields import EbookField
24 from catalogue.models import Tag, Fragment, BookMedia
25 from catalogue.utils import create_zip, gallery_url, gallery_path, split_tags
26 from catalogue.models.tag import prefetched_relations
27 from catalogue import app_settings
28 from catalogue import tasks
29 from wolnelektury.utils import makedirs
31 bofh_storage = BofhFileSystemStorage()
35 class UploadToPath(object):
36 def __init__(self, path):
39 def __call__(self, instance, filename):
40 return self.path % instance.slug
43 _cover_upload_to = UploadToPath('book/cover/%s.jpg')
44 _cover_thumb_upload_to = UploadToPath('book/cover_thumb/%s.jpg')
45 _cover_api_thumb_opload_to = UploadToPath('book/cover_api_thumb/%s.jpg')
48 def _ebook_upload_to(upload_path):
49 return UploadToPath(upload_path)
52 class Book(models.Model):
53 """Represents a book imported from WL-XML."""
54 title = models.CharField(_('title'), max_length=32767)
55 sort_key = models.CharField(_('sort key'), max_length=120, db_index=True, editable=False)
56 sort_key_author = models.CharField(
57 _('sort key by author'), max_length=120, db_index=True, editable=False, default=u'')
58 slug = models.SlugField(_('slug'), max_length=120, db_index=True, unique=True)
59 common_slug = models.SlugField(_('slug'), max_length=120, db_index=True)
60 language = models.CharField(_('language code'), max_length=3, db_index=True, default=app_settings.DEFAULT_LANGUAGE)
61 description = models.TextField(_('description'), blank=True)
62 created_at = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True)
63 changed_at = models.DateTimeField(_('change date'), auto_now=True, db_index=True)
64 parent_number = models.IntegerField(_('parent number'), default=0)
65 extra_info = jsonfield.JSONField(_('extra information'), default={})
66 gazeta_link = models.CharField(blank=True, max_length=240)
67 wiki_link = models.CharField(blank=True, max_length=240)
68 print_on_demand = models.BooleanField(_('print on demand'), default=False)
69 recommended = models.BooleanField(_('recommended'), default=False)
71 # files generated during publication
74 null=True, blank=True,
75 upload_to=_cover_upload_to,
76 storage=bofh_storage, max_length=255)
77 # Cleaner version of cover for thumbs
78 cover_thumb = EbookField(
79 'cover_thumb', _('cover thumbnail'),
80 null=True, blank=True,
81 upload_to=_cover_thumb_upload_to,
83 cover_api_thumb = EbookField(
84 'cover_api_thumb', _('cover thumbnail for API'),
85 null=True, blank=True,
86 upload_to=_cover_api_thumb_opload_to,
88 ebook_formats = constants.EBOOK_FORMATS
89 formats = ebook_formats + ['html', 'xml']
91 parent = models.ForeignKey('self', blank=True, null=True, related_name='children')
92 ancestor = models.ManyToManyField('self', blank=True, editable=False, related_name='descendant', symmetrical=False)
94 cached_author = models.CharField(blank=True, max_length=240, db_index=True)
95 has_audience = models.BooleanField(default=False)
97 objects = models.Manager()
98 tagged = managers.ModelTaggedItemManager(Tag)
99 tags = managers.TagDescriptor(Tag)
100 tag_relations = GenericRelation(Tag.intermediary_table_model)
102 html_built = django.dispatch.Signal()
103 published = django.dispatch.Signal()
105 short_html_url_name = 'catalogue_book_short'
107 class AlreadyExists(Exception):
111 ordering = ('sort_key_author', 'sort_key')
112 verbose_name = _('book')
113 verbose_name_plural = _('books')
114 app_label = 'catalogue'
116 def __unicode__(self):
119 def get_initial(self):
121 return re.search(r'\w', self.title, re.U).group(0)
122 except AttributeError:
126 return self.tags.filter(category='author')
128 def tag_unicode(self, category):
129 relations = prefetched_relations(self, category)
131 return ', '.join(rel.tag.name for rel in relations)
133 return ', '.join(self.tags.filter(category=category).values_list('name', flat=True))
135 def tags_by_category(self):
136 return split_tags(self.tags.exclude(category__in=('set', 'theme')))
138 def author_unicode(self):
139 return self.cached_author
141 def translator(self):
142 translators = self.extra_info.get('translators')
145 if len(translators) > 3:
146 translators = translators[:2]
150 return ', '.join(u'\xa0'.join(reversed(translator.split(', ', 1))) for translator in translators) + others
152 def cover_source(self):
153 return self.extra_info.get('cover_source', self.parent.cover_source() if self.parent else '')
155 def save(self, force_insert=False, force_update=False, **kwargs):
156 from sortify import sortify
158 self.sort_key = sortify(self.title)[:120]
159 self.title = unicode(self.title) # ???
162 author = self.authors().first().sort_key
163 except AttributeError:
165 self.sort_key_author = author
167 self.cached_author = self.tag_unicode('author')
168 self.has_audience = 'audience' in self.extra_info
170 ret = super(Book, self).save(force_insert, force_update, **kwargs)
175 def get_absolute_url(self):
176 return 'catalogue.views.book_detail', [self.slug]
180 def create_url(slug):
181 return 'catalogue.views.book_detail', [slug]
183 def gallery_path(self):
184 return gallery_path(self.slug)
186 def gallery_url(self):
187 return gallery_url(self.slug)
193 def language_code(self):
194 return constants.LANGUAGES_3TO2.get(self.language, self.language)
196 def language_name(self):
197 return dict(settings.LANGUAGES).get(self.language_code(), "")
199 def is_foreign(self):
200 return self.language_code() != settings.LANGUAGE_CODE
202 def has_media(self, type_):
203 if type_ in Book.formats:
204 return bool(getattr(self, "%s_file" % type_))
206 return self.media.filter(type=type_).exists()
209 return self.has_media('mp3')
211 def get_media(self, type_):
212 if self.has_media(type_):
213 if type_ in Book.formats:
214 return getattr(self, "%s_file" % type_)
216 return self.media.filter(type=type_)
221 return self.get_media("mp3")
224 return self.get_media("odt")
227 return self.get_media("ogg")
230 return self.get_media("daisy")
232 def has_description(self):
233 return len(self.description) > 0
234 has_description.short_description = _('description')
235 has_description.boolean = True
238 def has_mp3_file(self):
239 return bool(self.has_media("mp3"))
240 has_mp3_file.short_description = 'MP3'
241 has_mp3_file.boolean = True
243 def has_ogg_file(self):
244 return bool(self.has_media("ogg"))
245 has_ogg_file.short_description = 'OGG'
246 has_ogg_file.boolean = True
248 def has_daisy_file(self):
249 return bool(self.has_media("daisy"))
250 has_daisy_file.short_description = 'DAISY'
251 has_daisy_file.boolean = True
253 def get_audiobooks(self):
255 for m in self.media.filter(type='ogg').order_by().iterator():
256 ogg_files[m.name] = m
260 for mp3 in self.media.filter(type='mp3').iterator():
261 # ogg files are always from the same project
262 meta = mp3.extra_info
263 project = meta.get('project')
266 project = u'CzytamySłuchając'
268 projects.add((project, meta.get('funded_by', '')))
272 ogg = ogg_files.get(mp3.name)
275 audiobooks.append(media)
277 projects = sorted(projects)
278 return audiobooks, projects
280 def wldocument(self, parse_dublincore=True, inherit=True):
281 from catalogue.import_utils import ORMDocProvider
282 from librarian.parser import WLDocument
284 if inherit and self.parent:
285 meta_fallbacks = self.parent.cover_info()
287 meta_fallbacks = None
289 return WLDocument.from_file(
291 provider=ORMDocProvider(self),
292 parse_dublincore=parse_dublincore,
293 meta_fallbacks=meta_fallbacks)
296 def zip_format(format_):
297 def pretty_file_name(book):
298 return "%s/%s.%s" % (
299 book.extra_info['author'],
303 field_name = "%s_file" % format_
304 books = Book.objects.filter(parent=None).exclude(**{field_name: ""})
305 paths = [(pretty_file_name(b), getattr(b, field_name).path) for b in books.iterator()]
306 return create_zip(paths, app_settings.FORMAT_ZIPS[format_])
308 def zip_audiobooks(self, format_):
309 bm = BookMedia.objects.filter(book=self, type=format_)
310 paths = map(lambda bm: (None, bm.file.path), bm)
311 return create_zip(paths, "%s_%s" % (self.slug, format_))
313 def search_index(self, book_info=None, index=None, index_tags=True, commit=True):
315 from search.index import Index
318 index.index_book(self, book_info)
324 index.index.rollback()
327 def download_pictures(self, remote_gallery_url):
328 gallery_path = self.gallery_path()
329 # delete previous files, so we don't include old files in ebooks
330 if os.path.isdir(gallery_path):
331 for filename in os.listdir(gallery_path):
332 file_path = os.path.join(gallery_path, filename)
334 ilustr_elements = list(self.wldocument().edoc.findall('//ilustr'))
336 makedirs(gallery_path)
337 for ilustr in ilustr_elements:
338 ilustr_src = ilustr.get('src')
339 ilustr_path = os.path.join(gallery_path, ilustr_src)
340 urllib.urlretrieve('%s/%s' % (remote_gallery_url, ilustr_src), ilustr_path)
343 def from_xml_file(cls, xml_file, **kwargs):
344 from django.core.files import File
345 from librarian import dcparser
347 # use librarian to parse meta-data
348 book_info = dcparser.parse(xml_file)
350 if not isinstance(xml_file, File):
351 xml_file = File(open(xml_file))
354 return cls.from_text_and_meta(xml_file, book_info, **kwargs)
359 def from_text_and_meta(cls, raw_file, book_info, overwrite=False, dont_build=None, search_index=True,
360 search_index_tags=True, remote_gallery_url=None):
361 if dont_build is None:
363 dont_build = set.union(set(dont_build), set(app_settings.DONT_BUILD))
365 # check for parts before we do anything
367 if hasattr(book_info, 'parts'):
368 for part_url in book_info.parts:
370 children.append(Book.objects.get(slug=part_url.slug))
371 except Book.DoesNotExist:
372 raise Book.DoesNotExist(_('Book "%s" does not exist.') % part_url.slug)
375 book_slug = book_info.url.slug
376 if re.search(r'[^a-z0-9-]', book_slug):
377 raise ValueError('Invalid characters in slug')
378 book, created = Book.objects.get_or_create(slug=book_slug)
385 raise Book.AlreadyExists(_('Book %s already exists') % book_slug)
386 # Save shelves for this book
387 book_shelves = list(book.tags.filter(category='set'))
388 old_cover = book.cover_info()
391 book.xml_file.save('%s.xml' % book.slug, raw_file, save=False)
393 book.language = book_info.language
394 book.title = book_info.title
395 if book_info.variant_of:
396 book.common_slug = book_info.variant_of.slug
398 book.common_slug = book.slug
399 book.extra_info = book_info.to_dict()
402 meta_tags = Tag.tags_from_info(book_info)
404 for tag in meta_tags:
405 if not tag.for_books:
409 book.tags = set(meta_tags + book_shelves)
411 cover_changed = old_cover != book.cover_info()
412 obsolete_children = set(b for b in book.children.all()
413 if b not in children)
414 notify_cover_changed = []
415 for n, child_book in enumerate(children):
416 new_child = child_book.parent != book
417 child_book.parent = book
418 child_book.parent_number = n
420 if new_child or cover_changed:
421 notify_cover_changed.append(child_book)
422 # Disown unfaithful children and let them cope on their own.
423 for child in obsolete_children:
425 child.parent_number = 0
428 notify_cover_changed.append(child)
430 cls.repopulate_ancestors()
431 tasks.update_counters.delay()
433 if remote_gallery_url:
434 book.download_pictures(remote_gallery_url)
436 # No saves beyond this point.
439 if 'cover' not in dont_build:
440 book.cover.build_delay()
441 book.cover_thumb.build_delay()
442 book.cover_api_thumb.build_delay()
444 # Build HTML and ebooks.
445 book.html_file.build_delay()
447 for format_ in constants.EBOOK_FORMATS_WITHOUT_CHILDREN:
448 if format_ not in dont_build:
449 getattr(book, '%s_file' % format_).build_delay()
450 for format_ in constants.EBOOK_FORMATS_WITH_CHILDREN:
451 if format_ not in dont_build:
452 getattr(book, '%s_file' % format_).build_delay()
454 if not settings.NO_SEARCH_INDEX and search_index:
455 tasks.index_book.delay(book.id, book_info=book_info, index_tags=search_index_tags)
457 for child in notify_cover_changed:
458 child.parent_cover_changed()
460 book.save() # update sort_key_author
461 cls.published.send(sender=cls, instance=book)
466 def repopulate_ancestors(cls):
467 """Fixes the ancestry cache."""
469 cursor = connection.cursor()
470 if connection.vendor == 'postgres':
471 cursor.execute("TRUNCATE catalogue_book_ancestor")
473 WITH RECURSIVE ancestry AS (
474 SELECT book.id, book.parent_id
475 FROM catalogue_book AS book
476 WHERE book.parent_id IS NOT NULL
478 SELECT ancestor.id, book.parent_id
479 FROM ancestry AS ancestor, catalogue_book AS book
480 WHERE ancestor.parent_id = book.id
481 AND book.parent_id IS NOT NULL
483 INSERT INTO catalogue_book_ancestor
484 (from_book_id, to_book_id)
490 cursor.execute("DELETE FROM catalogue_book_ancestor")
491 for b in cls.objects.exclude(parent=None):
493 while parent is not None:
494 b.ancestor.add(parent)
495 parent = parent.parent
497 def flush_includes(self, languages=True):
500 if languages is True:
501 languages = [lc for (lc, _ln) in settings.LANGUAGES]
503 template % (self.pk, lang)
505 '/katalog/b/%d/mini.%s.html',
506 '/katalog/b/%d/mini_nolink.%s.html',
507 '/katalog/b/%d/short.%s.html',
508 '/katalog/b/%d/wide.%s.html',
509 '/api/include/book/%d.%s.json',
510 '/api/include/book/%d.%s.xml',
512 for lang in languages
515 def cover_info(self, inherit=True):
516 """Returns a dictionary to serve as fallback for BookInfo.
518 For now, the only thing inherited is the cover image.
522 for field in ('cover_url', 'cover_by', 'cover_source'):
523 val = self.extra_info.get(field)
528 if inherit and need and self.parent is not None:
529 parent_info = self.parent.cover_info()
530 parent_info.update(info)
534 def related_themes(self):
535 return Tag.objects.usage_for_queryset(
536 Fragment.objects.filter(models.Q(book=self) | models.Q(book__ancestor=self)),
537 counts=True).filter(category='theme')
539 def parent_cover_changed(self):
540 """Called when parent book's cover image is changed."""
541 if not self.cover_info(inherit=False):
542 if 'cover' not in app_settings.DONT_BUILD:
543 self.cover.build_delay()
544 self.cover_thumb.build_delay()
545 self.cover_api_thumb.build_delay()
546 for format_ in constants.EBOOK_FORMATS_WITH_COVERS:
547 if format_ not in app_settings.DONT_BUILD:
548 getattr(self, '%s_file' % format_).build_delay()
549 for child in self.children.all():
550 child.parent_cover_changed()
552 def other_versions(self):
553 """Find other versions (i.e. in other languages) of the book."""
554 return type(self).objects.filter(common_slug=self.common_slug).exclude(pk=self.pk)
559 while parent is not None:
560 books.insert(0, parent)
561 parent = parent.parent
564 def pretty_title(self, html_links=False):
565 names = [(tag.name, tag.get_absolute_url()) for tag in self.authors().only('name', 'category', 'slug')]
566 books = self.parents() + [self]
567 names.extend([(b.title, b.get_absolute_url()) for b in books])
570 names = ['<a href="%s">%s</a>' % (tag[1], tag[0]) for tag in names]
572 names = [tag[0] for tag in names]
573 return ', '.join(names)
576 publisher = self.extra_info['publisher']
577 if isinstance(publisher, basestring):
579 elif isinstance(publisher, list):
580 return ', '.join(publisher)
583 def tagged_top_level(cls, tags):
584 """ Returns top-level books tagged with `tags`.
586 It only returns those books which don't have ancestors which are
587 also tagged with those tags.
590 objects = cls.tagged.with_all(tags)
591 return objects.exclude(ancestor__in=objects)
594 def book_list(cls, book_filter=None):
595 """Generates a hierarchical listing of all books.
597 Books are optionally filtered with a test function.
602 books = cls.objects.order_by('parent_number', 'sort_key').only('title', 'parent', 'slug')
604 books = books.filter(book_filter).distinct()
606 book_ids = set(b['pk'] for b in books.values("pk").iterator())
607 for book in books.iterator():
608 parent = book.parent_id
609 if parent not in book_ids:
611 books_by_parent.setdefault(parent, []).append(book)
613 for book in books.iterator():
614 books_by_parent.setdefault(book.parent_id, []).append(book)
617 books_by_author = OrderedDict()
618 for tag in Tag.objects.filter(category='author').iterator():
619 books_by_author[tag] = []
621 for book in books_by_parent.get(None, ()):
622 authors = list(book.authors().only('pk'))
624 for author in authors:
625 books_by_author[author].append(book)
629 return books_by_author, orphans, books_by_parent
632 "SP": (1, u"szkoła podstawowa"),
633 "SP1": (1, u"szkoła podstawowa"),
634 "SP2": (1, u"szkoła podstawowa"),
635 "SP3": (1, u"szkoła podstawowa"),
636 "P": (1, u"szkoła podstawowa"),
637 "G": (2, u"gimnazjum"),
639 "LP": (3, u"liceum"),
642 def audiences_pl(self):
643 audiences = self.extra_info.get('audiences', [])
644 audiences = sorted(set([self._audiences_pl.get(a, (99, a)) for a in audiences]))
645 return [a[1] for a in audiences]
647 def stage_note(self):
648 stage = self.extra_info.get('stage')
649 if stage and stage < '0.4':
650 return (_('This work needs modernisation'),
651 reverse('infopage', args=['wymagajace-uwspolczesnienia']))
655 def choose_fragment(self):
656 fragments = self.fragments.order_by()
657 fragments_count = fragments.count()
658 if not fragments_count and self.children.exists():
659 fragments = Fragment.objects.filter(book__ancestor=self).order_by()
660 fragments_count = fragments.count()
662 return fragments[randint(0, fragments_count - 1)]
664 return self.parent.choose_fragment()
668 def fragment_data(self):
669 fragment = self.choose_fragment()
671 return {'title': fragment.book.pretty_title(), 'html': fragment.get_short_text()}
675 def update_popularity(self):
676 count = self.tags.filter(category='set').values('user').order_by('user').distinct().count()
678 pop = self.popularity
681 except BookPopularity.DoesNotExist:
682 BookPopularity.objects.create(book=self, count=count)
684 def ridero_link(self):
685 return 'https://ridero.eu/%s/books/wl_%s/' % (get_language(), self.slug.replace('-', '_'))
688 def add_file_fields():
689 for format_ in Book.formats:
690 field_name = "%s_file" % format_
691 # This weird globals() assignment makes Django migrations comfortable.
692 _upload_to = _ebook_upload_to('book/%s/%%s.%s' % (format_, format_))
693 _upload_to.__name__ = '_%s_upload_to' % format_
694 globals()[_upload_to.__name__] = _upload_to
697 format_, _("%s file" % format_.upper()),
698 upload_to=_upload_to,
699 storage=bofh_storage,
703 ).contribute_to_class(Book, field_name)
708 class BookPopularity(models.Model):
709 book = models.OneToOneField(Book, related_name='popularity')
710 count = models.IntegerField(default=0)