fix for variable displacement
[wolnelektury.git] / apps / catalogue / models.py
1 # -*- coding: utf-8 -*-
2 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
3 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
4 #
5 from collections import namedtuple
6
7 from django.db import models
8 from django.db.models import permalink
9 import django.dispatch
10 from django.core.cache import get_cache
11 from django.utils.translation import ugettext_lazy as _
12 from django.contrib.auth.models import User
13 from django.template.loader import render_to_string
14 from django.utils.datastructures import SortedDict
15 from django.utils.safestring import mark_safe
16 from django.utils.translation import get_language
17 from django.core.urlresolvers import reverse
18 from django.db.models.signals import post_save, pre_delete, post_delete
19 import jsonfield
20
21 from django.conf import settings
22
23 from newtagging.models import TagBase, tags_updated
24 from newtagging import managers
25 from catalogue.fields import OverwritingFileField
26 from catalogue.utils import create_zip, split_tags, truncate_html_words
27 from catalogue import tasks
28 import re
29
30
31 # Those are hard-coded here so that makemessages sees them.
32 TAG_CATEGORIES = (
33     ('author', _('author')),
34     ('epoch', _('epoch')),
35     ('kind', _('kind')),
36     ('genre', _('genre')),
37     ('theme', _('theme')),
38     ('set', _('set')),
39     ('book', _('book')),
40 )
41
42
43 permanent_cache = get_cache('permanent')
44
45
46 class TagSubcategoryManager(models.Manager):
47     def __init__(self, subcategory):
48         super(TagSubcategoryManager, self).__init__()
49         self.subcategory = subcategory
50
51     def get_query_set(self):
52         return super(TagSubcategoryManager, self).get_query_set().filter(category=self.subcategory)
53
54
55 class Tag(TagBase):
56     """A tag attachable to books and fragments (and possibly anything).
57     
58     Used to represent searchable metadata (authors, epochs, genres, kinds),
59     fragment themes (motifs) and some book hierarchy related kludges."""
60     name = models.CharField(_('name'), max_length=50, db_index=True)
61     slug = models.SlugField(_('slug'), max_length=120, db_index=True)
62     sort_key = models.CharField(_('sort key'), max_length=120, db_index=True)
63     category = models.CharField(_('category'), max_length=50, blank=False, null=False,
64         db_index=True, choices=TAG_CATEGORIES)
65     description = models.TextField(_('description'), blank=True)
66
67     user = models.ForeignKey(User, blank=True, null=True)
68     book_count = models.IntegerField(_('book count'), blank=True, null=True)
69     gazeta_link = models.CharField(blank=True, max_length=240)
70     wiki_link = models.CharField(blank=True, max_length=240)
71
72     created_at    = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True)
73     changed_at    = models.DateTimeField(_('creation date'), auto_now=True, db_index=True)
74
75     class UrlDeprecationWarning(DeprecationWarning):
76         pass
77
78     categories_rev = {
79         'autor': 'author',
80         'epoka': 'epoch',
81         'rodzaj': 'kind',
82         'gatunek': 'genre',
83         'motyw': 'theme',
84         'polka': 'set',
85     }
86     categories_dict = dict((item[::-1] for item in categories_rev.iteritems()))
87
88     class Meta:
89         ordering = ('sort_key',)
90         verbose_name = _('tag')
91         verbose_name_plural = _('tags')
92         unique_together = (("slug", "category"),)
93
94     def __unicode__(self):
95         return self.name
96
97     def __repr__(self):
98         return "Tag(slug=%r)" % self.slug
99
100     @permalink
101     def get_absolute_url(self):
102         return ('catalogue.views.tagged_object_list', [self.url_chunk])
103
104     def has_description(self):
105         return len(self.description) > 0
106     has_description.short_description = _('description')
107     has_description.boolean = True
108
109     def get_count(self):
110         """Returns global book count for book tags, fragment count for themes."""
111
112         if self.category == 'book':
113             # never used
114             objects = Book.objects.none()
115         elif self.category == 'theme':
116             objects = Fragment.tagged.with_all((self,))
117         else:
118             objects = Book.tagged.with_all((self,)).order_by()
119             if self.category != 'set':
120                 # eliminate descendants
121                 l_tags = Tag.objects.filter(slug__in=[book.book_tag_slug() for book in objects.iterator()])
122                 descendants_keys = [book.pk for book in Book.tagged.with_any(l_tags).iterator()]
123                 if descendants_keys:
124                     objects = objects.exclude(pk__in=descendants_keys)
125         return objects.count()
126
127     @staticmethod
128     def get_tag_list(tags):
129         if isinstance(tags, basestring):
130             real_tags = []
131             ambiguous_slugs = []
132             category = None
133             deprecated = False
134             tags_splitted = tags.split('/')
135             for name in tags_splitted:
136                 if category:
137                     real_tags.append(Tag.objects.get(slug=name, category=category))
138                     category = None
139                 elif name in Tag.categories_rev:
140                     category = Tag.categories_rev[name]
141                 else:
142                     try:
143                         real_tags.append(Tag.objects.exclude(category='book').get(slug=name))
144                         deprecated = True 
145                     except Tag.MultipleObjectsReturned, e:
146                         ambiguous_slugs.append(name)
147
148             if category:
149                 # something strange left off
150                 raise Tag.DoesNotExist()
151             if ambiguous_slugs:
152                 # some tags should be qualified
153                 e = Tag.MultipleObjectsReturned()
154                 e.tags = real_tags
155                 e.ambiguous_slugs = ambiguous_slugs
156                 raise e
157             if deprecated:
158                 e = Tag.UrlDeprecationWarning()
159                 e.tags = real_tags
160                 raise e
161             return real_tags
162         else:
163             return TagBase.get_tag_list(tags)
164
165     @property
166     def url_chunk(self):
167         return '/'.join((Tag.categories_dict[self.category], self.slug))
168
169     @staticmethod
170     def tags_from_info(info):
171         from slughifi import slughifi
172         from sortify import sortify
173         meta_tags = []
174         categories = (('kinds', 'kind'), ('genres', 'genre'), ('authors', 'author'), ('epochs', 'epoch'))
175         for field_name, category in categories:
176             try:
177                 tag_names = getattr(info, field_name)
178             except:
179                 try:
180                     tag_names = [getattr(info, category)]
181                 except:
182                     # For instance, Pictures do not have 'genre' field.
183                     continue
184             for tag_name in tag_names:
185                 tag_sort_key = tag_name
186                 if category == 'author':
187                     tag_sort_key = tag_name.last_name
188                     tag_name = tag_name.readable()
189                 tag, created = Tag.objects.get_or_create(slug=slughifi(tag_name), category=category)
190                 if created:
191                     tag.name = tag_name
192                     tag.sort_key = sortify(tag_sort_key.lower())
193                     tag.save()
194                 meta_tags.append(tag)
195         return meta_tags
196
197
198
199 def get_dynamic_path(media, filename, ext=None, maxlen=100):
200     from slughifi import slughifi
201
202     # how to put related book's slug here?
203     if not ext:
204         # BookMedia case
205         ext = media.formats[media.type].ext
206     if media is None or not media.name:
207         name = slughifi(filename.split(".")[0])
208     else:
209         name = slughifi(media.name)
210     return 'book/%s/%s.%s' % (ext, name[:maxlen-len('book/%s/.%s' % (ext, ext))-4], ext)
211
212
213 # TODO: why is this hard-coded ?
214 def book_upload_path(ext=None, maxlen=100):
215     return lambda *args: get_dynamic_path(*args, ext=ext, maxlen=maxlen)
216
217
218 class BookMedia(models.Model):
219     """Represents media attached to a book."""
220     FileFormat = namedtuple("FileFormat", "name ext")
221     formats = SortedDict([
222         ('mp3', FileFormat(name='MP3', ext='mp3')),
223         ('ogg', FileFormat(name='Ogg Vorbis', ext='ogg')),
224         ('daisy', FileFormat(name='DAISY', ext='daisy.zip')),
225     ])
226     format_choices = [(k, _('%s file') % t.name)
227             for k, t in formats.items()]
228
229     type        = models.CharField(_('type'), choices=format_choices, max_length="100")
230     name        = models.CharField(_('name'), max_length="100")
231     file        = OverwritingFileField(_('file'), upload_to=book_upload_path())
232     uploaded_at = models.DateTimeField(_('creation date'), auto_now_add=True, editable=False)
233     extra_info  = jsonfield.JSONField(_('extra information'), default='{}', editable=False)
234     book = models.ForeignKey('Book', related_name='media')
235     source_sha1 = models.CharField(null=True, blank=True, max_length=40, editable=False)
236
237     def __unicode__(self):
238         return "%s (%s)" % (self.name, self.file.name.split("/")[-1])
239
240     class Meta:
241         ordering            = ('type', 'name')
242         verbose_name        = _('book media')
243         verbose_name_plural = _('book media')
244
245     def save(self, *args, **kwargs):
246         from slughifi import slughifi
247         from catalogue.utils import ExistingFile, remove_zip
248
249         try:
250             old = BookMedia.objects.get(pk=self.pk)
251         except BookMedia.DoesNotExist:
252             old = None
253         else:
254             # if name changed, change the file name, too
255             if slughifi(self.name) != slughifi(old.name):
256                 self.file.save(None, ExistingFile(self.file.path), save=False, leave=True)
257
258         super(BookMedia, self).save(*args, **kwargs)
259
260         # remove the zip package for book with modified media
261         if old:
262             remove_zip("%s_%s" % (old.book.slug, old.type))
263         remove_zip("%s_%s" % (self.book.slug, self.type))
264
265         extra_info = self.extra_info
266         extra_info.update(self.read_meta())
267         self.extra_info = extra_info
268         self.source_sha1 = self.read_source_sha1(self.file.path, self.type)
269         return super(BookMedia, self).save(*args, **kwargs)
270
271     def read_meta(self):
272         """
273             Reads some metadata from the audiobook.
274         """
275         import mutagen
276         from mutagen import id3
277
278         artist_name = director_name = project = funded_by = ''
279         if self.type == 'mp3':
280             try:
281                 audio = id3.ID3(self.file.path)
282                 artist_name = ', '.join(', '.join(tag.text) for tag in audio.getall('TPE1'))
283                 director_name = ', '.join(', '.join(tag.text) for tag in audio.getall('TPE3'))
284                 project = ", ".join([t.data for t in audio.getall('PRIV') 
285                         if t.owner=='wolnelektury.pl?project'])
286                 funded_by = ", ".join([t.data for t in audio.getall('PRIV') 
287                         if t.owner=='wolnelektury.pl?funded_by'])
288             except:
289                 pass
290         elif self.type == 'ogg':
291             try:
292                 audio = mutagen.File(self.file.path)
293                 artist_name = ', '.join(audio.get('artist', []))
294                 director_name = ', '.join(audio.get('conductor', []))
295                 project = ", ".join(audio.get('project', []))
296                 funded_by = ", ".join(audio.get('funded_by', []))
297             except:
298                 pass
299         else:
300             return {}
301         return {'artist_name': artist_name, 'director_name': director_name,
302                 'project': project, 'funded_by': funded_by}
303
304     @staticmethod
305     def read_source_sha1(filepath, filetype):
306         """
307             Reads source file SHA1 from audiobok metadata.
308         """
309         import mutagen
310         from mutagen import id3
311
312         if filetype == 'mp3':
313             try:
314                 audio = id3.ID3(filepath)
315                 return [t.data for t in audio.getall('PRIV') 
316                         if t.owner=='wolnelektury.pl?flac_sha1'][0]
317             except:
318                 return None
319         elif filetype == 'ogg':
320             try:
321                 audio = mutagen.File(filepath)
322                 return audio.get('flac_sha1', [None])[0] 
323             except:
324                 return None
325         else:
326             return None
327
328
329 class Book(models.Model):
330     """Represents a book imported from WL-XML."""
331     title         = models.CharField(_('title'), max_length=120)
332     sort_key = models.CharField(_('sort key'), max_length=120, db_index=True, editable=False)
333     slug = models.SlugField(_('slug'), max_length=120, db_index=True,
334             unique=True)
335     common_slug = models.SlugField(_('slug'), max_length=120, db_index=True)
336     language = models.CharField(_('language code'), max_length=3, db_index=True,
337                     default=settings.CATALOGUE_DEFAULT_LANGUAGE)
338     description   = models.TextField(_('description'), blank=True)
339     created_at    = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True)
340     changed_at    = models.DateTimeField(_('creation date'), auto_now=True, db_index=True)
341     parent_number = models.IntegerField(_('parent number'), default=0)
342     extra_info    = jsonfield.JSONField(_('extra information'), default='{}')
343     gazeta_link   = models.CharField(blank=True, max_length=240)
344     wiki_link     = models.CharField(blank=True, max_length=240)
345     # files generated during publication
346
347     cover = models.FileField(_('cover'), upload_to=book_upload_path('png'),
348                 null=True, blank=True)
349     ebook_formats = ['pdf', 'epub', 'mobi', 'fb2', 'txt']
350     formats = ebook_formats + ['html', 'xml']
351
352     parent        = models.ForeignKey('self', blank=True, null=True, related_name='children')
353
354     _related_info = jsonfield.JSONField(blank=True, null=True, editable=False)
355
356     objects  = models.Manager()
357     tagged   = managers.ModelTaggedItemManager(Tag)
358     tags     = managers.TagDescriptor(Tag)
359
360     html_built = django.dispatch.Signal()
361     published = django.dispatch.Signal()
362
363     class AlreadyExists(Exception):
364         pass
365
366     class Meta:
367         ordering = ('sort_key',)
368         verbose_name = _('book')
369         verbose_name_plural = _('books')
370
371     def __unicode__(self):
372         return self.title
373
374     def save(self, force_insert=False, force_update=False, reset_short_html=True, **kwargs):
375         from sortify import sortify
376
377         self.sort_key = sortify(self.title)
378
379         ret = super(Book, self).save(force_insert, force_update)
380
381         if reset_short_html:
382             self.reset_short_html()
383
384         return ret
385
386     @permalink
387     def get_absolute_url(self):
388         return ('catalogue.views.book_detail', [self.slug])
389
390     @property
391     def name(self):
392         return self.title
393
394     def book_tag_slug(self):
395         return ('l-' + self.slug)[:120]
396
397     def book_tag(self):
398         slug = self.book_tag_slug()
399         book_tag, created = Tag.objects.get_or_create(slug=slug, category='book')
400         if created:
401             book_tag.name = self.title[:50]
402             book_tag.sort_key = self.title.lower()
403             book_tag.save()
404         return book_tag
405
406     def has_media(self, type_):
407         if type_ in Book.formats:
408             return bool(getattr(self, "%s_file" % type_))
409         else:
410             return self.media.filter(type=type_).exists()
411
412     def get_media(self, type_):
413         if self.has_media(type_):
414             if type_ in Book.formats:
415                 return getattr(self, "%s_file" % type_)
416             else:                                             
417                 return self.media.filter(type=type_)
418         else:
419             return None
420
421     def get_mp3(self):
422         return self.get_media("mp3")
423     def get_odt(self):
424         return self.get_media("odt")
425     def get_ogg(self):
426         return self.get_media("ogg")
427     def get_daisy(self):
428         return self.get_media("daisy")                       
429
430     def reset_short_html(self):
431         if self.id is None:
432             return
433
434         type(self).objects.filter(pk=self.pk).update(_related_info=None)
435         # Fragment.short_html relies on book's tags, so reset it here too
436         for fragm in self.fragments.all().iterator():
437             fragm.reset_short_html()
438
439     def has_description(self):
440         return len(self.description) > 0
441     has_description.short_description = _('description')
442     has_description.boolean = True
443
444     # ugly ugly ugly
445     def has_mp3_file(self):
446         return bool(self.has_media("mp3"))
447     has_mp3_file.short_description = 'MP3'
448     has_mp3_file.boolean = True
449
450     def has_ogg_file(self):
451         return bool(self.has_media("ogg"))
452     has_ogg_file.short_description = 'OGG'
453     has_ogg_file.boolean = True
454
455     def has_daisy_file(self):
456         return bool(self.has_media("daisy"))
457     has_daisy_file.short_description = 'DAISY'
458     has_daisy_file.boolean = True
459
460     def wldocument(self, parse_dublincore=True):
461         from catalogue.import_utils import ORMDocProvider
462         from librarian.parser import WLDocument
463
464         return WLDocument.from_file(self.xml_file.path,
465                 provider=ORMDocProvider(self),
466                 parse_dublincore=parse_dublincore)
467
468     def build_cover(self, book_info=None):
469         """(Re)builds the cover image."""
470         from StringIO import StringIO
471         from django.core.files.base import ContentFile
472         from librarian.cover import WLCover
473
474         if book_info is None:
475             book_info = self.wldocument().book_info
476
477         cover = WLCover(book_info).image()
478         imgstr = StringIO()
479         cover.save(imgstr, 'png')
480         self.cover.save(None, ContentFile(imgstr.getvalue()))
481
482     def build_html(self):
483         from django.core.files.base import ContentFile
484         from slughifi import slughifi
485         from sortify import sortify
486         from librarian import html
487
488         meta_tags = list(self.tags.filter(
489             category__in=('author', 'epoch', 'genre', 'kind')))
490         book_tag = self.book_tag()
491
492         html_output = self.wldocument(parse_dublincore=False).as_html()
493         if html_output:
494             self.html_file.save('%s.html' % self.slug,
495                     ContentFile(html_output.get_string()))
496
497             # get ancestor l-tags for adding to new fragments
498             ancestor_tags = []
499             p = self.parent
500             while p:
501                 ancestor_tags.append(p.book_tag())
502                 p = p.parent
503
504             # Delete old fragments and create them from scratch
505             self.fragments.all().delete()
506             # Extract fragments
507             closed_fragments, open_fragments = html.extract_fragments(self.html_file.path)
508             for fragment in closed_fragments.values():
509                 try:
510                     theme_names = [s.strip() for s in fragment.themes.split(',')]
511                 except AttributeError:
512                     continue
513                 themes = []
514                 for theme_name in theme_names:
515                     if not theme_name:
516                         continue
517                     tag, created = Tag.objects.get_or_create(slug=slughifi(theme_name), category='theme')
518                     if created:
519                         tag.name = theme_name
520                         tag.sort_key = sortify(theme_name.lower())
521                         tag.save()
522                     themes.append(tag)
523                 if not themes:
524                     continue
525
526                 text = fragment.to_string()
527                 short_text = truncate_html_words(text, 15)
528                 if text == short_text:
529                     short_text = ''
530                 new_fragment = Fragment.objects.create(anchor=fragment.id, book=self,
531                     text=text, short_text=short_text)
532
533                 new_fragment.save()
534                 new_fragment.tags = set(meta_tags + themes + [book_tag] + ancestor_tags)
535             self.save()
536             self.html_built.send(sender=self)
537             return True
538         return False
539
540     # Thin wrappers for builder tasks
541     def build_pdf(self, *args, **kwargs):
542         """(Re)builds PDF."""
543         return tasks.build_pdf.delay(self.pk, *args, **kwargs)
544     def build_epub(self, *args, **kwargs):
545         """(Re)builds EPUB."""
546         return tasks.build_epub.delay(self.pk, *args, **kwargs)
547     def build_mobi(self, *args, **kwargs):
548         """(Re)builds MOBI."""
549         return tasks.build_mobi.delay(self.pk, *args, **kwargs)
550     def build_fb2(self, *args, **kwargs):
551         """(Re)build FB2"""
552         return tasks.build_fb2.delay(self.pk, *args, **kwargs)
553     def build_txt(self, *args, **kwargs):
554         """(Re)builds TXT."""
555         return tasks.build_txt.delay(self.pk, *args, **kwargs)
556
557     @staticmethod
558     def zip_format(format_):
559         def pretty_file_name(book):
560             return "%s/%s.%s" % (
561                 b.extra_info['author'],
562                 b.slug,
563                 format_)
564
565         field_name = "%s_file" % format_
566         books = Book.objects.filter(parent=None).exclude(**{field_name: ""})
567         paths = [(pretty_file_name(b), getattr(b, field_name).path)
568                     for b in books.iterator()]
569         return create_zip(paths,
570                     getattr(settings, "ALL_%s_ZIP" % format_.upper()))
571
572     def zip_audiobooks(self, format_):
573         bm = BookMedia.objects.filter(book=self, type=format_)
574         paths = map(lambda bm: (None, bm.file.path), bm)
575         return create_zip(paths, "%s_%s" % (self.slug, format_))
576
577     def search_index(self, book_info=None, reuse_index=False, index_tags=True):
578         import search
579         if reuse_index:
580             idx = search.ReusableIndex()
581         else:
582             idx = search.Index()
583             
584         idx.open()
585         try:
586             idx.index_book(self, book_info)
587             if index_tags:
588                 idx.index_tags()
589         finally:
590             idx.close()
591
592     @classmethod
593     def from_xml_file(cls, xml_file, **kwargs):
594         from django.core.files import File
595         from librarian import dcparser
596
597         # use librarian to parse meta-data
598         book_info = dcparser.parse(xml_file)
599
600         if not isinstance(xml_file, File):
601             xml_file = File(open(xml_file))
602
603         try:
604             return cls.from_text_and_meta(xml_file, book_info, **kwargs)
605         finally:
606             xml_file.close()
607
608     @classmethod
609     def from_text_and_meta(cls, raw_file, book_info, overwrite=False,
610             build_epub=True, build_txt=True, build_pdf=True, build_mobi=True, build_fb2=True,
611             search_index=True, search_index_tags=True, search_index_reuse=False):
612
613         # check for parts before we do anything
614         children = []
615         if hasattr(book_info, 'parts'):
616             for part_url in book_info.parts:
617                 try:
618                     children.append(Book.objects.get(slug=part_url.slug))
619                 except Book.DoesNotExist:
620                     raise Book.DoesNotExist(_('Book "%s" does not exist.') %
621                             part_url.slug)
622
623
624         # Read book metadata
625         book_slug = book_info.url.slug
626         if re.search(r'[^a-z0-9-]', book_slug):
627             raise ValueError('Invalid characters in slug')
628         book, created = Book.objects.get_or_create(slug=book_slug)
629
630         if created:
631             book_shelves = []
632         else:
633             if not overwrite:
634                 raise Book.AlreadyExists(_('Book %s already exists') % (
635                         book_slug))
636             # Save shelves for this book
637             book_shelves = list(book.tags.filter(category='set'))
638
639         book.language = book_info.language
640         book.title = book_info.title
641         if book_info.variant_of:
642             book.common_slug = book_info.variant_of.slug
643         else:
644             book.common_slug = book.slug
645         book.extra_info = book_info.to_dict()
646         book.save()
647
648         meta_tags = Tag.tags_from_info(book_info)
649
650         book.tags = set(meta_tags + book_shelves)
651
652         book_tag = book.book_tag()
653
654         for n, child_book in enumerate(children):
655             child_book.parent = book
656             child_book.parent_number = n
657             child_book.save()
658
659         # Save XML and HTML files
660         book.xml_file.save('%s.xml' % book.slug, raw_file, save=False)
661
662         # delete old fragments when overwriting
663         book.fragments.all().delete()
664
665         if book.build_html():
666             if not settings.NO_BUILD_TXT and build_txt:
667                 book.build_txt()
668
669         book.build_cover(book_info)
670
671         if not settings.NO_BUILD_EPUB and build_epub:
672             book.build_epub()
673
674         if not settings.NO_BUILD_PDF and build_pdf:
675             book.build_pdf()
676
677         if not settings.NO_BUILD_MOBI and build_mobi:
678             book.build_mobi()
679
680         if not settings.NO_BUILD_FB2 and build_fb2:
681             book.build_fb2()
682
683         if not settings.NO_SEARCH_INDEX and search_index:
684             book.search_index(index_tags=search_index_tags, reuse_index=search_index_reuse)
685             #index_book.delay(book.id, book_info)
686
687         book_descendants = list(book.children.all())
688         descendants_tags = set()
689         # add l-tag to descendants and their fragments
690         while len(book_descendants) > 0:
691             child_book = book_descendants.pop(0)
692             descendants_tags.update(child_book.tags)
693             child_book.tags = list(child_book.tags) + [book_tag]
694             child_book.save()
695             for fragment in child_book.fragments.all().iterator():
696                 fragment.tags = set(list(fragment.tags) + [book_tag])
697             book_descendants += list(child_book.children.all())
698
699         for tag in descendants_tags:
700             tasks.touch_tag(tag)
701
702         book.save()
703
704         # refresh cache
705         book.reset_tag_counter()
706         book.reset_theme_counter()
707
708         cls.published.send(sender=book)
709         return book
710
711     def related_info(self):
712         """Keeps info about related objects (tags, media) in cache field."""
713         if self._related_info is not None:
714             return self._related_info
715         else:
716             rel = {'tags': {}, 'media': {}}
717
718             tags = self.tags.filter(category__in=(
719                     'author', 'kind', 'genre', 'epoch'))
720             tags = split_tags(tags)
721             for category in tags:
722                 rel['tags'][category] = [
723                         (t.name, t.slug) for t in tags[category]]
724
725             for media_format in BookMedia.formats:
726                 rel['media'][media_format] = self.has_media(media_format)
727
728             book = self
729             parents = []
730             while book.parent:
731                 parents.append((book.parent.title, book.parent.slug))
732                 book = book.parent
733             parents = parents[::-1]
734             if parents:
735                 rel['parents'] = parents
736
737             if self.pk:
738                 type(self).objects.filter(pk=self.pk).update(_related_info=rel)
739             return rel
740
741     def related_themes(self):
742         theme_counter = self.theme_counter
743         book_themes = list(Tag.objects.filter(pk__in=theme_counter.keys()))
744         for tag in book_themes:
745             tag.count = theme_counter[tag.pk]
746         return book_themes
747
748     def reset_tag_counter(self):
749         if self.id is None:
750             return
751
752         cache_key = "Book.tag_counter/%d" % self.id
753         permanent_cache.delete(cache_key)
754         if self.parent:
755             self.parent.reset_tag_counter()
756
757     @property
758     def tag_counter(self):
759         if self.id:
760             cache_key = "Book.tag_counter/%d" % self.id
761             tags = permanent_cache.get(cache_key)
762         else:
763             tags = None
764
765         if tags is None:
766             tags = {}
767             for child in self.children.all().order_by().iterator():
768                 for tag_pk, value in child.tag_counter.iteritems():
769                     tags[tag_pk] = tags.get(tag_pk, 0) + value
770             for tag in self.tags.exclude(category__in=('book', 'theme', 'set')).order_by().iterator():
771                 tags[tag.pk] = 1
772
773             if self.id:
774                 permanent_cache.set(cache_key, tags)
775         return tags
776
777     def reset_theme_counter(self):
778         if self.id is None:
779             return
780
781         cache_key = "Book.theme_counter/%d" % self.id
782         permanent_cache.delete(cache_key)
783         if self.parent:
784             self.parent.reset_theme_counter()
785
786     @property
787     def theme_counter(self):
788         if self.id:
789             cache_key = "Book.theme_counter/%d" % self.id
790             tags = permanent_cache.get(cache_key)
791         else:
792             tags = None
793
794         if tags is None:
795             tags = {}
796             for fragment in Fragment.tagged.with_any([self.book_tag()]).order_by().iterator():
797                 for tag in fragment.tags.filter(category='theme').order_by().iterator():
798                     tags[tag.pk] = tags.get(tag.pk, 0) + 1
799
800             if self.id:
801                 permanent_cache.set(cache_key, tags)
802         return tags
803
804     def pretty_title(self, html_links=False):
805         book = self
806         names = list(book.tags.filter(category='author'))
807
808         books = []
809         while book:
810             books.append(book)
811             book = book.parent
812         names.extend(reversed(books))
813
814         if html_links:
815             names = ['<a href="%s">%s</a>' % (tag.get_absolute_url(), tag.name) for tag in names]
816         else:
817             names = [tag.name for tag in names]
818
819         return ', '.join(names)
820
821     @classmethod
822     def tagged_top_level(cls, tags):
823         """ Returns top-level books tagged with `tags`.
824
825         It only returns those books which don't have ancestors which are
826         also tagged with those tags.
827
828         """
829         # get relevant books and their tags
830         objects = cls.tagged.with_all(tags)
831         # eliminate descendants
832         l_tags = Tag.objects.filter(category='book',
833             slug__in=[book.book_tag_slug() for book in objects.iterator()])
834         descendants_keys = [book.pk for book in cls.tagged.with_any(l_tags).iterator()]
835         if descendants_keys:
836             objects = objects.exclude(pk__in=descendants_keys)
837
838         return objects
839
840     @classmethod
841     def book_list(cls, filter=None):
842         """Generates a hierarchical listing of all books.
843
844         Books are optionally filtered with a test function.
845
846         """
847
848         books_by_parent = {}
849         books = cls.objects.all().order_by('parent_number', 'sort_key').only(
850                 'title', 'parent', 'slug')
851         if filter:
852             books = books.filter(filter).distinct()
853             
854             book_ids = set(b['pk'] for b in books.values("pk").iterator())
855             for book in books.iterator():
856                 parent = book.parent_id
857                 if parent not in book_ids:
858                     parent = None
859                 books_by_parent.setdefault(parent, []).append(book)
860         else:
861             for book in books.iterator():
862                 books_by_parent.setdefault(book.parent_id, []).append(book)
863
864         orphans = []
865         books_by_author = SortedDict()
866         for tag in Tag.objects.filter(category='author').iterator():
867             books_by_author[tag] = []
868
869         for book in books_by_parent.get(None,()):
870             authors = list(book.tags.filter(category='author'))
871             if authors:
872                 for author in authors:
873                     books_by_author[author].append(book)
874             else:
875                 orphans.append(book)
876
877         return books_by_author, orphans, books_by_parent
878
879     _audiences_pl = {
880         "SP1": (1, u"szkoła podstawowa"),
881         "SP2": (1, u"szkoła podstawowa"),
882         "P": (1, u"szkoła podstawowa"),
883         "G": (2, u"gimnazjum"),
884         "L": (3, u"liceum"),
885         "LP": (3, u"liceum"),
886     }
887     def audiences_pl(self):
888         audiences = self.extra_info.get('audiences', [])
889         audiences = sorted(set([self._audiences_pl[a] for a in audiences]))
890         return [a[1] for a in audiences]
891
892     def choose_fragment(self):
893         tag = self.book_tag()
894         fragments = Fragment.tagged.with_any([tag])
895         if fragments.exists():
896             return fragments.order_by('?')[0]
897         elif self.parent:
898             return self.parent.choose_fragment()
899         else:
900             return None
901
902
903 def _has_factory(ftype):
904     has = lambda self: bool(getattr(self, "%s_file" % ftype))
905     has.short_description = ftype.upper()
906     has.__doc__ = None
907     has.boolean = True
908     has.__name__ = "has_%s_file" % ftype
909     return has
910
911     
912 # add the file fields
913 for t in Book.formats:
914     field_name = "%s_file" % t
915     models.FileField(_("%s file" % t.upper()),
916             upload_to=book_upload_path(t),
917             blank=True).contribute_to_class(Book, field_name)
918
919     setattr(Book, "has_%s_file" % t, _has_factory(t))
920
921
922 class Fragment(models.Model):
923     """Represents a themed fragment of a book."""
924     text = models.TextField()
925     short_text = models.TextField(editable=False)
926     anchor = models.CharField(max_length=120)
927     book = models.ForeignKey(Book, related_name='fragments')
928
929     objects = models.Manager()
930     tagged = managers.ModelTaggedItemManager(Tag)
931     tags = managers.TagDescriptor(Tag)
932
933     class Meta:
934         ordering = ('book', 'anchor',)
935         verbose_name = _('fragment')
936         verbose_name_plural = _('fragments')
937
938     def get_absolute_url(self):
939         return '%s#m%s' % (reverse('book_text', args=[self.book.slug]), self.anchor)
940
941     def reset_short_html(self):
942         if self.id is None:
943             return
944
945         cache_key = "Fragment.short_html/%d/%s"
946         for lang, langname in settings.LANGUAGES:
947             permanent_cache.delete(cache_key % (self.id, lang))
948
949     def get_short_text(self):
950         """Returns short version of the fragment."""
951         return self.short_text if self.short_text else self.text
952
953     def short_html(self):
954         if self.id:
955             cache_key = "Fragment.short_html/%d/%s" % (self.id, get_language())
956             short_html = permanent_cache.get(cache_key)
957         else:
958             short_html = None
959
960         if short_html is not None:
961             return mark_safe(short_html)
962         else:
963             short_html = unicode(render_to_string('catalogue/fragment_short.html',
964                 {'fragment': self}))
965             if self.id:
966                 permanent_cache.set(cache_key, short_html)
967             return mark_safe(short_html)
968
969
970 class Collection(models.Model):
971     """A collection of books, which might be defined before publishing them."""
972     title = models.CharField(_('title'), max_length=120, db_index=True)
973     slug = models.SlugField(_('slug'), max_length=120, primary_key=True)
974     description = models.TextField(_('description'), null=True, blank=True)
975
976     models.SlugField(_('slug'), max_length=120, unique=True, db_index=True)
977     book_slugs = models.TextField(_('book slugs'))
978
979     class Meta:
980         ordering = ('title',)
981         verbose_name = _('collection')
982         verbose_name_plural = _('collections')
983
984     def __unicode__(self):
985         return self.title
986
987
988 ###########
989 #
990 # SIGNALS
991 #
992 ###########
993
994
995 def _tags_updated_handler(sender, affected_tags, **kwargs):
996     # reset tag global counter
997     # we want Tag.changed_at updated for API to know the tag was touched
998     for tag in affected_tags:
999         tasks.touch_tag(tag)
1000
1001     # if book tags changed, reset book tag counter
1002     if isinstance(sender, Book) and \
1003                 Tag.objects.filter(pk__in=(tag.pk for tag in affected_tags)).\
1004                     exclude(category__in=('book', 'theme', 'set')).count():
1005         sender.reset_tag_counter()
1006     # if fragment theme changed, reset book theme counter
1007     elif isinstance(sender, Fragment) and \
1008                 Tag.objects.filter(pk__in=(tag.pk for tag in affected_tags)).\
1009                     filter(category='theme').count():
1010         sender.book.reset_theme_counter()
1011 tags_updated.connect(_tags_updated_handler)
1012
1013
1014 def _pre_delete_handler(sender, instance, **kwargs):
1015     """ refresh Book on BookMedia delete """
1016     if sender == BookMedia:
1017         instance.book.save()
1018 pre_delete.connect(_pre_delete_handler)
1019
1020
1021 def _post_save_handler(sender, instance, **kwargs):
1022     """ refresh all the short_html stuff on BookMedia update """
1023     if sender == BookMedia:
1024         instance.book.save()
1025 post_save.connect(_post_save_handler)
1026
1027
1028 if not settings.NO_SEARCH_INDEX:
1029     @django.dispatch.receiver(post_delete, sender=Book)
1030     def _remove_book_from_index_handler(sender, instance, **kwargs):
1031         """ remove the book from search index, when it is deleted."""
1032         import search
1033         search.JVM.attachCurrentThread()
1034         idx = search.Index()
1035         idx.open(timeout=10000)  # 10 seconds timeout.
1036         try:
1037             idx.remove_book(instance)
1038             idx.index_tags()
1039         finally:
1040             idx.close()