b82879ba5b51eed65a8a8dbd7e8e5d828a3c0fab
[wolnelektury.git] / apps / catalogue / models.py
1 # -*- coding: utf-8 -*-
2 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
3 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
4 #
5 from datetime import datetime
6
7 from django.db import models
8 from django.db.models import permalink, Q
9 import django.dispatch
10 from django.core.cache import cache
11 from django.core.files.storage import DefaultStorage
12 from django.utils.translation import ugettext_lazy as _
13 from django.contrib.auth.models import User
14 from django.template.loader import render_to_string
15 from django.utils.datastructures import SortedDict
16 from django.utils.safestring import mark_safe
17 from django.utils.translation import get_language
18 from django.core.urlresolvers import reverse
19 from django.db.models.signals import post_save, m2m_changed, pre_delete
20
21 from django.conf import settings
22
23 from newtagging.models import TagBase, tags_updated
24 from newtagging import managers
25 from catalogue.fields import JSONField, OverwritingFileField
26 from catalogue.utils import create_zip
27 from shutil import copy
28 from glob import glob
29 import re
30 from os import path
31
32
33 TAG_CATEGORIES = (
34     ('author', _('author')),
35     ('epoch', _('epoch')),
36     ('kind', _('kind')),
37     ('genre', _('genre')),
38     ('theme', _('theme')),
39     ('set', _('set')),
40     ('book', _('book')),
41 )
42
43 MEDIA_FORMATS = (
44     ('odt', _('ODT file')),
45     ('mp3', _('MP3 file')),
46     ('ogg', _('OGG file')),
47     ('daisy', _('DAISY file')), 
48 )
49
50 # not quite, but Django wants you to set a timeout
51 CACHE_FOREVER = 2419200  # 28 days
52
53
54 class TagSubcategoryManager(models.Manager):
55     def __init__(self, subcategory):
56         super(TagSubcategoryManager, self).__init__()
57         self.subcategory = subcategory
58
59     def get_query_set(self):
60         return super(TagSubcategoryManager, self).get_query_set().filter(category=self.subcategory)
61
62
63 class Tag(TagBase):
64     name = models.CharField(_('name'), max_length=50, db_index=True)
65     slug = models.SlugField(_('slug'), max_length=120, db_index=True)
66     sort_key = models.CharField(_('sort key'), max_length=120, db_index=True)
67     category = models.CharField(_('category'), max_length=50, blank=False, null=False,
68         db_index=True, choices=TAG_CATEGORIES)
69     description = models.TextField(_('description'), blank=True)
70     main_page = models.BooleanField(_('main page'), default=False, db_index=True, help_text=_('Show tag on main page'))
71
72     user = models.ForeignKey(User, blank=True, null=True)
73     book_count = models.IntegerField(_('book count'), blank=True, null=True)
74     gazeta_link = models.CharField(blank=True, max_length=240)
75     wiki_link = models.CharField(blank=True, max_length=240)
76
77     created_at    = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True)
78     changed_at    = models.DateTimeField(_('creation date'), auto_now=True, db_index=True)
79
80     class UrlDeprecationWarning(DeprecationWarning):
81         pass
82
83     categories_rev = {
84         'autor': 'author',
85         'epoka': 'epoch',
86         'rodzaj': 'kind',
87         'gatunek': 'genre',
88         'motyw': 'theme',
89         'polka': 'set',
90     }
91     categories_dict = dict((item[::-1] for item in categories_rev.iteritems()))
92
93     class Meta:
94         ordering = ('sort_key',)
95         verbose_name = _('tag')
96         verbose_name_plural = _('tags')
97         unique_together = (("slug", "category"),)
98
99     def __unicode__(self):
100         return self.name
101
102     def __repr__(self):
103         return "Tag(slug=%r)" % self.slug
104
105     @permalink
106     def get_absolute_url(self):
107         return ('catalogue.views.tagged_object_list', [self.url_chunk])
108
109     def has_description(self):
110         return len(self.description) > 0
111     has_description.short_description = _('description')
112     has_description.boolean = True
113
114     def get_count(self):
115         """ returns global book count for book tags, fragment count for themes """
116
117         if self.book_count is None:
118             if self.category == 'book':
119                 # never used
120                 objects = Book.objects.none()
121             elif self.category == 'theme':
122                 objects = Fragment.tagged.with_all((self,))
123             else:
124                 objects = Book.tagged.with_all((self,)).order_by()
125                 if self.category != 'set':
126                     # eliminate descendants
127                     l_tags = Tag.objects.filter(slug__in=[book.book_tag_slug() for book in objects])
128                     descendants_keys = [book.pk for book in Book.tagged.with_any(l_tags)]
129                     if descendants_keys:
130                         objects = objects.exclude(pk__in=descendants_keys)
131             self.book_count = objects.count()
132             self.save()
133         return self.book_count
134
135     @staticmethod
136     def get_tag_list(tags):
137         if isinstance(tags, basestring):
138             real_tags = []
139             ambiguous_slugs = []
140             category = None
141             deprecated = False
142             tags_splitted = tags.split('/')
143             for name in tags_splitted:
144                 if category:
145                     real_tags.append(Tag.objects.get(slug=name, category=category))
146                     category = None
147                 elif name in Tag.categories_rev:
148                     category = Tag.categories_rev[name]
149                 else:
150                     try:
151                         real_tags.append(Tag.objects.exclude(category='book').get(slug=name))
152                         deprecated = True 
153                     except Tag.MultipleObjectsReturned, e:
154                         ambiguous_slugs.append(name)
155
156             if category:
157                 # something strange left off
158                 raise Tag.DoesNotExist()
159             if ambiguous_slugs:
160                 # some tags should be qualified
161                 e = Tag.MultipleObjectsReturned()
162                 e.tags = real_tags
163                 e.ambiguous_slugs = ambiguous_slugs
164                 raise e
165             if deprecated:
166                 e = Tag.UrlDeprecationWarning()
167                 e.tags = real_tags
168                 raise e
169             return real_tags
170         else:
171             return TagBase.get_tag_list(tags)
172
173     @property
174     def url_chunk(self):
175         return '/'.join((Tag.categories_dict[self.category], self.slug))
176
177
178 def get_dynamic_path(media, filename, ext=None, maxlen=100):
179     from slughifi import slughifi
180
181     # how to put related book's slug here?
182     if not ext:
183         if media.type == 'daisy':
184             ext = 'daisy.zip'
185         else:
186             ext = media.type
187     if media is None or not media.name:
188         name = slughifi(filename.split(".")[0])
189     else:
190         name = slughifi(media.name)
191     return 'book/%s/%s.%s' % (ext, name[:maxlen-len('book/%s/.%s' % (ext, ext))-4], ext)
192
193
194 # TODO: why is this hard-coded ?
195 def book_upload_path(ext=None, maxlen=100):
196     return lambda *args: get_dynamic_path(*args, ext=ext, maxlen=maxlen)
197
198
199 def get_customized_pdf_path(book, customizations):
200     """
201     Returns a MEDIA_ROOT relative path for a customized pdf. The name will contain a hash of customization options.
202     """
203     customizations.sort()
204     h = hash(tuple(customizations))
205     pdf_name = '%s-custom-%s' % (book.slug, h)
206     pdf_file = models.get_dynamic_path(None, pdf_name, ext='pdf')
207     return pdf_file
208
209
210 def get_existing_customized_pdf(book):
211     """
212     Returns a list of paths to generated customized pdf of a book
213     """
214     pdf_glob = '%s-custom-' % (book.slug,)
215     pdf_glob = get_dynamic_path(None, pdf_glob, ext='pdf')
216     pdf_glob = re.sub(r"[.]([a-z0-9]+)$", "*.\\1", pdf_glob)
217     return glob(path.join(settings.MEDIA_ROOT, pdf_glob))
218
219
220 class BookMedia(models.Model):
221     type        = models.CharField(_('type'), choices=MEDIA_FORMATS, max_length="100")
222     name        = models.CharField(_('name'), max_length="100")
223     file        = OverwritingFileField(_('file'), upload_to=book_upload_path())
224     uploaded_at = models.DateTimeField(_('creation date'), auto_now_add=True, editable=False)
225     extra_info  = JSONField(_('extra information'), default='{}', editable=False)
226     book = models.ForeignKey('Book', related_name='media')
227     source_sha1 = models.CharField(null=True, blank=True, max_length=40, editable=False)
228
229     def __unicode__(self):
230         return "%s (%s)" % (self.name, self.file.name.split("/")[-1])
231
232     class Meta:
233         ordering            = ('type', 'name')
234         verbose_name        = _('book media')
235         verbose_name_plural = _('book media')
236
237     def save(self, *args, **kwargs):
238         from slughifi import slughifi
239         from catalogue.utils import ExistingFile, remove_zip
240
241         try:
242             old = BookMedia.objects.get(pk=self.pk)
243         except BookMedia.DoesNotExist, e:
244             pass
245         else:
246             # if name changed, change the file name, too
247             if slughifi(self.name) != slughifi(old.name):
248                 self.file.save(None, ExistingFile(self.file.path), save=False, leave=True)
249
250         super(BookMedia, self).save(*args, **kwargs)
251
252         # remove the zip package for book with modified media
253         remove_zip(self.book.slug)
254
255         extra_info = self.get_extra_info_value()
256         extra_info.update(self.read_meta())
257         self.set_extra_info_value(extra_info)
258         self.source_sha1 = self.read_source_sha1(self.file.path, self.type)
259         return super(BookMedia, self).save(*args, **kwargs)
260
261     def read_meta(self):
262         """
263             Reads some metadata from the audiobook.
264         """
265         import mutagen
266         from mutagen import id3
267
268         artist_name = director_name = project = funded_by = ''
269         if self.type == 'mp3':
270             try:
271                 audio = id3.ID3(self.file.path)
272                 artist_name = ', '.join(', '.join(tag.text) for tag in audio.getall('TPE1'))
273                 director_name = ', '.join(', '.join(tag.text) for tag in audio.getall('TPE3'))
274                 project = ", ".join([t.data for t in audio.getall('PRIV') 
275                         if t.owner=='wolnelektury.pl?project'])
276                 funded_by = ", ".join([t.data for t in audio.getall('PRIV') 
277                         if t.owner=='wolnelektury.pl?funded_by'])
278             except:
279                 pass
280         elif self.type == 'ogg':
281             try:
282                 audio = mutagen.File(self.file.path)
283                 artist_name = ', '.join(audio.get('artist', []))
284                 director_name = ', '.join(audio.get('conductor', []))
285                 project = ", ".join(audio.get('project', []))
286                 funded_by = ", ".join(audio.get('funded_by', []))
287             except:
288                 pass
289         else:
290             return {}
291         return {'artist_name': artist_name, 'director_name': director_name,
292                 'project': project, 'funded_by': funded_by}
293
294     @staticmethod
295     def read_source_sha1(filepath, filetype):
296         """
297             Reads source file SHA1 from audiobok metadata.
298         """
299         import mutagen
300         from mutagen import id3
301
302         if filetype == 'mp3':
303             try:
304                 audio = id3.ID3(filepath)
305                 return [t.data for t in audio.getall('PRIV') 
306                         if t.owner=='wolnelektury.pl?flac_sha1'][0]
307             except:
308                 return None
309         elif filetype == 'ogg':
310             try:
311                 audio = mutagen.File(filepath)
312                 return audio.get('flac_sha1', [None])[0] 
313             except:
314                 return None
315         else:
316             return None
317
318
319 class Book(models.Model):
320     title         = models.CharField(_('title'), max_length=120)
321     sort_key = models.CharField(_('sort key'), max_length=120, db_index=True, editable=False)
322     slug          = models.SlugField(_('slug'), max_length=120, unique=True, db_index=True)
323     description   = models.TextField(_('description'), blank=True)
324     created_at    = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True)
325     changed_at    = models.DateTimeField(_('creation date'), auto_now=True, db_index=True)
326     parent_number = models.IntegerField(_('parent number'), default=0)
327     extra_info    = JSONField(_('extra information'), default='{}')
328     gazeta_link   = models.CharField(blank=True, max_length=240)
329     wiki_link     = models.CharField(blank=True, max_length=240)
330     # files generated during publication
331
332     file_types = ['epub', 'html', 'mobi', 'pdf', 'txt', 'xml']
333     
334     parent        = models.ForeignKey('self', blank=True, null=True, related_name='children')
335     objects  = models.Manager()
336     tagged   = managers.ModelTaggedItemManager(Tag)
337     tags     = managers.TagDescriptor(Tag)
338
339     html_built = django.dispatch.Signal()
340     published = django.dispatch.Signal()
341
342     class AlreadyExists(Exception):
343         pass
344
345     class Meta:
346         ordering = ('sort_key',)
347         verbose_name = _('book')
348         verbose_name_plural = _('books')
349
350     def __unicode__(self):
351         return self.title
352
353     def save(self, force_insert=False, force_update=False, reset_short_html=True, **kwargs):
354         from sortify import sortify
355
356         self.sort_key = sortify(self.title)
357
358         ret = super(Book, self).save(force_insert, force_update)
359
360         if reset_short_html:
361             self.reset_short_html()
362
363         return ret
364
365     @permalink
366     def get_absolute_url(self):
367         return ('catalogue.views.book_detail', [self.slug])
368
369     @property
370     def name(self):
371         return self.title
372
373     def book_tag_slug(self):
374         return ('l-' + self.slug)[:120]
375
376     def book_tag(self):
377         slug = self.book_tag_slug()
378         book_tag, created = Tag.objects.get_or_create(slug=slug, category='book')
379         if created:
380             book_tag.name = self.title[:50]
381             book_tag.sort_key = self.title.lower()
382             book_tag.save()
383         return book_tag
384
385     def has_media(self, type):
386         if type in Book.file_types:
387             return bool(getattr(self, "%s_file" % type))
388         else:
389             return self.media.filter(type=type).exists()
390
391     def get_media(self, type):
392         if self.has_media(type):
393             if type in Book.file_types:
394                 return getattr(self, "%s_file" % type)
395             else:                                             
396                 return self.media.filter(type=type)
397         else:
398             return None
399
400     def get_mp3(self):
401         return self.get_media("mp3")
402     def get_odt(self):
403         return self.get_media("odt")
404     def get_ogg(self):
405         return self.get_media("ogg")
406     def get_daisy(self):
407         return self.get_media("daisy")                       
408
409     def reset_short_html(self):
410         if self.id is None:
411             return
412
413         cache_key = "Book.short_html/%d/%s"
414         for lang, langname in settings.LANGUAGES:
415             cache.delete(cache_key % (self.id, lang))
416         # Fragment.short_html relies on book's tags, so reset it here too
417         for fragm in self.fragments.all():
418             fragm.reset_short_html()
419
420     def short_html(self):
421         if self.id:
422             cache_key = "Book.short_html/%d/%s" % (self.id, get_language())
423             short_html = cache.get(cache_key)
424         else:
425             short_html = None
426
427         if short_html is not None:
428             return mark_safe(short_html)
429         else:
430             tags = self.tags.filter(~Q(category__in=('set', 'theme', 'book')))
431             tags = [mark_safe(u'<a href="%s">%s</a>' % (tag.get_absolute_url(), tag.name)) for tag in tags]
432
433             formats = []
434             # files generated during publication
435             if self.has_media("html"):
436                 formats.append(u'<a href="%s">%s</a>' % (reverse('book_text', kwargs={'slug': self.slug}), _('Read online')))
437             if self.has_media("pdf"):
438                 formats.append(u'<a href="%s">PDF</a>' % self.get_media('pdf').url)
439             if self.has_media("mobi"):
440                 formats.append(u'<a href="%s">MOBI</a>' % self.get_media('mobi').url)
441             if self.root_ancestor.has_media("epub"):
442                 formats.append(u'<a href="%s">EPUB</a>' % self.root_ancestor.get_media('epub').url)
443             if self.has_media("txt"):
444                 formats.append(u'<a href="%s">TXT</a>' % self.get_media('txt').url)
445             # other files
446             for m in self.media.order_by('type'):
447                 formats.append(u'<a href="%s">%s</a>' % (m.file.url, m.type.upper()))
448
449             formats = [mark_safe(format) for format in formats]
450
451             short_html = unicode(render_to_string('catalogue/book_short.html',
452                 {'book': self, 'tags': tags, 'formats': formats}))
453
454             if self.id:
455                 cache.set(cache_key, short_html, CACHE_FOREVER)
456             return mark_safe(short_html)
457
458     @property
459     def root_ancestor(self):
460         """ returns the oldest ancestor """
461
462         if not hasattr(self, '_root_ancestor'):
463             book = self
464             while book.parent:
465                 book = book.parent
466             self._root_ancestor = book
467         return self._root_ancestor
468
469
470     def has_description(self):
471         return len(self.description) > 0
472     has_description.short_description = _('description')
473     has_description.boolean = True
474
475     # ugly ugly ugly
476     def has_odt_file(self):
477         return bool(self.has_media("odt"))
478     has_odt_file.short_description = 'ODT'
479     has_odt_file.boolean = True
480
481     def has_mp3_file(self):
482         return bool(self.has_media("mp3"))
483     has_mp3_file.short_description = 'MP3'
484     has_mp3_file.boolean = True
485
486     def has_ogg_file(self):
487         return bool(self.has_media("ogg"))
488     has_ogg_file.short_description = 'OGG'
489     has_ogg_file.boolean = True
490
491     def has_daisy_file(self):
492         return bool(self.has_media("daisy"))
493     has_daisy_file.short_description = 'DAISY'
494     has_daisy_file.boolean = True
495
496     def build_pdf(self, customizations=None, file_name=None):
497         """ (Re)builds the pdf file.
498         customizations - customizations which are passed to LaTeX class file.
499         file_name - save the pdf file under a different name and DO NOT save it in db.
500         """
501         from tempfile import NamedTemporaryFile
502         from os import unlink
503         from django.core.files import File
504         from librarian import pdf
505         from catalogue.utils import ORMDocProvider, remove_zip
506
507         try:
508             pdf_file = NamedTemporaryFile(delete=False)
509             pdf.transform(ORMDocProvider(self),
510                       file_path=str(self.xml_file.path),
511                       output_file=pdf_file,
512                       customizations=customizations
513                       )
514
515             if file_name is None:
516                 # we'd like to be sure not to overwrite changes happening while
517                 # (timely) pdf generation is taking place (async celery scenario)
518                 current_self = Book.objects.get(id=self.id)
519                 current_self.pdf_file.save('%s.pdf' % self.slug, File(open(pdf_file.name)))
520                 self.pdf_file = current_self.pdf_file
521             else:
522                 print "safing %s" % file_name
523                 print "to: %s" % DefaultStorage().path(file_name)
524                 DefaultStorage().save(file_name, File(open(pdf_file.name)))
525         finally:
526             unlink(pdf_file.name)
527
528         # remove cached downloadables
529         remove_zip(settings.ALL_PDF_ZIP)
530         for customized_pdf in get_existing_customized_pdf(self):
531             unlink(customized_pdf)
532
533     def build_mobi(self):
534         """ (Re)builds the MOBI file.
535
536         """
537         from tempfile import NamedTemporaryFile
538         from os import unlink
539         from django.core.files import File
540         from librarian import mobi
541         from catalogue.utils import ORMDocProvider, remove_zip
542
543         try:
544             mobi_file = NamedTemporaryFile(suffix='.mobi', delete=False)
545             mobi.transform(ORMDocProvider(self), verbose=1,
546                       file_path=str(self.xml_file.path),
547                       output_file=mobi_file.name,
548                       )
549
550             self.mobi_file.save('%s.mobi' % self.slug, File(open(mobi_file.name)))
551         finally:
552             unlink(mobi_file.name)
553
554         # remove zip with all mobi files
555         remove_zip(settings.ALL_MOBI_ZIP)
556
557     def build_epub(self, remove_descendants=True):
558         """ (Re)builds the epub file.
559             If book has a parent, does nothing.
560             Unless remove_descendants is False, descendants' epubs are removed.
561         """
562         from StringIO import StringIO
563         from hashlib import sha1
564         from django.core.files.base import ContentFile
565         from librarian import epub, NoDublinCore
566         from catalogue.utils import ORMDocProvider, remove_zip
567
568         if self.parent:
569             # don't need an epub
570             return
571
572         epub_file = StringIO()
573         try:
574             epub.transform(ORMDocProvider(self), self.slug, output_file=epub_file)
575             self.epub_file.save('%s.epub' % self.slug, ContentFile(epub_file.getvalue()))
576             FileRecord(slug=self.slug, type='epub', sha1=sha1(epub_file.getvalue()).hexdigest()).save()
577         except NoDublinCore:
578             pass
579
580         book_descendants = list(self.children.all())
581         while len(book_descendants) > 0:
582             child_book = book_descendants.pop(0)
583             if remove_descendants and child_book.has_epub_file():
584                 child_book.epub_file.delete()
585             # save anyway, to refresh short_html
586             child_book.save()
587             book_descendants += list(child_book.children.all())
588
589         # remove zip package with all epub files
590         remove_zip(settings.ALL_EPUB_ZIP)
591
592     def build_txt(self):
593         from StringIO import StringIO
594         from django.core.files.base import ContentFile
595         from librarian import text
596
597         out = StringIO()
598         text.transform(open(self.xml_file.path), out)
599         self.txt_file.save('%s.txt' % self.slug, ContentFile(out.getvalue()))
600
601
602     def build_html(self):
603         from tempfile import NamedTemporaryFile
604         from markupstring import MarkupString
605         from django.core.files import File
606         from slughifi import slughifi
607         from librarian import html
608
609         meta_tags = list(self.tags.filter(
610             category__in=('author', 'epoch', 'genre', 'kind')))
611         book_tag = self.book_tag()
612
613         html_file = NamedTemporaryFile()
614         if html.transform(self.xml_file.path, html_file, parse_dublincore=False):
615             self.html_file.save('%s.html' % self.slug, File(html_file))
616
617             # get ancestor l-tags for adding to new fragments
618             ancestor_tags = []
619             p = self.parent
620             while p:
621                 ancestor_tags.append(p.book_tag())
622                 p = p.parent
623
624             # Delete old fragments and create them from scratch
625             self.fragments.all().delete()
626             # Extract fragments
627             closed_fragments, open_fragments = html.extract_fragments(self.html_file.path)
628             for fragment in closed_fragments.values():
629                 try:
630                     theme_names = [s.strip() for s in fragment.themes.split(',')]
631                 except AttributeError:
632                     continue
633                 themes = []
634                 for theme_name in theme_names:
635                     if not theme_name:
636                         continue
637                     tag, created = Tag.objects.get_or_create(slug=slughifi(theme_name), category='theme')
638                     if created:
639                         tag.name = theme_name
640                         tag.sort_key = theme_name.lower()
641                         tag.save()
642                     themes.append(tag)
643                 if not themes:
644                     continue
645
646                 text = fragment.to_string()
647                 short_text = ''
648                 if (len(MarkupString(text)) > 240):
649                     short_text = unicode(MarkupString(text)[:160])
650                 new_fragment = Fragment.objects.create(anchor=fragment.id, book=self,
651                     text=text, short_text=short_text)
652
653                 new_fragment.save()
654                 new_fragment.tags = set(meta_tags + themes + [book_tag] + ancestor_tags)
655             self.save()
656             self.html_built.send(sender=self)
657             return True
658         return False
659
660     @staticmethod
661     def zip_format(format_):
662         def pretty_file_name(book):
663             return "%s/%s.%s" % (
664                 b.get_extra_info_value()['author'],
665                 b.slug,
666                 format_)
667
668         field_name = "%s_file" % format_
669         books = Book.objects.filter(parent=None).exclude(**{field_name: ""})
670         paths = [(pretty_file_name(b), getattr(b, field_name).path)
671                     for b in books]
672         result = create_zip.delay(paths,
673                     getattr(settings, "ALL_%s_ZIP" % format_.upper()))
674         return result.wait()
675
676     def zip_audiobooks(self):
677         bm = BookMedia.objects.filter(book=self, type='mp3')
678         paths = map(lambda bm: (None, bm.file.path), bm)
679         result = create_zip.delay(paths, self.slug)
680         return result.wait()
681
682     @classmethod
683     def from_xml_file(cls, xml_file, **kwargs):
684         from django.core.files import File
685         from librarian import dcparser
686
687         # use librarian to parse meta-data
688         book_info = dcparser.parse(xml_file)
689
690         if not isinstance(xml_file, File):
691             xml_file = File(open(xml_file))
692
693         try:
694             return cls.from_text_and_meta(xml_file, book_info, **kwargs)
695         finally:
696             xml_file.close()
697
698     @classmethod
699     def from_text_and_meta(cls, raw_file, book_info, overwrite=False,
700             build_epub=True, build_txt=True, build_pdf=True, build_mobi=True):
701         import re
702         from slughifi import slughifi
703         from sortify import sortify
704
705         # check for parts before we do anything
706         children = []
707         if hasattr(book_info, 'parts'):
708             for part_url in book_info.parts:
709                 base, slug = part_url.rsplit('/', 1)
710                 try:
711                     children.append(Book.objects.get(slug=slug))
712                 except Book.DoesNotExist, e:
713                     raise Book.DoesNotExist(_('Book with slug = "%s" does not exist.') % slug)
714
715
716         # Read book metadata
717         book_base, book_slug = book_info.url.rsplit('/', 1)
718         if re.search(r'[^a-zA-Z0-9-]', book_slug):
719             raise ValueError('Invalid characters in slug')
720         book, created = Book.objects.get_or_create(slug=book_slug)
721
722         if created:
723             book_shelves = []
724         else:
725             if not overwrite:
726                 raise Book.AlreadyExists(_('Book %s already exists') % book_slug)
727             # Save shelves for this book
728             book_shelves = list(book.tags.filter(category='set'))
729
730         book.title = book_info.title
731         book.set_extra_info_value(book_info.to_dict())
732         book.save()
733
734         meta_tags = []
735         categories = (('kinds', 'kind'), ('genres', 'genre'), ('authors', 'author'), ('epochs', 'epoch'))
736         for field_name, category in categories:
737             try:
738                 tag_names = getattr(book_info, field_name)
739             except:
740                 tag_names = [getattr(book_info, category)]
741             for tag_name in tag_names:
742                 tag_sort_key = tag_name
743                 if category == 'author':
744                     tag_sort_key = tag_name.last_name
745                     tag_name = ' '.join(tag_name.first_names) + ' ' + tag_name.last_name
746                 tag, created = Tag.objects.get_or_create(slug=slughifi(tag_name), category=category)
747                 if created:
748                     tag.name = tag_name
749                     tag.sort_key = sortify(tag_sort_key.lower())
750                     tag.save()
751                 meta_tags.append(tag)
752
753         book.tags = set(meta_tags + book_shelves)
754
755         book_tag = book.book_tag()
756
757         for n, child_book in enumerate(children):
758             child_book.parent = book
759             child_book.parent_number = n
760             child_book.save()
761
762         # Save XML and HTML files
763         book.xml_file.save('%s.xml' % book.slug, raw_file, save=False)
764
765         # delete old fragments when overwriting
766         book.fragments.all().delete()
767
768         if book.build_html():
769             if not settings.NO_BUILD_TXT and build_txt:
770                 book.build_txt()
771
772         if not settings.NO_BUILD_EPUB and build_epub:
773             book.root_ancestor.build_epub()
774
775         if not settings.NO_BUILD_PDF and build_pdf:
776             book.root_ancestor.build_pdf()
777
778         if not settings.NO_BUILD_MOBI and build_mobi:
779             book.build_mobi()
780
781         book_descendants = list(book.children.all())
782         # add l-tag to descendants and their fragments
783         # delete unnecessary EPUB files
784         while len(book_descendants) > 0:
785             child_book = book_descendants.pop(0)
786             child_book.tags = list(child_book.tags) + [book_tag]
787             child_book.save()
788             for fragment in child_book.fragments.all():
789                 fragment.tags = set(list(fragment.tags) + [book_tag])
790             book_descendants += list(child_book.children.all())
791
792         book.save()
793
794         # refresh cache
795         book.reset_tag_counter()
796         book.reset_theme_counter()
797
798         cls.published.send(sender=book)
799         return book
800
801     def reset_tag_counter(self):
802         if self.id is None:
803             return
804
805         cache_key = "Book.tag_counter/%d" % self.id
806         cache.delete(cache_key)
807         if self.parent:
808             self.parent.reset_tag_counter()
809
810     @property
811     def tag_counter(self):
812         if self.id:
813             cache_key = "Book.tag_counter/%d" % self.id
814             tags = cache.get(cache_key)
815         else:
816             tags = None
817
818         if tags is None:
819             tags = {}
820             for child in self.children.all().order_by():
821                 for tag_pk, value in child.tag_counter.iteritems():
822                     tags[tag_pk] = tags.get(tag_pk, 0) + value
823             for tag in self.tags.exclude(category__in=('book', 'theme', 'set')).order_by():
824                 tags[tag.pk] = 1
825
826             if self.id:
827                 cache.set(cache_key, tags, CACHE_FOREVER)
828         return tags
829
830     def reset_theme_counter(self):
831         if self.id is None:
832             return
833
834         cache_key = "Book.theme_counter/%d" % self.id
835         cache.delete(cache_key)
836         if self.parent:
837             self.parent.reset_theme_counter()
838
839     @property
840     def theme_counter(self):
841         if self.id:
842             cache_key = "Book.theme_counter/%d" % self.id
843             tags = cache.get(cache_key)
844         else:
845             tags = None
846
847         if tags is None:
848             tags = {}
849             for fragment in Fragment.tagged.with_any([self.book_tag()]).order_by():
850                 for tag in fragment.tags.filter(category='theme').order_by():
851                     tags[tag.pk] = tags.get(tag.pk, 0) + 1
852
853             if self.id:
854                 cache.set(cache_key, tags, CACHE_FOREVER)
855         return tags
856
857     def pretty_title(self, html_links=False):
858         book = self
859         names = list(book.tags.filter(category='author'))
860
861         books = []
862         while book:
863             books.append(book)
864             book = book.parent
865         names.extend(reversed(books))
866
867         if html_links:
868             names = ['<a href="%s">%s</a>' % (tag.get_absolute_url(), tag.name) for tag in names]
869         else:
870             names = [tag.name for tag in names]
871
872         return ', '.join(names)
873
874     @classmethod
875     def tagged_top_level(cls, tags):
876         """ Returns top-level books tagged with `tags'.
877
878         It only returns those books which don't have ancestors which are
879         also tagged with those tags.
880
881         """
882         # get relevant books and their tags
883         objects = cls.tagged.with_all(tags)
884         # eliminate descendants
885         l_tags = Tag.objects.filter(category='book', slug__in=[book.book_tag_slug() for book in objects])
886         descendants_keys = [book.pk for book in cls.tagged.with_any(l_tags)]
887         if descendants_keys:
888             objects = objects.exclude(pk__in=descendants_keys)
889
890         return objects
891
892     @classmethod
893     def book_list(cls, filter=None):
894         """Generates a hierarchical listing of all books.
895
896         Books are optionally filtered with a test function.
897
898         """
899
900         books_by_parent = {}
901         books = cls.objects.all().order_by('parent_number', 'sort_key').only('title', 'parent', 'slug')
902         if filter:
903             books = books.filter(filter).distinct()
904             book_ids = set((book.pk for book in books))
905             for book in books:
906                 parent = book.parent_id
907                 if parent not in book_ids:
908                     parent = None
909                 books_by_parent.setdefault(parent, []).append(book)
910         else:
911             for book in books:
912                 books_by_parent.setdefault(book.parent_id, []).append(book)
913
914         orphans = []
915         books_by_author = SortedDict()
916         for tag in Tag.objects.filter(category='author'):
917             books_by_author[tag] = []
918
919         for book in books_by_parent.get(None,()):
920             authors = list(book.tags.filter(category='author'))
921             if authors:
922                 for author in authors:
923                     books_by_author[author].append(book)
924             else:
925                 orphans.append(book)
926
927         return books_by_author, orphans, books_by_parent
928
929     _audiences_pl = {
930         "SP1": (1, u"szkoła podstawowa"),
931         "SP2": (1, u"szkoła podstawowa"),
932         "P": (1, u"szkoła podstawowa"),
933         "G": (2, u"gimnazjum"),
934         "L": (3, u"liceum"),
935         "LP": (3, u"liceum"),
936     }
937     def audiences_pl(self):
938         audiences = self.get_extra_info_value().get('audiences', [])
939         audiences = sorted(set([self._audiences_pl[a] for a in audiences]))
940         return [a[1] for a in audiences]
941
942
943 def _has_factory(ftype):
944     has = lambda self: bool(getattr(self, "%s_file" % ftype))
945     has.short_description = t.upper()
946     has.boolean = True
947     has.__name__ = "has_%s_file" % ftype
948     return has
949
950     
951 # add the file fields
952 for t in Book.file_types:
953     field_name = "%s_file" % t
954     models.FileField(_("%s file" % t.upper()),
955             upload_to=book_upload_path(t),
956             blank=True).contribute_to_class(Book, field_name)
957
958     setattr(Book, "has_%s_file" % t, _has_factory(t))
959
960
961 class Fragment(models.Model):
962     text = models.TextField()
963     short_text = models.TextField(editable=False)
964     anchor = models.CharField(max_length=120)
965     book = models.ForeignKey(Book, related_name='fragments')
966
967     objects = models.Manager()
968     tagged = managers.ModelTaggedItemManager(Tag)
969     tags = managers.TagDescriptor(Tag)
970
971     class Meta:
972         ordering = ('book', 'anchor',)
973         verbose_name = _('fragment')
974         verbose_name_plural = _('fragments')
975
976     def get_absolute_url(self):
977         return '%s#m%s' % (reverse('book_text', kwargs={'slug': self.book.slug}), self.anchor)
978
979     def reset_short_html(self):
980         if self.id is None:
981             return
982
983         cache_key = "Fragment.short_html/%d/%s"
984         for lang, langname in settings.LANGUAGES:
985             cache.delete(cache_key % (self.id, lang))
986
987     def short_html(self):
988         if self.id:
989             cache_key = "Fragment.short_html/%d/%s" % (self.id, get_language())
990             short_html = cache.get(cache_key)
991         else:
992             short_html = None
993
994         if short_html is not None:
995             return mark_safe(short_html)
996         else:
997             short_html = unicode(render_to_string('catalogue/fragment_short.html',
998                 {'fragment': self}))
999             if self.id:
1000                 cache.set(cache_key, short_html, CACHE_FOREVER)
1001             return mark_safe(short_html)
1002
1003
1004 class FileRecord(models.Model):
1005     slug = models.SlugField(_('slug'), max_length=120, db_index=True)
1006     type = models.CharField(_('type'), max_length=20, db_index=True)
1007     sha1 = models.CharField(_('sha-1 hash'), max_length=40)
1008     time = models.DateTimeField(_('time'), auto_now_add=True)
1009
1010     class Meta:
1011         ordering = ('-time','-slug', '-type')
1012         verbose_name = _('file record')
1013         verbose_name_plural = _('file records')
1014
1015     def __unicode__(self):
1016         return "%s %s.%s" % (self.sha1,  self.slug, self.type)
1017
1018 ###########
1019 #
1020 # SIGNALS
1021 #
1022 ###########
1023
1024
1025 def _tags_updated_handler(sender, affected_tags, **kwargs):
1026     # reset tag global counter
1027     # we want Tag.changed_at updated for API to know the tag was touched
1028     Tag.objects.filter(pk__in=[tag.pk for tag in affected_tags]).update(book_count=None, changed_at=datetime.now())
1029
1030     # if book tags changed, reset book tag counter
1031     if isinstance(sender, Book) and \
1032                 Tag.objects.filter(pk__in=(tag.pk for tag in affected_tags)).\
1033                     exclude(category__in=('book', 'theme', 'set')).count():
1034         sender.reset_tag_counter()
1035     # if fragment theme changed, reset book theme counter
1036     elif isinstance(sender, Fragment) and \
1037                 Tag.objects.filter(pk__in=(tag.pk for tag in affected_tags)).\
1038                     filter(category='theme').count():
1039         sender.book.reset_theme_counter()
1040 tags_updated.connect(_tags_updated_handler)
1041
1042
1043 def _pre_delete_handler(sender, instance, **kwargs):
1044     """ refresh Book on BookMedia delete """
1045     if sender == BookMedia:
1046         instance.book.save()
1047 pre_delete.connect(_pre_delete_handler)
1048
1049 def _post_save_handler(sender, instance, **kwargs):
1050     """ refresh all the short_html stuff on BookMedia update """
1051     if sender == BookMedia:
1052         instance.book.save()
1053 post_save.connect(_post_save_handler)