apps/catalogue/models.py

   1 # -*- coding: utf-8 -*-
   2 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
   3 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
   4 #
   5 from datetime import datetime
   6
   7 from django.db import models
   8 from django.db.models import permalink, Q
   9 import django.dispatch
  10 from django.core.cache import cache
  11 from django.utils.translation import ugettext_lazy as _
  12 from django.contrib.auth.models import User
  13 from django.core.files import File
  14 from django.template.loader import render_to_string
  15 from django.utils.safestring import mark_safe
  16 from django.utils.translation import get_language
  17 from django.core.urlresolvers import reverse
  18 from django.db.models.signals import post_save, m2m_changed, pre_delete
  19
  20 from django.conf import settings
  21
  22 from newtagging.models import TagBase, tags_updated
  23 from newtagging import managers
  24 from catalogue.fields import JSONField, OverwritingFileField
  25 from catalogue.utils import ExistingFile, ORMDocProvider, create_zip, remove_zip
  26
  27 from librarian import dcparser, html, epub, NoDublinCore
  28 import mutagen
  29 from mutagen import id3
  30 from slughifi import slughifi
  31 from sortify import sortify
  32 from os import unlink
  33
  34 import search
  35
  36 TAG_CATEGORIES = (
  37     ('author', _('author')),
  38     ('epoch', _('epoch')),
  39     ('kind', _('kind')),
  40     ('genre', _('genre')),
  41     ('theme', _('theme')),
  42     ('set', _('set')),
  43     ('book', _('book')),
  44 )
  45
  46 MEDIA_FORMATS = (
  47     ('odt', _('ODT file')),
  48     ('mp3', _('MP3 file')),
  49     ('ogg', _('OGG file')),
  50     ('daisy', _('DAISY file')),
  51 )
  52
  53 # not quite, but Django wants you to set a timeout
  54 CACHE_FOREVER = 2419200  # 28 days
  55
  56
  57 class TagSubcategoryManager(models.Manager):
  58     def __init__(self, subcategory):
  59         super(TagSubcategoryManager, self).__init__()
  60         self.subcategory = subcategory
  61
  62     def get_query_set(self):
  63         return super(TagSubcategoryManager, self).get_query_set().filter(category=self.subcategory)
  64
  65
  66 class Tag(TagBase):
  67     name = models.CharField(_('name'), max_length=50, db_index=True)
  68     slug = models.SlugField(_('slug'), max_length=120, db_index=True)
  69     sort_key = models.CharField(_('sort key'), max_length=120, db_index=True)
  70     category = models.CharField(_('category'), max_length=50, blank=False, null=False,
  71         db_index=True, choices=TAG_CATEGORIES)
  72     description = models.TextField(_('description'), blank=True)
  73     main_page = models.BooleanField(_('main page'), default=False, db_index=True, help_text=_('Show tag on main page'))
  74
  75     user = models.ForeignKey(User, blank=True, null=True)
  76     book_count = models.IntegerField(_('book count'), blank=True, null=True)
  77     gazeta_link = models.CharField(blank=True, max_length=240)
  78     wiki_link = models.CharField(blank=True, max_length=240)
  79
  80     created_at    = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True)
  81     changed_at    = models.DateTimeField(_('creation date'), auto_now=True, db_index=True)
  82
  83     class UrlDeprecationWarning(DeprecationWarning):
  84         pass
  85
  86     categories_rev = {
  87         'autor': 'author',
  88         'epoka': 'epoch',
  89         'rodzaj': 'kind',
  90         'gatunek': 'genre',
  91         'motyw': 'theme',
  92         'polka': 'set',
  93     }
  94     categories_dict = dict((item[::-1] for item in categories_rev.iteritems()))
  95
  96     class Meta:
  97         ordering = ('sort_key',)
  98         verbose_name = _('tag')
  99         verbose_name_plural = _('tags')
 100         unique_together = (("slug", "category"),)
 101
 102     def __unicode__(self):
 103         return self.name
 104
 105     def __repr__(self):
 106         return "Tag(slug=%r)" % self.slug
 107
 108     @permalink
 109     def get_absolute_url(self):
 110         return ('catalogue.views.tagged_object_list', [self.url_chunk])
 111
 112     def has_description(self):
 113         return len(self.description) > 0
 114     has_description.short_description = _('description')
 115     has_description.boolean = True
 116
 117     def get_count(self):
 118         """ returns global book count for book tags, fragment count for themes """
 119
 120         if self.book_count is None:
 121             if self.category == 'book':
 122                 # never used
 123                 objects = Book.objects.none()
 124             elif self.category == 'theme':
 125                 objects = Fragment.tagged.with_all((self,))
 126             else:
 127                 objects = Book.tagged.with_all((self,)).order_by()
 128                 if self.category != 'set':
 129                     # eliminate descendants
 130                     l_tags = Tag.objects.filter(slug__in=[book.book_tag_slug() for book in objects])
 131                     descendants_keys = [book.pk for book in Book.tagged.with_any(l_tags)]
 132                     if descendants_keys:
 133                         objects = objects.exclude(pk__in=descendants_keys)
 134             self.book_count = objects.count()
 135             self.save()
 136         return self.book_count
 137
 138     @staticmethod
 139     def get_tag_list(tags):
 140         if isinstance(tags, basestring):
 141             real_tags = []
 142             ambiguous_slugs = []
 143             category = None
 144             deprecated = False
 145             tags_splitted = tags.split('/')
 146             for name in tags_splitted:
 147                 if category:
 148                     real_tags.append(Tag.objects.get(slug=name, category=category))
 149                     category = None
 150                 elif name in Tag.categories_rev:
 151                     category = Tag.categories_rev[name]
 152                 else:
 153                     try:
 154                         real_tags.append(Tag.objects.exclude(category='book').get(slug=name))
 155                         deprecated = True
 156                     except Tag.MultipleObjectsReturned, e:
 157                         ambiguous_slugs.append(name)
 158
 159             if category:
 160                 # something strange left off
 161                 raise Tag.DoesNotExist()
 162             if ambiguous_slugs:
 163                 # some tags should be qualified
 164                 e = Tag.MultipleObjectsReturned()
 165                 e.tags = real_tags
 166                 e.ambiguous_slugs = ambiguous_slugs
 167                 raise e
 168             if deprecated:
 169                 e = Tag.UrlDeprecationWarning()
 170                 e.tags = real_tags
 171                 raise e
 172             return real_tags
 173         else:
 174             return TagBase.get_tag_list(tags)
 175
 176     @property
 177     def url_chunk(self):
 178         return '/'.join((Tag.categories_dict[self.category], self.slug))
 179
 180
 181 # TODO: why is this hard-coded ?
 182 def book_upload_path(ext=None, maxlen=100):
 183     def get_dynamic_path(media, filename, ext=ext):
 184         # how to put related book's slug here?
 185         if not ext:
 186             if media.type == 'daisy':
 187                 ext = 'daisy.zip'
 188             else:
 189                 ext = media.type
 190         if not media.name:
 191             name = slughifi(filename.split(".")[0])
 192         else:
 193             name = slughifi(media.name)
 194         return 'book/%s/%s.%s' % (ext, name[:maxlen-len('book/%s/.%s' % (ext, ext))-4], ext)
 195     return get_dynamic_path
 196
 197
 198 class BookMedia(models.Model):
 199     type        = models.CharField(_('type'), choices=MEDIA_FORMATS, max_length="100")
 200     name        = models.CharField(_('name'), max_length="100")
 201     file        = OverwritingFileField(_('file'), upload_to=book_upload_path())
 202     uploaded_at = models.DateTimeField(_('creation date'), auto_now_add=True, editable=False)
 203     extra_info  = JSONField(_('extra information'), default='{}', editable=False)
 204     book = models.ForeignKey('Book', related_name='media')
 205     source_sha1 = models.CharField(null=True, blank=True, max_length=40, editable=False)
 206
 207     def __unicode__(self):
 208         return "%s (%s)" % (self.name, self.file.name.split("/")[-1])
 209
 210     class Meta:
 211         ordering            = ('type', 'name')
 212         verbose_name        = _('book media')
 213         verbose_name_plural = _('book media')
 214
 215     def save(self, *args, **kwargs):
 216         try:
 217             old = BookMedia.objects.get(pk=self.pk)
 218         except BookMedia.DoesNotExist, e:
 219             pass
 220         else:
 221             # if name changed, change the file name, too
 222             if slughifi(self.name) != slughifi(old.name):
 223                 self.file.save(None, ExistingFile(self.file.path), save=False, leave=True)
 224
 225         super(BookMedia, self).save(*args, **kwargs)
 226
 227         # remove the zip package for book with modified media
 228         remove_zip(self.book.slug)
 229
 230         extra_info = self.get_extra_info_value()
 231         extra_info.update(self.read_meta())
 232         self.set_extra_info_value(extra_info)
 233         self.source_sha1 = self.read_source_sha1(self.file.path, self.type)
 234         return super(BookMedia, self).save(*args, **kwargs)
 235
 236     def read_meta(self):
 237         """
 238             Reads some metadata from the audiobook.
 239         """
 240
 241         artist_name = director_name = project = funded_by = ''
 242         if self.type == 'mp3':
 243             try:
 244                 audio = id3.ID3(self.file.path)
 245                 artist_name = ', '.join(', '.join(tag.text) for tag in audio.getall('TPE1'))
 246                 director_name = ', '.join(', '.join(tag.text) for tag in audio.getall('TPE3'))
 247                 project = ", ".join([t.data for t in audio.getall('PRIV')
 248                         if t.owner=='wolnelektury.pl?project'])
 249                 funded_by = ", ".join([t.data for t in audio.getall('PRIV')
 250                         if t.owner=='wolnelektury.pl?funded_by'])
 251             except:
 252                 pass
 253         elif self.type == 'ogg':
 254             try:
 255                 audio = mutagen.File(self.file.path)
 256                 artist_name = ', '.join(audio.get('artist', []))
 257                 director_name = ', '.join(audio.get('conductor', []))
 258                 project = ", ".join(audio.get('project', []))
 259                 funded_by = ", ".join(audio.get('funded_by', []))
 260             except:
 261                 pass
 262         else:
 263             return {}
 264         return {'artist_name': artist_name, 'director_name': director_name,
 265                 'project': project, 'funded_by': funded_by}
 266
 267     @staticmethod
 268     def read_source_sha1(filepath, filetype):
 269         """
 270             Reads source file SHA1 from audiobok metadata.
 271         """
 272
 273         if filetype == 'mp3':
 274             try:
 275                 audio = id3.ID3(filepath)
 276                 return [t.data for t in audio.getall('PRIV')
 277                         if t.owner=='wolnelektury.pl?flac_sha1'][0]
 278             except:
 279                 return None
 280         elif filetype == 'ogg':
 281             try:
 282                 audio = mutagen.File(filepath)
 283                 return audio.get('flac_sha1', [None])[0]
 284             except:
 285                 return None
 286         else:
 287             return None
 288
 289
 290 class Book(models.Model):
 291     title         = models.CharField(_('title'), max_length=120)
 292     sort_key = models.CharField(_('sort key'), max_length=120, db_index=True, editable=False)
 293     slug          = models.SlugField(_('slug'), max_length=120, unique=True, db_index=True)
 294     description   = models.TextField(_('description'), blank=True)
 295     created_at    = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True)
 296     changed_at    = models.DateTimeField(_('creation date'), auto_now=True, db_index=True)
 297     parent_number = models.IntegerField(_('parent number'), default=0)
 298     extra_info    = JSONField(_('extra information'), default='{}')
 299     gazeta_link   = models.CharField(blank=True, max_length=240)
 300     wiki_link     = models.CharField(blank=True, max_length=240)
 301     # files generated during publication
 302
 303     file_types = ['epub', 'html', 'mobi', 'pdf', 'txt', 'xml']
 304
 305     parent        = models.ForeignKey('self', blank=True, null=True, related_name='children')
 306     objects  = models.Manager()
 307     tagged   = managers.ModelTaggedItemManager(Tag)
 308     tags     = managers.TagDescriptor(Tag)
 309
 310     html_built = django.dispatch.Signal()
 311
 312     class AlreadyExists(Exception):
 313         pass
 314
 315     class Meta:
 316         ordering = ('sort_key',)
 317         verbose_name = _('book')
 318         verbose_name_plural = _('books')
 319
 320     def __unicode__(self):
 321         return self.title
 322
 323     def save(self, force_insert=False, force_update=False, reset_short_html=True, **kwargs):
 324         self.sort_key = sortify(self.title)
 325
 326         ret = super(Book, self).save(force_insert, force_update)
 327
 328         if reset_short_html:
 329             self.reset_short_html()
 330
 331         return ret
 332
 333     @permalink
 334     def get_absolute_url(self):
 335         return ('catalogue.views.book_detail', [self.slug])
 336
 337     @property
 338     def name(self):
 339         return self.title
 340
 341     def book_tag_slug(self):
 342         return ('l-' + self.slug)[:120]
 343
 344     def book_tag(self):
 345         slug = self.book_tag_slug()
 346         book_tag, created = Tag.objects.get_or_create(slug=slug, category='book')
 347         if created:
 348             book_tag.name = self.title[:50]
 349             book_tag.sort_key = self.title.lower()
 350             book_tag.save()
 351         return book_tag
 352
 353     def has_media(self, type):
 354         if type in Book.file_types:
 355             return bool(getattr(self, "%s_file" % type))
 356         else:
 357             return self.media.filter(type=type).exists()
 358
 359     def get_media(self, type):
 360         if self.has_media(type):
 361             if type in Book.file_types:
 362                 return getattr(self, "%s_file" % type)
 363             else:
 364                 return self.media.filter(type=type)
 365         else:
 366             return None
 367
 368     def get_mp3(self):
 369         return self.get_media("mp3")
 370     def get_odt(self):
 371         return self.get_media("odt")
 372     def get_ogg(self):
 373         return self.get_media("ogg")
 374     def get_daisy(self):
 375         return self.get_media("daisy")
 376
 377     def reset_short_html(self):
 378         if self.id is None:
 379             return
 380
 381         cache_key = "Book.short_html/%d/%s"
 382         for lang, langname in settings.LANGUAGES:
 383             cache.delete(cache_key % (self.id, lang))
 384         # Fragment.short_html relies on book's tags, so reset it here too
 385         for fragm in self.fragments.all():
 386             fragm.reset_short_html()
 387
 388     def short_html(self):
 389         if self.id:
 390             cache_key = "Book.short_html/%d/%s" % (self.id, get_language())
 391             short_html = cache.get(cache_key)
 392         else:
 393             short_html = None
 394
 395         if short_html is not None:
 396             return mark_safe(short_html)
 397         else:
 398             tags = self.tags.filter(~Q(category__in=('set', 'theme', 'book')))
 399             tags = [mark_safe(u'<a href="%s">%s</a>' % (tag.get_absolute_url(), tag.name)) for tag in tags]
 400
 401             formats = []
 402             # files generated during publication
 403             if self.has_media("html"):
 404                 formats.append(u'<a href="%s">%s</a>' % (reverse('book_text', kwargs={'slug': self.slug}), _('Read online')))
 405             if self.has_media("pdf"):
 406                 formats.append(u'<a href="%s">PDF</a>' % self.get_media('pdf').url)
 407             if self.has_media("mobi"):
 408                 formats.append(u'<a href="%s">MOBI</a>' % self.get_media('mobi').url)
 409             if self.root_ancestor.has_media("epub"):
 410                 formats.append(u'<a href="%s">EPUB</a>' % self.root_ancestor.get_media('epub').url)
 411             if self.has_media("txt"):
 412                 formats.append(u'<a href="%s">TXT</a>' % self.get_media('txt').url)
 413             # other files
 414             for m in self.media.order_by('type'):
 415                 formats.append(u'<a href="%s">%s</a>' % (m.file.url, m.type.upper()))
 416
 417             formats = [mark_safe(format) for format in formats]
 418
 419             short_html = unicode(render_to_string('catalogue/book_short.html',
 420                 {'book': self, 'tags': tags, 'formats': formats}))
 421
 422             if self.id:
 423                 cache.set(cache_key, short_html, CACHE_FOREVER)
 424             return mark_safe(short_html)
 425
 426     @property
 427     def root_ancestor(self):
 428         """ returns the oldest ancestor """
 429
 430         if not hasattr(self, '_root_ancestor'):
 431             book = self
 432             while book.parent:
 433                 book = book.parent
 434             self._root_ancestor = book
 435         return self._root_ancestor
 436
 437
 438     def has_description(self):
 439         return len(self.description) > 0
 440     has_description.short_description = _('description')
 441     has_description.boolean = True
 442
 443     # ugly ugly ugly
 444     def has_odt_file(self):
 445         return bool(self.has_media("odt"))
 446     has_odt_file.short_description = 'ODT'
 447     has_odt_file.boolean = True
 448
 449     def has_mp3_file(self):
 450         return bool(self.has_media("mp3"))
 451     has_mp3_file.short_description = 'MP3'
 452     has_mp3_file.boolean = True
 453
 454     def has_ogg_file(self):
 455         return bool(self.has_media("ogg"))
 456     has_ogg_file.short_description = 'OGG'
 457     has_ogg_file.boolean = True
 458
 459     def has_daisy_file(self):
 460         return bool(self.has_media("daisy"))
 461     has_daisy_file.short_description = 'DAISY'
 462     has_daisy_file.boolean = True
 463
 464     def build_pdf(self):
 465         """ (Re)builds the pdf file.
 466
 467         """
 468         from librarian import pdf
 469         from tempfile import NamedTemporaryFile
 470         import os
 471
 472         try:
 473             pdf_file = NamedTemporaryFile(delete=False)
 474             pdf.transform(ORMDocProvider(self),
 475                       file_path=str(self.xml_file.path),
 476                       output_file=pdf_file,
 477                       )
 478
 479             self.pdf_file.save('%s.pdf' % self.slug, File(open(pdf_file.name)))
 480         finally:
 481             unlink(pdf_file.name)
 482
 483         # remove zip with all pdf files
 484         remove_zip(settings.ALL_PDF_ZIP)
 485
 486     def build_mobi(self):
 487         """ (Re)builds the MOBI file.
 488
 489         """
 490         from librarian import mobi
 491         from tempfile import NamedTemporaryFile
 492         import os
 493
 494         try:
 495             mobi_file = NamedTemporaryFile(suffix='.mobi', delete=False)
 496             mobi.transform(ORMDocProvider(self), verbose=1,
 497                       file_path=str(self.xml_file.path),
 498                       output_file=mobi_file.name,
 499                       )
 500
 501             self.mobi_file.save('%s.mobi' % self.slug, File(open(mobi_file.name)))
 502         finally:
 503             unlink(mobi_file.name)
 504
 505         # remove zip with all mobi files
 506         remove_zip(settings.ALL_MOBI_ZIP)
 507
 508     def build_epub(self, remove_descendants=True):
 509         """ (Re)builds the epub file.
 510             If book has a parent, does nothing.
 511             Unless remove_descendants is False, descendants' epubs are removed.
 512         """
 513         from StringIO import StringIO
 514         from hashlib import sha1
 515         from django.core.files.base import ContentFile
 516
 517         if self.parent:
 518             # don't need an epub
 519             return
 520
 521         epub_file = StringIO()
 522         try:
 523             epub.transform(ORMDocProvider(self), self.slug, output_file=epub_file)
 524             self.epub_file.save('%s.epub' % self.slug, ContentFile(epub_file.getvalue()))
 525             FileRecord(slug=self.slug, type='epub', sha1=sha1(epub_file.getvalue()).hexdigest()).save()
 526         except NoDublinCore:
 527             pass
 528
 529         book_descendants = list(self.children.all())
 530         while len(book_descendants) > 0:
 531             child_book = book_descendants.pop(0)
 532             if remove_descendants and child_book.has_epub_file():
 533                 child_book.epub_file.delete()
 534             # save anyway, to refresh short_html
 535             child_book.save()
 536             book_descendants += list(child_book.children.all())
 537
 538         # remove zip package with all epub files
 539         remove_zip(settings.ALL_EPUB_ZIP)
 540
 541     def build_txt(self):
 542         from StringIO import StringIO
 543         from django.core.files.base import ContentFile
 544         from librarian import text
 545
 546         out = StringIO()
 547         text.transform(open(self.xml_file.path), out)
 548         self.txt_file.save('%s.txt' % self.slug, ContentFile(out.getvalue()))
 549
 550
 551     def build_html(self):
 552         from tempfile import NamedTemporaryFile
 553         from markupstring import MarkupString
 554
 555         meta_tags = list(self.tags.filter(
 556             category__in=('author', 'epoch', 'genre', 'kind')))
 557         book_tag = self.book_tag()
 558
 559         html_file = NamedTemporaryFile()
 560         if html.transform(self.xml_file.path, html_file, parse_dublincore=False):
 561             self.html_file.save('%s.html' % self.slug, File(html_file))
 562
 563             # get ancestor l-tags for adding to new fragments
 564             ancestor_tags = []
 565             p = self.parent
 566             while p:
 567                 ancestor_tags.append(p.book_tag())
 568                 p = p.parent
 569
 570             # Delete old fragments and create them from scratch
 571             self.fragments.all().delete()
 572             # Extract fragments
 573             closed_fragments, open_fragments = html.extract_fragments(self.html_file.path)
 574             for fragment in closed_fragments.values():
 575                 try:
 576                     theme_names = [s.strip() for s in fragment.themes.split(',')]
 577                 except AttributeError:
 578                     continue
 579                 themes = []
 580                 for theme_name in theme_names:
 581                     if not theme_name:
 582                         continue
 583                     tag, created = Tag.objects.get_or_create(slug=slughifi(theme_name), category='theme')
 584                     if created:
 585                         tag.name = theme_name
 586                         tag.sort_key = theme_name.lower()
 587                         tag.save()
 588                     themes.append(tag)
 589                 if not themes:
 590                     continue
 591
 592                 text = fragment.to_string()
 593                 short_text = ''
 594                 if (len(MarkupString(text)) > 240):
 595                     short_text = unicode(MarkupString(text)[:160])
 596                 new_fragment = Fragment.objects.create(anchor=fragment.id, book=self,
 597                     text=text, short_text=short_text)
 598
 599                 new_fragment.save()
 600                 new_fragment.tags = set(meta_tags + themes + [book_tag] + ancestor_tags)
 601             self.save()
 602             self.html_built.send(sender=self)
 603             return True
 604         return False
 605
 606     @staticmethod
 607     def zip_format(format_):
 608         def pretty_file_name(book):
 609             return "%s/%s.%s" % (
 610                 b.get_extra_info_value()['author'],
 611                 b.slug,
 612                 format_)
 613
 614         field_name = "%s_file" % format_
 615         books = Book.objects.filter(parent=None).exclude(**{field_name: ""})
 616         paths = [(pretty_file_name(b), getattr(b, field_name).path)
 617                     for b in books]
 618         result = create_zip.delay(paths,
 619                     getattr(settings, "ALL_%s_ZIP" % format_.upper()))
 620         return result.wait()
 621
 622     def zip_audiobooks(self):
 623         bm = BookMedia.objects.filter(book=self, type='mp3')
 624         paths = map(lambda bm: (None, bm.file.path), bm)
 625         result = create_zip.delay(paths, self.slug)
 626         return result.wait()
 627
 628     def search_index(self):
 629         with search.ReusableIndex() as idx:
 630             idx.index_book(self)
 631
 632     @classmethod
 633     def from_xml_file(cls, xml_file, **kwargs):
 634         # use librarian to parse meta-data
 635         book_info = dcparser.parse(xml_file)
 636
 637         if not isinstance(xml_file, File):
 638             xml_file = File(open(xml_file))
 639
 640         try:
 641             return cls.from_text_and_meta(xml_file, book_info, **kwargs)
 642         finally:
 643             xml_file.close()
 644
 645     @classmethod
 646     def from_text_and_meta(cls, raw_file, book_info, overwrite=False,
 647             build_epub=True, build_txt=True, build_pdf=True, build_mobi=True,
 648             search_index=True):
 649         import re
 650
 651         # check for parts before we do anything
 652         children = []
 653         if hasattr(book_info, 'parts'):
 654             for part_url in book_info.parts:
 655                 base, slug = part_url.rsplit('/', 1)
 656                 try:
 657                     children.append(Book.objects.get(slug=slug))
 658                 except Book.DoesNotExist, e:
 659                     raise Book.DoesNotExist(_('Book with slug = "%s" does not exist.') % slug)
 660
 661
 662         # Read book metadata
 663         book_base, book_slug = book_info.url.rsplit('/', 1)
 664         if re.search(r'[^a-zA-Z0-9-]', book_slug):
 665             raise ValueError('Invalid characters in slug')
 666         book, created = Book.objects.get_or_create(slug=book_slug)
 667
 668         if created:
 669             book_shelves = []
 670         else:
 671             if not overwrite:
 672                 raise Book.AlreadyExists(_('Book %s already exists') % book_slug)
 673             # Save shelves for this book
 674             book_shelves = list(book.tags.filter(category='set'))
 675
 676         book.title = book_info.title
 677         book.set_extra_info_value(book_info.to_dict())
 678         book.save()
 679
 680         meta_tags = []
 681         categories = (('kinds', 'kind'), ('genres', 'genre'), ('authors', 'author'), ('epochs', 'epoch'))
 682         for field_name, category in categories:
 683             try:
 684                 tag_names = getattr(book_info, field_name)
 685             except:
 686                 tag_names = [getattr(book_info, category)]
 687             for tag_name in tag_names:
 688                 tag_sort_key = tag_name
 689                 if category == 'author':
 690                     tag_sort_key = tag_name.last_name
 691                     tag_name = ' '.join(tag_name.first_names) + ' ' + tag_name.last_name
 692                 tag, created = Tag.objects.get_or_create(slug=slughifi(tag_name), category=category)
 693                 if created:
 694                     tag.name = tag_name
 695                     tag.sort_key = sortify(tag_sort_key.lower())
 696                     tag.save()
 697                 meta_tags.append(tag)
 698
 699         book.tags = set(meta_tags + book_shelves)
 700
 701         book_tag = book.book_tag()
 702
 703         for n, child_book in enumerate(children):
 704             child_book.parent = book
 705             child_book.parent_number = n
 706             child_book.save()
 707
 708         # Save XML and HTML files
 709         book.xml_file.save('%s.xml' % book.slug, raw_file, save=False)
 710
 711         # delete old fragments when overwriting
 712         book.fragments.all().delete()
 713
 714         if book.build_html():
 715             if not settings.NO_BUILD_TXT and build_txt:
 716                 book.build_txt()
 717
 718         if not settings.NO_BUILD_EPUB and build_epub:
 719             book.root_ancestor.build_epub()
 720
 721         if not settings.NO_BUILD_PDF and build_pdf:
 722             book.root_ancestor.build_pdf()
 723
 724         if not settings.NO_BUILD_MOBI and build_mobi:
 725             book.build_mobi()
 726
 727         if not settings.NO_SEARCH_INDEX and search_index:
 728             book.search_index()
 729
 730         book_descendants = list(book.children.all())
 731         # add l-tag to descendants and their fragments
 732         # delete unnecessary EPUB files
 733         while len(book_descendants) > 0:
 734             child_book = book_descendants.pop(0)
 735             child_book.tags = list(child_book.tags) + [book_tag]
 736             child_book.save()
 737             for fragment in child_book.fragments.all():
 738                 fragment.tags = set(list(fragment.tags) + [book_tag])
 739             book_descendants += list(child_book.children.all())
 740
 741         book.save()
 742
 743         # refresh cache
 744         book.reset_tag_counter()
 745         book.reset_theme_counter()
 746
 747         return book
 748
 749     def reset_tag_counter(self):
 750         if self.id is None:
 751             return
 752
 753         cache_key = "Book.tag_counter/%d" % self.id
 754         cache.delete(cache_key)
 755         if self.parent:
 756             self.parent.reset_tag_counter()
 757
 758     @property
 759     def tag_counter(self):
 760         if self.id:
 761             cache_key = "Book.tag_counter/%d" % self.id
 762             tags = cache.get(cache_key)
 763         else:
 764             tags = None
 765
 766         if tags is None:
 767             tags = {}
 768             for child in self.children.all().order_by():
 769                 for tag_pk, value in child.tag_counter.iteritems():
 770                     tags[tag_pk] = tags.get(tag_pk, 0) + value
 771             for tag in self.tags.exclude(category__in=('book', 'theme', 'set')).order_by():
 772                 tags[tag.pk] = 1
 773
 774             if self.id:
 775                 cache.set(cache_key, tags, CACHE_FOREVER)
 776         return tags
 777
 778     def reset_theme_counter(self):
 779         if self.id is None:
 780             return
 781
 782         cache_key = "Book.theme_counter/%d" % self.id
 783         cache.delete(cache_key)
 784         if self.parent:
 785             self.parent.reset_theme_counter()
 786
 787     @property
 788     def theme_counter(self):
 789         if self.id:
 790             cache_key = "Book.theme_counter/%d" % self.id
 791             tags = cache.get(cache_key)
 792         else:
 793             tags = None
 794
 795         if tags is None:
 796             tags = {}
 797             for fragment in Fragment.tagged.with_any([self.book_tag()]).order_by():
 798                 for tag in fragment.tags.filter(category='theme').order_by():
 799                     tags[tag.pk] = tags.get(tag.pk, 0) + 1
 800
 801             if self.id:
 802                 cache.set(cache_key, tags, CACHE_FOREVER)
 803         return tags
 804
 805     def pretty_title(self, html_links=False):
 806         book = self
 807         names = list(book.tags.filter(category='author'))
 808
 809         books = []
 810         while book:
 811             books.append(book)
 812             book = book.parent
 813         names.extend(reversed(books))
 814
 815         if html_links:
 816             names = ['<a href="%s">%s</a>' % (tag.get_absolute_url(), tag.name) for tag in names]
 817         else:
 818             names = [tag.name for tag in names]
 819
 820         return ', '.join(names)
 821
 822     @classmethod
 823     def tagged_top_level(cls, tags):
 824         """ Returns top-level books tagged with `tags'.
 825
 826         It only returns those books which don't have ancestors which are
 827         also tagged with those tags.
 828
 829         """
 830         # get relevant books and their tags
 831         objects = cls.tagged.with_all(tags)
 832         # eliminate descendants
 833         l_tags = Tag.objects.filter(category='book', slug__in=[book.book_tag_slug() for book in objects])
 834         descendants_keys = [book.pk for book in cls.tagged.with_any(l_tags)]
 835         if descendants_keys:
 836             objects = objects.exclude(pk__in=descendants_keys)
 837
 838         return objects
 839
 840
 841 def _has_factory(ftype):
 842     has = lambda self: bool(getattr(self, "%s_file" % ftype))
 843     has.short_description = t.upper()
 844     has.boolean = True
 845     has.__name__ = "has_%s_file" % ftype
 846     return has
 847
 848
 849 # add the file fields
 850 for t in Book.file_types:
 851     field_name = "%s_file" % t
 852     models.FileField(_("%s file" % t.upper()),
 853             upload_to=book_upload_path(t),
 854             blank=True).contribute_to_class(Book, field_name)
 855
 856     setattr(Book, "has_%s_file" % t, _has_factory(t))
 857
 858
 859 class Fragment(models.Model):
 860     text = models.TextField()
 861     short_text = models.TextField(editable=False)
 862     anchor = models.CharField(max_length=120)
 863     book = models.ForeignKey(Book, related_name='fragments')
 864
 865     objects = models.Manager()
 866     tagged = managers.ModelTaggedItemManager(Tag)
 867     tags = managers.TagDescriptor(Tag)
 868
 869     class Meta:
 870         ordering = ('book', 'anchor',)
 871         verbose_name = _('fragment')
 872         verbose_name_plural = _('fragments')
 873
 874     def get_absolute_url(self):
 875         return '%s#m%s' % (reverse('book_text', kwargs={'slug': self.book.slug}), self.anchor)
 876
 877     def reset_short_html(self):
 878         if self.id is None:
 879             return
 880
 881         cache_key = "Fragment.short_html/%d/%s"
 882         for lang, langname in settings.LANGUAGES:
 883             cache.delete(cache_key % (self.id, lang))
 884
 885     def short_html(self):
 886         if self.id:
 887             cache_key = "Fragment.short_html/%d/%s" % (self.id, get_language())
 888             short_html = cache.get(cache_key)
 889         else:
 890             short_html = None
 891
 892         if short_html is not None:
 893             return mark_safe(short_html)
 894         else:
 895             short_html = unicode(render_to_string('catalogue/fragment_short.html',
 896                 {'fragment': self}))
 897             if self.id:
 898                 cache.set(cache_key, short_html, CACHE_FOREVER)
 899             return mark_safe(short_html)
 900
 901
 902 class FileRecord(models.Model):
 903     slug = models.SlugField(_('slug'), max_length=120, db_index=True)
 904     type = models.CharField(_('type'), max_length=20, db_index=True)
 905     sha1 = models.CharField(_('sha-1 hash'), max_length=40)
 906     time = models.DateTimeField(_('time'), auto_now_add=True)
 907
 908     class Meta:
 909         ordering = ('-time','-slug', '-type')
 910         verbose_name = _('file record')
 911         verbose_name_plural = _('file records')
 912
 913     def __unicode__(self):
 914         return "%s %s.%s" % (self.sha1,  self.slug, self.type)
 915
 916 ###########
 917 #
 918 # SIGNALS
 919 #
 920 ###########
 921
 922
 923 def _tags_updated_handler(sender, affected_tags, **kwargs):
 924     # reset tag global counter
 925     # we want Tag.changed_at updated for API to know the tag was touched
 926     Tag.objects.filter(pk__in=[tag.pk for tag in affected_tags]).update(book_count=None, changed_at=datetime.now())
 927
 928     # if book tags changed, reset book tag counter
 929     if isinstance(sender, Book) and \
 930                 Tag.objects.filter(pk__in=(tag.pk for tag in affected_tags)).\
 931                     exclude(category__in=('book', 'theme', 'set')).count():
 932         sender.reset_tag_counter()
 933     # if fragment theme changed, reset book theme counter
 934     elif isinstance(sender, Fragment) and \
 935                 Tag.objects.filter(pk__in=(tag.pk for tag in affected_tags)).\
 936                     filter(category='theme').count():
 937         sender.book.reset_theme_counter()
 938 tags_updated.connect(_tags_updated_handler)
 939
 940
 941 def _pre_delete_handler(sender, instance, **kwargs):
 942     """ refresh Book on BookMedia delete """
 943     if sender == BookMedia:
 944         instance.book.save()
 945 pre_delete.connect(_pre_delete_handler)
 946
 947 def _post_save_handler(sender, instance, **kwargs):
 948     """ refresh all the short_html stuff on BookMedia update """
 949     if sender == BookMedia:
 950         instance.book.save()
 951 post_save.connect(_post_save_handler)