apps/catalogue/models.py

   1 # -*- coding: utf-8 -*-
   2 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
   3 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
   4 #
   5 from datetime import datetime
   6
   7 from django.db import models
   8 from django.db.models import permalink, Q
   9 import django.dispatch
  10 from django.core.cache import cache
  11 from django.utils.translation import ugettext_lazy as _
  12 from django.contrib.auth.models import User
  13 from django.core.files import File
  14 from django.template.loader import render_to_string
  15 from django.utils.safestring import mark_safe
  16 from django.utils.translation import get_language
  17 from django.core.urlresolvers import reverse
  18 from django.db.models.signals import post_save, m2m_changed, pre_delete
  19
  20 from django.conf import settings
  21
  22 from newtagging.models import TagBase, tags_updated
  23 from newtagging import managers
  24 from catalogue.fields import JSONField, OverwritingFileField
  25 from catalogue.utils import ExistingFile, ORMDocProvider, create_zip, remove_zip
  26
  27 from librarian import dcparser, html, epub, NoDublinCore
  28 import mutagen
  29 from mutagen import id3
  30 from slughifi import slughifi
  31 from sortify import sortify
  32 from os import unlink
  33
  34 import search
  35
  36 TAG_CATEGORIES = (
  37     ('author', _('author')),
  38     ('epoch', _('epoch')),
  39     ('kind', _('kind')),
  40     ('genre', _('genre')),
  41     ('theme', _('theme')),
  42     ('set', _('set')),
  43     ('book', _('book')),
  44 )
  45
  46 MEDIA_FORMATS = (
  47     ('odt', _('ODT file')),
  48     ('mp3', _('MP3 file')),
  49     ('ogg', _('OGG file')),
  50     ('daisy', _('DAISY file')),
  51 )
  52
  53 # not quite, but Django wants you to set a timeout
  54 CACHE_FOREVER = 2419200  # 28 days
  55
  56
  57 class TagSubcategoryManager(models.Manager):
  58     def __init__(self, subcategory):
  59         super(TagSubcategoryManager, self).__init__()
  60         self.subcategory = subcategory
  61
  62     def get_query_set(self):
  63         return super(TagSubcategoryManager, self).get_query_set().filter(category=self.subcategory)
  64
  65
  66 class Tag(TagBase):
  67     name = models.CharField(_('name'), max_length=50, db_index=True)
  68     slug = models.SlugField(_('slug'), max_length=120, db_index=True)
  69     sort_key = models.CharField(_('sort key'), max_length=120, db_index=True)
  70     category = models.CharField(_('category'), max_length=50, blank=False, null=False,
  71         db_index=True, choices=TAG_CATEGORIES)
  72     description = models.TextField(_('description'), blank=True)
  73     main_page = models.BooleanField(_('main page'), default=False, db_index=True, help_text=_('Show tag on main page'))
  74
  75     user = models.ForeignKey(User, blank=True, null=True)
  76     book_count = models.IntegerField(_('book count'), blank=True, null=True)
  77     gazeta_link = models.CharField(blank=True, max_length=240)
  78     wiki_link = models.CharField(blank=True, max_length=240)
  79
  80     created_at    = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True)
  81     changed_at    = models.DateTimeField(_('creation date'), auto_now=True, db_index=True)
  82
  83     class UrlDeprecationWarning(DeprecationWarning):
  84         pass
  85
  86     categories_rev = {
  87         'autor': 'author',
  88         'epoka': 'epoch',
  89         'rodzaj': 'kind',
  90         'gatunek': 'genre',
  91         'motyw': 'theme',
  92         'polka': 'set',
  93     }
  94     categories_dict = dict((item[::-1] for item in categories_rev.iteritems()))
  95
  96     class Meta:
  97         ordering = ('sort_key',)
  98         verbose_name = _('tag')
  99         verbose_name_plural = _('tags')
 100         unique_together = (("slug", "category"),)
 101
 102     def __unicode__(self):
 103         return self.name
 104
 105     def __repr__(self):
 106         return "Tag(slug=%r)" % self.slug
 107
 108     @permalink
 109     def get_absolute_url(self):
 110         return ('catalogue.views.tagged_object_list', [self.url_chunk])
 111
 112     def has_description(self):
 113         return len(self.description) > 0
 114     has_description.short_description = _('description')
 115     has_description.boolean = True
 116
 117     def get_count(self):
 118         """ returns global book count for book tags, fragment count for themes """
 119
 120         if self.book_count is None:
 121             if self.category == 'book':
 122                 # never used
 123                 objects = Book.objects.none()
 124             elif self.category == 'theme':
 125                 objects = Fragment.tagged.with_all((self,))
 126             else:
 127                 objects = Book.tagged.with_all((self,)).order_by()
 128                 if self.category != 'set':
 129                     # eliminate descendants
 130                     l_tags = Tag.objects.filter(slug__in=[book.book_tag_slug() for book in objects])
 131                     descendants_keys = [book.pk for book in Book.tagged.with_any(l_tags)]
 132                     if descendants_keys:
 133                         objects = objects.exclude(pk__in=descendants_keys)
 134             self.book_count = objects.count()
 135             self.save()
 136         return self.book_count
 137
 138     @staticmethod
 139     def get_tag_list(tags):
 140         if isinstance(tags, basestring):
 141             real_tags = []
 142             ambiguous_slugs = []
 143             category = None
 144             deprecated = False
 145             tags_splitted = tags.split('/')
 146             for name in tags_splitted:
 147                 if category:
 148                     real_tags.append(Tag.objects.get(slug=name, category=category))
 149                     category = None
 150                 elif name in Tag.categories_rev:
 151                     category = Tag.categories_rev[name]
 152                 else:
 153                     try:
 154                         real_tags.append(Tag.objects.exclude(category='book').get(slug=name))
 155                         deprecated = True
 156                     except Tag.MultipleObjectsReturned, e:
 157                         ambiguous_slugs.append(name)
 158
 159             if category:
 160                 # something strange left off
 161                 raise Tag.DoesNotExist()
 162             if ambiguous_slugs:
 163                 # some tags should be qualified
 164                 e = Tag.MultipleObjectsReturned()
 165                 e.tags = real_tags
 166                 e.ambiguous_slugs = ambiguous_slugs
 167                 raise e
 168             if deprecated:
 169                 e = Tag.UrlDeprecationWarning()
 170                 e.tags = real_tags
 171                 raise e
 172             return real_tags
 173         else:
 174             return TagBase.get_tag_list(tags)
 175
 176     @property
 177     def url_chunk(self):
 178         return '/'.join((Tag.categories_dict[self.category], self.slug))
 179
 180
 181 # TODO: why is this hard-coded ?
 182 def book_upload_path(ext=None, maxlen=100):
 183     def get_dynamic_path(media, filename, ext=ext):
 184         # how to put related book's slug here?
 185         if not ext:
 186             if media.type == 'daisy':
 187                 ext = 'daisy.zip'
 188             else:
 189                 ext = media.type
 190         if not media.name:
 191             name = slughifi(filename.split(".")[0])
 192         else:
 193             name = slughifi(media.name)
 194         return 'book/%s/%s.%s' % (ext, name[:maxlen-len('book/%s/.%s' % (ext, ext))-4], ext)
 195     return get_dynamic_path
 196
 197
 198 class BookMedia(models.Model):
 199     type        = models.CharField(_('type'), choices=MEDIA_FORMATS, max_length="100")
 200     name        = models.CharField(_('name'), max_length="100")
 201     file        = OverwritingFileField(_('file'), upload_to=book_upload_path())
 202     uploaded_at = models.DateTimeField(_('creation date'), auto_now_add=True, editable=False)
 203     extra_info  = JSONField(_('extra information'), default='{}', editable=False)
 204     book = models.ForeignKey('Book', related_name='media')
 205     source_sha1 = models.CharField(null=True, blank=True, max_length=40, editable=False)
 206
 207     def __unicode__(self):
 208         return "%s (%s)" % (self.name, self.file.name.split("/")[-1])
 209
 210     class Meta:
 211         ordering            = ('type', 'name')
 212         verbose_name        = _('book media')
 213         verbose_name_plural = _('book media')
 214
 215     def save(self, *args, **kwargs):
 216         try:
 217             old = BookMedia.objects.get(pk=self.pk)
 218         except BookMedia.DoesNotExist, e:
 219             pass
 220         else:
 221             # if name changed, change the file name, too
 222             if slughifi(self.name) != slughifi(old.name):
 223                 self.file.save(None, ExistingFile(self.file.path), save=False, leave=True)
 224
 225         super(BookMedia, self).save(*args, **kwargs)
 226
 227         # remove the zip package for book with modified media
 228         remove_zip(self.book.slug)
 229
 230         extra_info = self.get_extra_info_value()
 231         extra_info.update(self.read_meta())
 232         self.set_extra_info_value(extra_info)
 233         self.source_sha1 = self.read_source_sha1(self.file.path, self.type)
 234         return super(BookMedia, self).save(*args, **kwargs)
 235
 236     def read_meta(self):
 237         """
 238             Reads some metadata from the audiobook.
 239         """
 240
 241         artist_name = director_name = project = funded_by = ''
 242         if self.type == 'mp3':
 243             try:
 244                 audio = id3.ID3(self.file.path)
 245                 artist_name = ', '.join(', '.join(tag.text) for tag in audio.getall('TPE1'))
 246                 director_name = ', '.join(', '.join(tag.text) for tag in audio.getall('TPE3'))
 247                 project = ", ".join([t.data for t in audio.getall('PRIV')
 248                         if t.owner=='wolnelektury.pl?project'])
 249                 funded_by = ", ".join([t.data for t in audio.getall('PRIV')
 250                         if t.owner=='wolnelektury.pl?funded_by'])
 251             except:
 252                 pass
 253         elif self.type == 'ogg':
 254             try:
 255                 audio = mutagen.File(self.file.path)
 256                 artist_name = ', '.join(audio.get('artist', []))
 257                 director_name = ', '.join(audio.get('conductor', []))
 258                 project = ", ".join(audio.get('project', []))
 259                 funded_by = ", ".join(audio.get('funded_by', []))
 260             except:
 261                 pass
 262         else:
 263             return {}
 264         return {'artist_name': artist_name, 'director_name': director_name,
 265                 'project': project, 'funded_by': funded_by}
 266
 267     @staticmethod
 268     def read_source_sha1(filepath, filetype):
 269         """
 270             Reads source file SHA1 from audiobok metadata.
 271         """
 272
 273         if filetype == 'mp3':
 274             try:
 275                 audio = id3.ID3(filepath)
 276                 return [t.data for t in audio.getall('PRIV')
 277                         if t.owner=='wolnelektury.pl?flac_sha1'][0]
 278             except:
 279                 return None
 280         elif filetype == 'ogg':
 281             try:
 282                 audio = mutagen.File(filepath)
 283                 return audio.get('flac_sha1', [None])[0]
 284             except:
 285                 return None
 286         else:
 287             return None
 288
 289
 290 class Book(models.Model):
 291     title         = models.CharField(_('title'), max_length=120)
 292     sort_key = models.CharField(_('sort key'), max_length=120, db_index=True, editable=False)
 293     slug          = models.SlugField(_('slug'), max_length=120, unique=True, db_index=True)
 294     description   = models.TextField(_('description'), blank=True)
 295     created_at    = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True)
 296     changed_at    = models.DateTimeField(_('creation date'), auto_now=True, db_index=True)
 297     parent_number = models.IntegerField(_('parent number'), default=0)
 298     extra_info    = JSONField(_('extra information'), default='{}')
 299     gazeta_link   = models.CharField(blank=True, max_length=240)
 300     wiki_link     = models.CharField(blank=True, max_length=240)
 301     # files generated during publication
 302
 303     file_types = ['epub', 'html', 'mobi', 'pdf', 'txt', 'xml']
 304
 305     parent        = models.ForeignKey('self', blank=True, null=True, related_name='children')
 306     objects  = models.Manager()
 307     tagged   = managers.ModelTaggedItemManager(Tag)
 308     tags     = managers.TagDescriptor(Tag)
 309
 310     html_built = django.dispatch.Signal()
 311
 312     class AlreadyExists(Exception):
 313         pass
 314
 315     class Meta:
 316         ordering = ('sort_key',)
 317         verbose_name = _('book')
 318         verbose_name_plural = _('books')
 319
 320     def __unicode__(self):
 321         return self.title
 322
 323     def save(self, force_insert=False, force_update=False, reset_short_html=True, **kwargs):
 324         self.sort_key = sortify(self.title)
 325
 326         ret = super(Book, self).save(force_insert, force_update)
 327
 328         if reset_short_html:
 329             self.reset_short_html()
 330
 331         return ret
 332
 333     @permalink
 334     def get_absolute_url(self):
 335         return ('catalogue.views.book_detail', [self.slug])
 336
 337     @property
 338     def name(self):
 339         return self.title
 340
 341     def book_tag_slug(self):
 342         return ('l-' + self.slug)[:120]
 343
 344     def book_tag(self):
 345         slug = self.book_tag_slug()
 346         book_tag, created = Tag.objects.get_or_create(slug=slug, category='book')
 347         if created:
 348             book_tag.name = self.title[:50]
 349             book_tag.sort_key = self.title.lower()
 350             book_tag.save()
 351         return book_tag
 352
 353     def has_media(self, type):
 354         if type in Book.file_types:
 355             return bool(getattr(self, "%s_file" % type))
 356         else:
 357             return self.media.filter(type=type).exists()
 358
 359     def get_media(self, type):
 360         if self.has_media(type):
 361             if type in Book.file_types:
 362                 return getattr(self, "%s_file" % type)
 363             else:
 364                 return self.media.filter(type=type)
 365         else:
 366             return None
 367
 368     def get_mp3(self):
 369         return self.get_media("mp3")
 370     def get_odt(self):
 371         return self.get_media("odt")
 372     def get_ogg(self):
 373         return self.get_media("ogg")
 374     def get_daisy(self):
 375         return self.get_media("daisy")
 376
 377     def reset_short_html(self):
 378         if self.id is None:
 379             return
 380
 381         cache_key = "Book.short_html/%d/%s"
 382         for lang, langname in settings.LANGUAGES:
 383             cache.delete(cache_key % (self.id, lang))
 384         # Fragment.short_html relies on book's tags, so reset it here too
 385         for fragm in self.fragments.all():
 386             fragm.reset_short_html()
 387
 388     def short_html(self):
 389         if self.id:
 390             cache_key = "Book.short_html/%d/%s" % (self.id, get_language())
 391             short_html = cache.get(cache_key)
 392         else:
 393             short_html = None
 394
 395         if short_html is not None:
 396             return mark_safe(short_html)
 397         else:
 398             tags = self.tags.filter(~Q(category__in=('set', 'theme', 'book')))
 399             tags = [mark_safe(u'<a href="%s">%s</a>' % (tag.get_absolute_url(), tag.name)) for tag in tags]
 400
 401             formats = []
 402             # files generated during publication
 403             if self.has_media("html"):
 404                 formats.append(u'<a href="%s">%s</a>' % (reverse('book_text', kwargs={'slug': self.slug}), _('Read online')))
 405             if self.has_media("pdf"):
 406                 formats.append(u'<a href="%s">PDF</a>' % self.get_media('pdf').url)
 407             if self.has_media("mobi"):
 408                 formats.append(u'<a href="%s">MOBI</a>' % self.get_media('mobi').url)
 409             if self.root_ancestor.has_media("epub"):
 410                 formats.append(u'<a href="%s">EPUB</a>' % self.root_ancestor.get_media('epub').url)
 411             if self.has_media("txt"):
 412                 formats.append(u'<a href="%s">TXT</a>' % self.get_media('txt').url)
 413             # other files
 414             for m in self.media.order_by('type'):
 415                 formats.append(u'<a href="%s">%s</a>' % (m.file.url, m.type.upper()))
 416
 417             formats = [mark_safe(format) for format in formats]
 418
 419             short_html = unicode(render_to_string('catalogue/book_short.html',
 420                 {'book': self, 'tags': tags, 'formats': formats}))
 421
 422             if self.id:
 423                 cache.set(cache_key, short_html, CACHE_FOREVER)
 424             return mark_safe(short_html)
 425
 426     @property
 427     def root_ancestor(self):
 428         """ returns the oldest ancestor """
 429
 430         if not hasattr(self, '_root_ancestor'):
 431             book = self
 432             while book.parent:
 433                 book = book.parent
 434             self._root_ancestor = book
 435         return self._root_ancestor
 436
 437
 438     def has_description(self):
 439         return len(self.description) > 0
 440     has_description.short_description = _('description')
 441     has_description.boolean = True
 442
 443     # ugly ugly ugly
 444     def has_odt_file(self):
 445         return bool(self.has_media("odt"))
 446     has_odt_file.short_description = 'ODT'
 447     has_odt_file.boolean = True
 448
 449     def has_mp3_file(self):
 450         return bool(self.has_media("mp3"))
 451     has_mp3_file.short_description = 'MP3'
 452     has_mp3_file.boolean = True
 453
 454     def has_ogg_file(self):
 455         return bool(self.has_media("ogg"))
 456     has_ogg_file.short_description = 'OGG'
 457     has_ogg_file.boolean = True
 458
 459     def has_daisy_file(self):
 460         return bool(self.has_media("daisy"))
 461     has_daisy_file.short_description = 'DAISY'
 462     has_daisy_file.boolean = True
 463
 464     def build_pdf(self):
 465         """ (Re)builds the pdf file.
 466
 467         """
 468         from librarian import pdf
 469         from tempfile import NamedTemporaryFile
 470         import os
 471
 472         try:
 473             pdf_file = NamedTemporaryFile(delete=False)
 474             pdf.transform(ORMDocProvider(self),
 475                       file_path=str(self.xml_file.path),
 476                       output_file=pdf_file,
 477                       )
 478
 479             self.pdf_file.save('%s.pdf' % self.slug, File(open(pdf_file.name)))
 480         finally:
 481             unlink(pdf_file.name)
 482
 483         # remove zip with all pdf files
 484         remove_zip(settings.ALL_PDF_ZIP)
 485
 486     def build_mobi(self):
 487         """ (Re)builds the MOBI file.
 488
 489         """
 490         from librarian import mobi
 491         from tempfile import NamedTemporaryFile
 492         import os
 493
 494         try:
 495             mobi_file = NamedTemporaryFile(suffix='.mobi', delete=False)
 496             mobi.transform(ORMDocProvider(self), verbose=1,
 497                       file_path=str(self.xml_file.path),
 498                       output_file=mobi_file.name,
 499                       )
 500
 501             self.mobi_file.save('%s.mobi' % self.slug, File(open(mobi_file.name)))
 502         finally:
 503             unlink(mobi_file.name)
 504
 505         # remove zip with all mobi files
 506         remove_zip(settings.ALL_MOBI_ZIP)
 507
 508     def build_epub(self, remove_descendants=True):
 509         """ (Re)builds the epub file.
 510             If book has a parent, does nothing.
 511             Unless remove_descendants is False, descendants' epubs are removed.
 512         """
 513         from StringIO import StringIO
 514         from hashlib import sha1
 515         from django.core.files.base import ContentFile
 516
 517         if self.parent:
 518             # don't need an epub
 519             return
 520
 521         epub_file = StringIO()
 522         try:
 523             epub.transform(ORMDocProvider(self), self.slug, output_file=epub_file)
 524             self.epub_file.save('%s.epub' % self.slug, ContentFile(epub_file.getvalue()))
 525             FileRecord(slug=self.slug, type='epub', sha1=sha1(epub_file.getvalue()).hexdigest()).save()
 526         except NoDublinCore:
 527             pass
 528
 529         book_descendants = list(self.children.all())
 530         while len(book_descendants) > 0:
 531             child_book = book_descendants.pop(0)
 532             if remove_descendants and child_book.has_epub_file():
 533                 child_book.epub_file.delete()
 534             # save anyway, to refresh short_html
 535             child_book.save()
 536             book_descendants += list(child_book.children.all())
 537
 538         # remove zip package with all epub files
 539         remove_zip(settings.ALL_EPUB_ZIP)
 540
 541     def build_txt(self):
 542         from StringIO import StringIO
 543         from django.core.files.base import ContentFile
 544         from librarian import text
 545
 546         out = StringIO()
 547         text.transform(open(self.xml_file.path), out)
 548         self.txt_file.save('%s.txt' % self.slug, ContentFile(out.getvalue()))
 549
 550
 551     def build_html(self):
 552         from tempfile import NamedTemporaryFile
 553         from markupstring import MarkupString
 554
 555         meta_tags = list(self.tags.filter(
 556             category__in=('author', 'epoch', 'genre', 'kind')))
 557         book_tag = self.book_tag()
 558
 559         html_file = NamedTemporaryFile()
 560         if html.transform(self.xml_file.path, html_file, parse_dublincore=False):
 561             self.html_file.save('%s.html' % self.slug, File(html_file))
 562
 563             # get ancestor l-tags for adding to new fragments
 564             ancestor_tags = []
 565             p = self.parent
 566             while p:
 567                 ancestor_tags.append(p.book_tag())
 568                 p = p.parent
 569
 570             # Delete old fragments and create them from scratch
 571             self.fragments.all().delete()
 572             # Extract fragments
 573             closed_fragments, open_fragments = html.extract_fragments(self.html_file.path)
 574             for fragment in closed_fragments.values():
 575                 try:
 576                     theme_names = [s.strip() for s in fragment.themes.split(',')]
 577                 except AttributeError:
 578                     continue
 579                 themes = []
 580                 for theme_name in theme_names:
 581                     if not theme_name:
 582                         continue
 583                     tag, created = Tag.objects.get_or_create(slug=slughifi(theme_name), category='theme')
 584                     if created:
 585                         tag.name = theme_name
 586                         tag.sort_key = theme_name.lower()
 587                         tag.save()
 588                     themes.append(tag)
 589                 if not themes:
 590                     continue
 591
 592                 text = fragment.to_string()
 593                 short_text = ''
 594                 if (len(MarkupString(text)) > 240):
 595                     short_text = unicode(MarkupString(text)[:160])
 596                 new_fragment = Fragment.objects.create(anchor=fragment.id, book=self,
 597                     text=text, short_text=short_text)
 598
 599                 new_fragment.save()
 600                 new_fragment.tags = set(meta_tags + themes + [book_tag] + ancestor_tags)
 601             self.save()
 602             self.html_built.send(sender=self)
 603             return True
 604         return False
 605
 606     @staticmethod
 607     def zip_format(format_):
 608         def pretty_file_name(book):
 609             return "%s/%s.%s" % (
 610                 b.get_extra_info_value()['author'],
 611                 b.slug,
 612                 format_)
 613
 614         field_name = "%s_file" % format_
 615         books = Book.objects.filter(parent=None).exclude(**{field_name: ""})
 616         paths = [(pretty_file_name(b), getattr(b, field_name).path)
 617                     for b in books]
 618         result = create_zip.delay(paths,
 619                     getattr(settings, "ALL_%s_ZIP" % format_.upper()))
 620         return result.wait()
 621
 622     def zip_audiobooks(self):
 623         bm = BookMedia.objects.filter(book=self, type='mp3')
 624         paths = map(lambda bm: (None, bm.file.path), bm)
 625         result = create_zip.delay(paths, self.slug)
 626         return result.wait()
 627
 628     def search_index(self):
 629         if settings.SEARCH_INDEX_PARALLEL:
 630             if instance(settings.SEARCH_INDEX_PARALLEL, int):
 631                 idx = search.ReusableIndex(threads=4)
 632             else:
 633                 idx = search.ReusableIndex()
 634         else:
 635             idx = search.Index()
 636
 637         idx.open()
 638         try:
 639             idx.index_book(self)
 640         finally:
 641             idx.close()
 642
 643     @classmethod
 644     def from_xml_file(cls, xml_file, **kwargs):
 645         # use librarian to parse meta-data
 646         book_info = dcparser.parse(xml_file)
 647
 648         if not isinstance(xml_file, File):
 649             xml_file = File(open(xml_file))
 650
 651         try:
 652             return cls.from_text_and_meta(xml_file, book_info, **kwargs)
 653         finally:
 654             xml_file.close()
 655
 656     @classmethod
 657     def from_text_and_meta(cls, raw_file, book_info, overwrite=False,
 658             build_epub=True, build_txt=True, build_pdf=True, build_mobi=True,
 659             search_index=True):
 660         import re
 661
 662         # check for parts before we do anything
 663         children = []
 664         if hasattr(book_info, 'parts'):
 665             for part_url in book_info.parts:
 666                 base, slug = part_url.rsplit('/', 1)
 667                 try:
 668                     children.append(Book.objects.get(slug=slug))
 669                 except Book.DoesNotExist, e:
 670                     raise Book.DoesNotExist(_('Book with slug = "%s" does not exist.') % slug)
 671
 672
 673         # Read book metadata
 674         book_base, book_slug = book_info.url.rsplit('/', 1)
 675         if re.search(r'[^a-zA-Z0-9-]', book_slug):
 676             raise ValueError('Invalid characters in slug')
 677         book, created = Book.objects.get_or_create(slug=book_slug)
 678
 679         if created:
 680             book_shelves = []
 681         else:
 682             if not overwrite:
 683                 raise Book.AlreadyExists(_('Book %s already exists') % book_slug)
 684             # Save shelves for this book
 685             book_shelves = list(book.tags.filter(category='set'))
 686
 687         book.title = book_info.title
 688         book.set_extra_info_value(book_info.to_dict())
 689         book.save()
 690
 691         meta_tags = []
 692         categories = (('kinds', 'kind'), ('genres', 'genre'), ('authors', 'author'), ('epochs', 'epoch'))
 693         for field_name, category in categories:
 694             try:
 695                 tag_names = getattr(book_info, field_name)
 696             except:
 697                 tag_names = [getattr(book_info, category)]
 698             for tag_name in tag_names:
 699                 tag_sort_key = tag_name
 700                 if category == 'author':
 701                     tag_sort_key = tag_name.last_name
 702                     tag_name = ' '.join(tag_name.first_names) + ' ' + tag_name.last_name
 703                 tag, created = Tag.objects.get_or_create(slug=slughifi(tag_name), category=category)
 704                 if created:
 705                     tag.name = tag_name
 706                     tag.sort_key = sortify(tag_sort_key.lower())
 707                     tag.save()
 708                 meta_tags.append(tag)
 709
 710         book.tags = set(meta_tags + book_shelves)
 711
 712         book_tag = book.book_tag()
 713
 714         for n, child_book in enumerate(children):
 715             child_book.parent = book
 716             child_book.parent_number = n
 717             child_book.save()
 718
 719         # Save XML and HTML files
 720         book.xml_file.save('%s.xml' % book.slug, raw_file, save=False)
 721
 722         # delete old fragments when overwriting
 723         book.fragments.all().delete()
 724
 725         if book.build_html():
 726             if not settings.NO_BUILD_TXT and build_txt:
 727                 book.build_txt()
 728
 729         if not settings.NO_BUILD_EPUB and build_epub:
 730             book.root_ancestor.build_epub()
 731
 732         if not settings.NO_BUILD_PDF and build_pdf:
 733             book.root_ancestor.build_pdf()
 734
 735         if not settings.NO_BUILD_MOBI and build_mobi:
 736             book.build_mobi()
 737
 738         if not settings.NO_SEARCH_INDEX and search_index:
 739             book.search_index()
 740
 741         book_descendants = list(book.children.all())
 742         # add l-tag to descendants and their fragments
 743         # delete unnecessary EPUB files
 744         while len(book_descendants) > 0:
 745             child_book = book_descendants.pop(0)
 746             child_book.tags = list(child_book.tags) + [book_tag]
 747             child_book.save()
 748             for fragment in child_book.fragments.all():
 749                 fragment.tags = set(list(fragment.tags) + [book_tag])
 750             book_descendants += list(child_book.children.all())
 751
 752         book.save()
 753
 754         # refresh cache
 755         book.reset_tag_counter()
 756         book.reset_theme_counter()
 757
 758         return book
 759
 760     def reset_tag_counter(self):
 761         if self.id is None:
 762             return
 763
 764         cache_key = "Book.tag_counter/%d" % self.id
 765         cache.delete(cache_key)
 766         if self.parent:
 767             self.parent.reset_tag_counter()
 768
 769     @property
 770     def tag_counter(self):
 771         if self.id:
 772             cache_key = "Book.tag_counter/%d" % self.id
 773             tags = cache.get(cache_key)
 774         else:
 775             tags = None
 776
 777         if tags is None:
 778             tags = {}
 779             for child in self.children.all().order_by():
 780                 for tag_pk, value in child.tag_counter.iteritems():
 781                     tags[tag_pk] = tags.get(tag_pk, 0) + value
 782             for tag in self.tags.exclude(category__in=('book', 'theme', 'set')).order_by():
 783                 tags[tag.pk] = 1
 784
 785             if self.id:
 786                 cache.set(cache_key, tags, CACHE_FOREVER)
 787         return tags
 788
 789     def reset_theme_counter(self):
 790         if self.id is None:
 791             return
 792
 793         cache_key = "Book.theme_counter/%d" % self.id
 794         cache.delete(cache_key)
 795         if self.parent:
 796             self.parent.reset_theme_counter()
 797
 798     @property
 799     def theme_counter(self):
 800         if self.id:
 801             cache_key = "Book.theme_counter/%d" % self.id
 802             tags = cache.get(cache_key)
 803         else:
 804             tags = None
 805
 806         if tags is None:
 807             tags = {}
 808             for fragment in Fragment.tagged.with_any([self.book_tag()]).order_by():
 809                 for tag in fragment.tags.filter(category='theme').order_by():
 810                     tags[tag.pk] = tags.get(tag.pk, 0) + 1
 811
 812             if self.id:
 813                 cache.set(cache_key, tags, CACHE_FOREVER)
 814         return tags
 815
 816     def pretty_title(self, html_links=False):
 817         book = self
 818         names = list(book.tags.filter(category='author'))
 819
 820         books = []
 821         while book:
 822             books.append(book)
 823             book = book.parent
 824         names.extend(reversed(books))
 825
 826         if html_links:
 827             names = ['<a href="%s">%s</a>' % (tag.get_absolute_url(), tag.name) for tag in names]
 828         else:
 829             names = [tag.name for tag in names]
 830
 831         return ', '.join(names)
 832
 833     @classmethod
 834     def tagged_top_level(cls, tags):
 835         """ Returns top-level books tagged with `tags'.
 836
 837         It only returns those books which don't have ancestors which are
 838         also tagged with those tags.
 839
 840         """
 841         # get relevant books and their tags
 842         objects = cls.tagged.with_all(tags)
 843         # eliminate descendants
 844         l_tags = Tag.objects.filter(category='book', slug__in=[book.book_tag_slug() for book in objects])
 845         descendants_keys = [book.pk for book in cls.tagged.with_any(l_tags)]
 846         if descendants_keys:
 847             objects = objects.exclude(pk__in=descendants_keys)
 848
 849         return objects
 850
 851
 852 def _has_factory(ftype):
 853     has = lambda self: bool(getattr(self, "%s_file" % ftype))
 854     has.short_description = t.upper()
 855     has.boolean = True
 856     has.__name__ = "has_%s_file" % ftype
 857     return has
 858
 859
 860 # add the file fields
 861 for t in Book.file_types:
 862     field_name = "%s_file" % t
 863     models.FileField(_("%s file" % t.upper()),
 864             upload_to=book_upload_path(t),
 865             blank=True).contribute_to_class(Book, field_name)
 866
 867     setattr(Book, "has_%s_file" % t, _has_factory(t))
 868
 869
 870 class Fragment(models.Model):
 871     text = models.TextField()
 872     short_text = models.TextField(editable=False)
 873     anchor = models.CharField(max_length=120)
 874     book = models.ForeignKey(Book, related_name='fragments')
 875
 876     objects = models.Manager()
 877     tagged = managers.ModelTaggedItemManager(Tag)
 878     tags = managers.TagDescriptor(Tag)
 879
 880     class Meta:
 881         ordering = ('book', 'anchor',)
 882         verbose_name = _('fragment')
 883         verbose_name_plural = _('fragments')
 884
 885     def get_absolute_url(self):
 886         return '%s#m%s' % (reverse('book_text', kwargs={'slug': self.book.slug}), self.anchor)
 887
 888     def reset_short_html(self):
 889         if self.id is None:
 890             return
 891
 892         cache_key = "Fragment.short_html/%d/%s"
 893         for lang, langname in settings.LANGUAGES:
 894             cache.delete(cache_key % (self.id, lang))
 895
 896     def short_html(self):
 897         if self.id:
 898             cache_key = "Fragment.short_html/%d/%s" % (self.id, get_language())
 899             short_html = cache.get(cache_key)
 900         else:
 901             short_html = None
 902
 903         if short_html is not None:
 904             return mark_safe(short_html)
 905         else:
 906             short_html = unicode(render_to_string('catalogue/fragment_short.html',
 907                 {'fragment': self}))
 908             if self.id:
 909                 cache.set(cache_key, short_html, CACHE_FOREVER)
 910             return mark_safe(short_html)
 911
 912
 913 class FileRecord(models.Model):
 914     slug = models.SlugField(_('slug'), max_length=120, db_index=True)
 915     type = models.CharField(_('type'), max_length=20, db_index=True)
 916     sha1 = models.CharField(_('sha-1 hash'), max_length=40)
 917     time = models.DateTimeField(_('time'), auto_now_add=True)
 918
 919     class Meta:
 920         ordering = ('-time','-slug', '-type')
 921         verbose_name = _('file record')
 922         verbose_name_plural = _('file records')
 923
 924     def __unicode__(self):
 925         return "%s %s.%s" % (self.sha1,  self.slug, self.type)
 926
 927 ###########
 928 #
 929 # SIGNALS
 930 #
 931 ###########
 932
 933
 934 def _tags_updated_handler(sender, affected_tags, **kwargs):
 935     # reset tag global counter
 936     # we want Tag.changed_at updated for API to know the tag was touched
 937     Tag.objects.filter(pk__in=[tag.pk for tag in affected_tags]).update(book_count=None, changed_at=datetime.now())
 938
 939     # if book tags changed, reset book tag counter
 940     if isinstance(sender, Book) and \
 941                 Tag.objects.filter(pk__in=(tag.pk for tag in affected_tags)).\
 942                     exclude(category__in=('book', 'theme', 'set')).count():
 943         sender.reset_tag_counter()
 944     # if fragment theme changed, reset book theme counter
 945     elif isinstance(sender, Fragment) and \
 946                 Tag.objects.filter(pk__in=(tag.pk for tag in affected_tags)).\
 947                     filter(category='theme').count():
 948         sender.book.reset_theme_counter()
 949 tags_updated.connect(_tags_updated_handler)
 950
 951
 952 def _pre_delete_handler(sender, instance, **kwargs):
 953     """ refresh Book on BookMedia delete """
 954     if sender == BookMedia:
 955         instance.book.save()
 956 pre_delete.connect(_pre_delete_handler)
 957
 958 def _post_save_handler(sender, instance, **kwargs):
 959     """ refresh all the short_html stuff on BookMedia update """
 960     if sender == BookMedia:
 961         instance.book.save()
 962 post_save.connect(_post_save_handler)