apps/catalogue/models.py

   1 # -*- coding: utf-8 -*-
   2 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
   3 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
   4 #
   5 from datetime import datetime
   6
   7 from django.db import models
   8 from django.db.models import permalink, Q
   9 import django.dispatch
  10 from django.core.cache import cache
  11 from django.utils.translation import ugettext_lazy as _
  12 from django.contrib.auth.models import User
  13 from django.core.files import File
  14 from django.template.loader import render_to_string
  15 from django.utils.safestring import mark_safe
  16 from django.utils.translation import get_language
  17 from django.core.urlresolvers import reverse
  18 from django.db.models.signals import post_save, m2m_changed, pre_delete
  19
  20 from django.conf import settings
  21
  22 from newtagging.models import TagBase, tags_updated
  23 from newtagging import managers
  24 from catalogue.fields import JSONField, OverwritingFileField
  25 from catalogue.utils import ExistingFile, ORMDocProvider, create_zip, remove_zip
  26
  27 from librarian import dcparser, html, epub, NoDublinCore
  28 import mutagen
  29 from mutagen import id3
  30 from slughifi import slughifi
  31 from sortify import sortify
  32 from os import unlink
  33
  34 TAG_CATEGORIES = (
  35     ('author', _('author')),
  36     ('epoch', _('epoch')),
  37     ('kind', _('kind')),
  38     ('genre', _('genre')),
  39     ('theme', _('theme')),
  40     ('set', _('set')),
  41     ('book', _('book')),
  42 )
  43
  44 MEDIA_FORMATS = (
  45     ('odt', _('ODT file')),
  46     ('mp3', _('MP3 file')),
  47     ('ogg', _('OGG file')),
  48     ('daisy', _('DAISY file')),
  49 )
  50
  51 # not quite, but Django wants you to set a timeout
  52 CACHE_FOREVER = 2419200  # 28 days
  53
  54
  55 class TagSubcategoryManager(models.Manager):
  56     def __init__(self, subcategory):
  57         super(TagSubcategoryManager, self).__init__()
  58         self.subcategory = subcategory
  59
  60     def get_query_set(self):
  61         return super(TagSubcategoryManager, self).get_query_set().filter(category=self.subcategory)
  62
  63
  64 class Tag(TagBase):
  65     name = models.CharField(_('name'), max_length=50, db_index=True)
  66     slug = models.SlugField(_('slug'), max_length=120, db_index=True)
  67     sort_key = models.CharField(_('sort key'), max_length=120, db_index=True)
  68     category = models.CharField(_('category'), max_length=50, blank=False, null=False,
  69         db_index=True, choices=TAG_CATEGORIES)
  70     description = models.TextField(_('description'), blank=True)
  71     main_page = models.BooleanField(_('main page'), default=False, db_index=True, help_text=_('Show tag on main page'))
  72
  73     user = models.ForeignKey(User, blank=True, null=True)
  74     book_count = models.IntegerField(_('book count'), blank=True, null=True)
  75     gazeta_link = models.CharField(blank=True, max_length=240)
  76     wiki_link = models.CharField(blank=True, max_length=240)
  77
  78     created_at    = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True)
  79     changed_at    = models.DateTimeField(_('creation date'), auto_now=True, db_index=True)
  80
  81     class UrlDeprecationWarning(DeprecationWarning):
  82         pass
  83
  84     categories_rev = {
  85         'autor': 'author',
  86         'epoka': 'epoch',
  87         'rodzaj': 'kind',
  88         'gatunek': 'genre',
  89         'motyw': 'theme',
  90         'polka': 'set',
  91     }
  92     categories_dict = dict((item[::-1] for item in categories_rev.iteritems()))
  93
  94     class Meta:
  95         ordering = ('sort_key',)
  96         verbose_name = _('tag')
  97         verbose_name_plural = _('tags')
  98         unique_together = (("slug", "category"),)
  99
 100     def __unicode__(self):
 101         return self.name
 102
 103     def __repr__(self):
 104         return "Tag(slug=%r)" % self.slug
 105
 106     @permalink
 107     def get_absolute_url(self):
 108         return ('catalogue.views.tagged_object_list', [self.url_chunk])
 109
 110     def has_description(self):
 111         return len(self.description) > 0
 112     has_description.short_description = _('description')
 113     has_description.boolean = True
 114
 115     def get_count(self):
 116         """ returns global book count for book tags, fragment count for themes """
 117
 118         if self.book_count is None:
 119             if self.category == 'book':
 120                 # never used
 121                 objects = Book.objects.none()
 122             elif self.category == 'theme':
 123                 objects = Fragment.tagged.with_all((self,))
 124             else:
 125                 objects = Book.tagged.with_all((self,)).order_by()
 126                 if self.category != 'set':
 127                     # eliminate descendants
 128                     l_tags = Tag.objects.filter(slug__in=[book.book_tag_slug() for book in objects])
 129                     descendants_keys = [book.pk for book in Book.tagged.with_any(l_tags)]
 130                     if descendants_keys:
 131                         objects = objects.exclude(pk__in=descendants_keys)
 132             self.book_count = objects.count()
 133             self.save()
 134         return self.book_count
 135
 136     @staticmethod
 137     def get_tag_list(tags):
 138         if isinstance(tags, basestring):
 139             real_tags = []
 140             ambiguous_slugs = []
 141             category = None
 142             deprecated = False
 143             tags_splitted = tags.split('/')
 144             for name in tags_splitted:
 145                 if category:
 146                     real_tags.append(Tag.objects.get(slug=name, category=category))
 147                     category = None
 148                 elif name in Tag.categories_rev:
 149                     category = Tag.categories_rev[name]
 150                 else:
 151                     try:
 152                         real_tags.append(Tag.objects.exclude(category='book').get(slug=name))
 153                         deprecated = True
 154                     except Tag.MultipleObjectsReturned, e:
 155                         ambiguous_slugs.append(name)
 156
 157             if category:
 158                 # something strange left off
 159                 raise Tag.DoesNotExist()
 160             if ambiguous_slugs:
 161                 # some tags should be qualified
 162                 e = Tag.MultipleObjectsReturned()
 163                 e.tags = real_tags
 164                 e.ambiguous_slugs = ambiguous_slugs
 165                 raise e
 166             if deprecated:
 167                 e = Tag.UrlDeprecationWarning()
 168                 e.tags = real_tags
 169                 raise e
 170             return real_tags
 171         else:
 172             return TagBase.get_tag_list(tags)
 173
 174     @property
 175     def url_chunk(self):
 176         return '/'.join((Tag.categories_dict[self.category], self.slug))
 177
 178
 179 # TODO: why is this hard-coded ?
 180 def book_upload_path(ext=None, maxlen=100):
 181     def get_dynamic_path(media, filename, ext=ext):
 182         # how to put related book's slug here?
 183         if not ext:
 184             if media.type == 'daisy':
 185                 ext = 'daisy.zip'
 186             else:
 187                 ext = media.type
 188         if not media.name:
 189             name = slughifi(filename.split(".")[0])
 190         else:
 191             name = slughifi(media.name)
 192         return 'book/%s/%s.%s' % (ext, name[:maxlen-len('book/%s/.%s' % (ext, ext))-4], ext)
 193     return get_dynamic_path
 194
 195
 196 class BookMedia(models.Model):
 197     type        = models.CharField(_('type'), choices=MEDIA_FORMATS, max_length="100")
 198     name        = models.CharField(_('name'), max_length="100")
 199     file        = OverwritingFileField(_('file'), upload_to=book_upload_path())
 200     uploaded_at = models.DateTimeField(_('creation date'), auto_now_add=True, editable=False)
 201     extra_info  = JSONField(_('extra information'), default='{}', editable=False)
 202     book = models.ForeignKey('Book', related_name='media')
 203     source_sha1 = models.CharField(null=True, blank=True, max_length=40, editable=False)
 204
 205     def __unicode__(self):
 206         return "%s (%s)" % (self.name, self.file.name.split("/")[-1])
 207
 208     class Meta:
 209         ordering            = ('type', 'name')
 210         verbose_name        = _('book media')
 211         verbose_name_plural = _('book media')
 212
 213     def save(self, *args, **kwargs):
 214         try:
 215             old = BookMedia.objects.get(pk=self.pk)
 216         except BookMedia.DoesNotExist, e:
 217             pass
 218         else:
 219             # if name changed, change the file name, too
 220             if slughifi(self.name) != slughifi(old.name):
 221                 self.file.save(None, ExistingFile(self.file.path), save=False, leave=True)
 222
 223         super(BookMedia, self).save(*args, **kwargs)
 224
 225         # remove the zip package for book with modified media
 226         remove_zip(self.book.slug)
 227
 228         extra_info = self.get_extra_info_value()
 229         extra_info.update(self.read_meta())
 230         self.set_extra_info_value(extra_info)
 231         self.source_sha1 = self.read_source_sha1(self.file.path, self.type)
 232         return super(BookMedia, self).save(*args, **kwargs)
 233
 234     def read_meta(self):
 235         """
 236             Reads some metadata from the audiobook.
 237         """
 238
 239         artist_name = director_name = project = funded_by = ''
 240         if self.type == 'mp3':
 241             try:
 242                 audio = id3.ID3(self.file.path)
 243                 artist_name = ', '.join(', '.join(tag.text) for tag in audio.getall('TPE1'))
 244                 director_name = ', '.join(', '.join(tag.text) for tag in audio.getall('TPE3'))
 245                 project = ", ".join([t.data for t in audio.getall('PRIV')
 246                         if t.owner=='wolnelektury.pl?project'])
 247                 funded_by = ", ".join([t.data for t in audio.getall('PRIV')
 248                         if t.owner=='wolnelektury.pl?funded_by'])
 249             except:
 250                 pass
 251         elif self.type == 'ogg':
 252             try:
 253                 audio = mutagen.File(self.file.path)
 254                 artist_name = ', '.join(audio.get('artist', []))
 255                 director_name = ', '.join(audio.get('conductor', []))
 256                 project = ", ".join(audio.get('project', []))
 257                 funded_by = ", ".join(audio.get('funded_by', []))
 258             except:
 259                 pass
 260         else:
 261             return {}
 262         return {'artist_name': artist_name, 'director_name': director_name,
 263                 'project': project, 'funded_by': funded_by}
 264
 265     @staticmethod
 266     def read_source_sha1(filepath, filetype):
 267         """
 268             Reads source file SHA1 from audiobok metadata.
 269         """
 270
 271         if filetype == 'mp3':
 272             try:
 273                 audio = id3.ID3(filepath)
 274                 return [t.data for t in audio.getall('PRIV')
 275                         if t.owner=='wolnelektury.pl?flac_sha1'][0]
 276             except:
 277                 return None
 278         elif filetype == 'ogg':
 279             try:
 280                 audio = mutagen.File(filepath)
 281                 return audio.get('flac_sha1', [None])[0]
 282             except:
 283                 return None
 284         else:
 285             return None
 286
 287
 288 class Book(models.Model):
 289     title         = models.CharField(_('title'), max_length=120)
 290     sort_key = models.CharField(_('sort key'), max_length=120, db_index=True, editable=False)
 291     slug          = models.SlugField(_('slug'), max_length=120, unique=True, db_index=True)
 292     description   = models.TextField(_('description'), blank=True)
 293     created_at    = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True)
 294     changed_at    = models.DateTimeField(_('creation date'), auto_now=True, db_index=True)
 295     parent_number = models.IntegerField(_('parent number'), default=0)
 296     extra_info    = JSONField(_('extra information'), default='{}')
 297     gazeta_link   = models.CharField(blank=True, max_length=240)
 298     wiki_link     = models.CharField(blank=True, max_length=240)
 299     # files generated during publication
 300
 301     file_types = ['epub', 'html', 'mobi', 'pdf', 'txt', 'xml']
 302
 303     parent        = models.ForeignKey('self', blank=True, null=True, related_name='children')
 304     objects  = models.Manager()
 305     tagged   = managers.ModelTaggedItemManager(Tag)
 306     tags     = managers.TagDescriptor(Tag)
 307
 308     html_built = django.dispatch.Signal()
 309
 310     class AlreadyExists(Exception):
 311         pass
 312
 313     class Meta:
 314         ordering = ('sort_key',)
 315         verbose_name = _('book')
 316         verbose_name_plural = _('books')
 317
 318     def __unicode__(self):
 319         return self.title
 320
 321     def save(self, force_insert=False, force_update=False, reset_short_html=True, **kwargs):
 322         self.sort_key = sortify(self.title)
 323
 324         ret = super(Book, self).save(force_insert, force_update)
 325
 326         if reset_short_html:
 327             self.reset_short_html()
 328
 329         return ret
 330
 331     @permalink
 332     def get_absolute_url(self):
 333         return ('catalogue.views.book_detail', [self.slug])
 334
 335     @property
 336     def name(self):
 337         return self.title
 338
 339     def book_tag_slug(self):
 340         return ('l-' + self.slug)[:120]
 341
 342     def book_tag(self):
 343         slug = self.book_tag_slug()
 344         book_tag, created = Tag.objects.get_or_create(slug=slug, category='book')
 345         if created:
 346             book_tag.name = self.title[:50]
 347             book_tag.sort_key = self.title.lower()
 348             book_tag.save()
 349         return book_tag
 350
 351     def has_media(self, type):
 352         if type in Book.file_types:
 353             return bool(getattr(self, "%s_file" % type))
 354         else:
 355             return self.media.filter(type=type).exists()
 356
 357     def get_media(self, type):
 358         if self.has_media(type):
 359             if type in Book.file_types:
 360                 return getattr(self, "%s_file" % type)
 361             else:
 362                 return self.media.filter(type=type)
 363         else:
 364             return None
 365
 366     def get_mp3(self):
 367         return self.get_media("mp3")
 368     def get_odt(self):
 369         return self.get_media("odt")
 370     def get_ogg(self):
 371         return self.get_media("ogg")
 372     def get_daisy(self):
 373         return self.get_media("daisy")
 374
 375     def reset_short_html(self):
 376         if self.id is None:
 377             return
 378
 379         cache_key = "Book.short_html/%d/%s"
 380         for lang, langname in settings.LANGUAGES:
 381             cache.delete(cache_key % (self.id, lang))
 382         # Fragment.short_html relies on book's tags, so reset it here too
 383         for fragm in self.fragments.all():
 384             fragm.reset_short_html()
 385
 386     def short_html(self):
 387         if self.id:
 388             cache_key = "Book.short_html/%d/%s" % (self.id, get_language())
 389             short_html = cache.get(cache_key)
 390         else:
 391             short_html = None
 392
 393         if short_html is not None:
 394             return mark_safe(short_html)
 395         else:
 396             tags = self.tags.filter(~Q(category__in=('set', 'theme', 'book')))
 397             tags = [mark_safe(u'<a href="%s">%s</a>' % (tag.get_absolute_url(), tag.name)) for tag in tags]
 398
 399             formats = []
 400             # files generated during publication
 401             if self.has_media("html"):
 402                 formats.append(u'<a href="%s">%s</a>' % (reverse('book_text', kwargs={'slug': self.slug}), _('Read online')))
 403             if self.has_media("pdf"):
 404                 formats.append(u'<a href="%s">PDF</a>' % self.get_media('pdf').url)
 405             if self.has_media("mobi"):
 406                 formats.append(u'<a href="%s">MOBI</a>' % self.get_media('mobi').url)
 407             if self.root_ancestor.has_media("epub"):
 408                 formats.append(u'<a href="%s">EPUB</a>' % self.root_ancestor.get_media('epub').url)
 409             if self.has_media("txt"):
 410                 formats.append(u'<a href="%s">TXT</a>' % self.get_media('txt').url)
 411             # other files
 412             for m in self.media.order_by('type'):
 413                 formats.append(u'<a href="%s">%s</a>' % (m.file.url, m.type.upper()))
 414
 415             formats = [mark_safe(format) for format in formats]
 416
 417             short_html = unicode(render_to_string('catalogue/book_short.html',
 418                 {'book': self, 'tags': tags, 'formats': formats}))
 419
 420             if self.id:
 421                 cache.set(cache_key, short_html, CACHE_FOREVER)
 422             return mark_safe(short_html)
 423
 424     @property
 425     def root_ancestor(self):
 426         """ returns the oldest ancestor """
 427
 428         if not hasattr(self, '_root_ancestor'):
 429             book = self
 430             while book.parent:
 431                 book = book.parent
 432             self._root_ancestor = book
 433         return self._root_ancestor
 434
 435
 436     def has_description(self):
 437         return len(self.description) > 0
 438     has_description.short_description = _('description')
 439     has_description.boolean = True
 440
 441     # ugly ugly ugly
 442     def has_odt_file(self):
 443         return bool(self.has_media("odt"))
 444     has_odt_file.short_description = 'ODT'
 445     has_odt_file.boolean = True
 446
 447     def has_mp3_file(self):
 448         return bool(self.has_media("mp3"))
 449     has_mp3_file.short_description = 'MP3'
 450     has_mp3_file.boolean = True
 451
 452     def has_ogg_file(self):
 453         return bool(self.has_media("ogg"))
 454     has_ogg_file.short_description = 'OGG'
 455     has_ogg_file.boolean = True
 456
 457     def has_daisy_file(self):
 458         return bool(self.has_media("daisy"))
 459     has_daisy_file.short_description = 'DAISY'
 460     has_daisy_file.boolean = True
 461
 462     def build_pdf(self):
 463         """ (Re)builds the pdf file.
 464
 465         """
 466         from librarian import pdf
 467         from tempfile import NamedTemporaryFile
 468         import os
 469
 470         try:
 471             pdf_file = NamedTemporaryFile(delete=False)
 472             pdf.transform(ORMDocProvider(self),
 473                       file_path=str(self.xml_file.path),
 474                       output_file=pdf_file,
 475                       )
 476
 477             self.pdf_file.save('%s.pdf' % self.slug, File(open(pdf_file.name)))
 478         finally:
 479             unlink(pdf_file.name)
 480
 481         # remove zip with all pdf files
 482         remove_zip(settings.ALL_PDF_ZIP)
 483
 484     def build_mobi(self):
 485         """ (Re)builds the MOBI file.
 486
 487         """
 488         from librarian import mobi
 489         from tempfile import NamedTemporaryFile
 490         import os
 491
 492         try:
 493             mobi_file = NamedTemporaryFile(suffix='.mobi', delete=False)
 494             mobi.transform(ORMDocProvider(self), verbose=1,
 495                       file_path=str(self.xml_file.path),
 496                       output_file=mobi_file.name,
 497                       )
 498
 499             self.mobi_file.save('%s.mobi' % self.slug, File(open(mobi_file.name)))
 500         finally:
 501             unlink(mobi_file.name)
 502
 503         # remove zip with all mobi files
 504         remove_zip(settings.ALL_MOBI_ZIP)
 505
 506     def build_epub(self, remove_descendants=True):
 507         """ (Re)builds the epub file.
 508             If book has a parent, does nothing.
 509             Unless remove_descendants is False, descendants' epubs are removed.
 510         """
 511         from StringIO import StringIO
 512         from hashlib import sha1
 513         from django.core.files.base import ContentFile
 514
 515         if self.parent:
 516             # don't need an epub
 517             return
 518
 519         epub_file = StringIO()
 520         try:
 521             epub.transform(ORMDocProvider(self), self.slug, output_file=epub_file)
 522             self.epub_file.save('%s.epub' % self.slug, ContentFile(epub_file.getvalue()))
 523             FileRecord(slug=self.slug, type='epub', sha1=sha1(epub_file.getvalue()).hexdigest()).save()
 524         except NoDublinCore:
 525             pass
 526
 527         book_descendants = list(self.children.all())
 528         while len(book_descendants) > 0:
 529             child_book = book_descendants.pop(0)
 530             if remove_descendants and child_book.has_epub_file():
 531                 child_book.epub_file.delete()
 532             # save anyway, to refresh short_html
 533             child_book.save()
 534             book_descendants += list(child_book.children.all())
 535
 536         # remove zip package with all epub files
 537         remove_zip(settings.ALL_EPUB_ZIP)
 538
 539     def build_txt(self):
 540         from StringIO import StringIO
 541         from django.core.files.base import ContentFile
 542         from librarian import text
 543
 544         out = StringIO()
 545         text.transform(open(self.xml_file.path), out)
 546         self.txt_file.save('%s.txt' % self.slug, ContentFile(out.getvalue()))
 547
 548
 549     def build_html(self):
 550         from tempfile import NamedTemporaryFile
 551         from markupstring import MarkupString
 552
 553         meta_tags = list(self.tags.filter(
 554             category__in=('author', 'epoch', 'genre', 'kind')))
 555         book_tag = self.book_tag()
 556
 557         html_file = NamedTemporaryFile()
 558         if html.transform(self.xml_file.path, html_file, parse_dublincore=False):
 559             self.html_file.save('%s.html' % self.slug, File(html_file))
 560
 561             # get ancestor l-tags for adding to new fragments
 562             ancestor_tags = []
 563             p = self.parent
 564             while p:
 565                 ancestor_tags.append(p.book_tag())
 566                 p = p.parent
 567
 568             # Delete old fragments and create them from scratch
 569             self.fragments.all().delete()
 570             # Extract fragments
 571             closed_fragments, open_fragments = html.extract_fragments(self.html_file.path)
 572             for fragment in closed_fragments.values():
 573                 try:
 574                     theme_names = [s.strip() for s in fragment.themes.split(',')]
 575                 except AttributeError:
 576                     continue
 577                 themes = []
 578                 for theme_name in theme_names:
 579                     if not theme_name:
 580                         continue
 581                     tag, created = Tag.objects.get_or_create(slug=slughifi(theme_name), category='theme')
 582                     if created:
 583                         tag.name = theme_name
 584                         tag.sort_key = theme_name.lower()
 585                         tag.save()
 586                     themes.append(tag)
 587                 if not themes:
 588                     continue
 589
 590                 text = fragment.to_string()
 591                 short_text = ''
 592                 if (len(MarkupString(text)) > 240):
 593                     short_text = unicode(MarkupString(text)[:160])
 594                 new_fragment = Fragment.objects.create(anchor=fragment.id, book=self,
 595                     text=text, short_text=short_text)
 596
 597                 new_fragment.save()
 598                 new_fragment.tags = set(meta_tags + themes + [book_tag] + ancestor_tags)
 599             self.save()
 600             self.html_built.send(sender=self)
 601             return True
 602         return False
 603
 604     @staticmethod
 605     def zip_format(format_):
 606         def pretty_file_name(book):
 607             return "%s/%s.%s" % (
 608                 b.get_extra_info_value()['author'],
 609                 b.slug,
 610                 format_)
 611
 612         field_name = "%s_file" % format_
 613         books = Book.objects.filter(parent=None).exclude(**{field_name: ""})
 614         paths = [(pretty_file_name(b), getattr(b, field_name).path)
 615                     for b in books]
 616         result = create_zip.delay(paths,
 617                     getattr(settings, "ALL_%s_ZIP" % format_.upper()))
 618
 619     def zip_audiobooks(self):
 620         bm = BookMedia.objects.filter(book=self, type='mp3')
 621         paths = map(lambda bm: (None, bm.file.path), bm)
 622         result = create_zip.delay(paths, self.slug)
 623         return result.wait()
 624
 625     @classmethod
 626     def from_xml_file(cls, xml_file, **kwargs):
 627         # use librarian to parse meta-data
 628         book_info = dcparser.parse(xml_file)
 629
 630         if not isinstance(xml_file, File):
 631             xml_file = File(open(xml_file))
 632
 633         try:
 634             return cls.from_text_and_meta(xml_file, book_info, **kwargs)
 635         finally:
 636             xml_file.close()
 637
 638     @classmethod
 639     def from_text_and_meta(cls, raw_file, book_info, overwrite=False,
 640             build_epub=True, build_txt=True, build_pdf=True, build_mobi=True):
 641         import re
 642
 643         # check for parts before we do anything
 644         children = []
 645         if hasattr(book_info, 'parts'):
 646             for part_url in book_info.parts:
 647                 base, slug = part_url.rsplit('/', 1)
 648                 try:
 649                     children.append(Book.objects.get(slug=slug))
 650                 except Book.DoesNotExist, e:
 651                     raise Book.DoesNotExist(_('Book with slug = "%s" does not exist.') % slug)
 652
 653
 654         # Read book metadata
 655         book_base, book_slug = book_info.url.rsplit('/', 1)
 656         if re.search(r'[^a-zA-Z0-9-]', book_slug):
 657             raise ValueError('Invalid characters in slug')
 658         book, created = Book.objects.get_or_create(slug=book_slug)
 659
 660         if created:
 661             book_shelves = []
 662         else:
 663             if not overwrite:
 664                 raise Book.AlreadyExists(_('Book %s already exists') % book_slug)
 665             # Save shelves for this book
 666             book_shelves = list(book.tags.filter(category='set'))
 667
 668         book.title = book_info.title
 669         book.set_extra_info_value(book_info.to_dict())
 670         book.save()
 671
 672         meta_tags = []
 673         categories = (('kinds', 'kind'), ('genres', 'genre'), ('authors', 'author'), ('epochs', 'epoch'))
 674         for field_name, category in categories:
 675             try:
 676                 tag_names = getattr(book_info, field_name)
 677             except:
 678                 tag_names = [getattr(book_info, category)]
 679             for tag_name in tag_names:
 680                 tag_sort_key = tag_name
 681                 if category == 'author':
 682                     tag_sort_key = tag_name.last_name
 683                     tag_name = ' '.join(tag_name.first_names) + ' ' + tag_name.last_name
 684                 tag, created = Tag.objects.get_or_create(slug=slughifi(tag_name), category=category)
 685                 if created:
 686                     tag.name = tag_name
 687                     tag.sort_key = sortify(tag_sort_key.lower())
 688                     tag.save()
 689                 meta_tags.append(tag)
 690
 691         book.tags = set(meta_tags + book_shelves)
 692
 693         book_tag = book.book_tag()
 694
 695         for n, child_book in enumerate(children):
 696             child_book.parent = book
 697             child_book.parent_number = n
 698             child_book.save()
 699
 700         # Save XML and HTML files
 701         book.xml_file.save('%s.xml' % book.slug, raw_file, save=False)
 702
 703         # delete old fragments when overwriting
 704         book.fragments.all().delete()
 705
 706         if book.build_html():
 707             if not settings.NO_BUILD_TXT and build_txt:
 708                 book.build_txt()
 709
 710         if not settings.NO_BUILD_EPUB and build_epub:
 711             book.root_ancestor.build_epub()
 712
 713         if not settings.NO_BUILD_PDF and build_pdf:
 714             book.root_ancestor.build_pdf()
 715
 716         if not settings.NO_BUILD_MOBI and build_mobi:
 717             book.build_mobi()
 718
 719         book_descendants = list(book.children.all())
 720         # add l-tag to descendants and their fragments
 721         # delete unnecessary EPUB files
 722         while len(book_descendants) > 0:
 723             child_book = book_descendants.pop(0)
 724             child_book.tags = list(child_book.tags) + [book_tag]
 725             child_book.save()
 726             for fragment in child_book.fragments.all():
 727                 fragment.tags = set(list(fragment.tags) + [book_tag])
 728             book_descendants += list(child_book.children.all())
 729
 730         book.save()
 731
 732         # refresh cache
 733         book.reset_tag_counter()
 734         book.reset_theme_counter()
 735
 736         return book
 737
 738     def reset_tag_counter(self):
 739         if self.id is None:
 740             return
 741
 742         cache_key = "Book.tag_counter/%d" % self.id
 743         cache.delete(cache_key)
 744         if self.parent:
 745             self.parent.reset_tag_counter()
 746
 747     @property
 748     def tag_counter(self):
 749         if self.id:
 750             cache_key = "Book.tag_counter/%d" % self.id
 751             tags = cache.get(cache_key)
 752         else:
 753             tags = None
 754
 755         if tags is None:
 756             tags = {}
 757             for child in self.children.all().order_by():
 758                 for tag_pk, value in child.tag_counter.iteritems():
 759                     tags[tag_pk] = tags.get(tag_pk, 0) + value
 760             for tag in self.tags.exclude(category__in=('book', 'theme', 'set')).order_by():
 761                 tags[tag.pk] = 1
 762
 763             if self.id:
 764                 cache.set(cache_key, tags, CACHE_FOREVER)
 765         return tags
 766
 767     def reset_theme_counter(self):
 768         if self.id is None:
 769             return
 770
 771         cache_key = "Book.theme_counter/%d" % self.id
 772         cache.delete(cache_key)
 773         if self.parent:
 774             self.parent.reset_theme_counter()
 775
 776     @property
 777     def theme_counter(self):
 778         if self.id:
 779             cache_key = "Book.theme_counter/%d" % self.id
 780             tags = cache.get(cache_key)
 781         else:
 782             tags = None
 783
 784         if tags is None:
 785             tags = {}
 786             for fragment in Fragment.tagged.with_any([self.book_tag()]).order_by():
 787                 for tag in fragment.tags.filter(category='theme').order_by():
 788                     tags[tag.pk] = tags.get(tag.pk, 0) + 1
 789
 790             if self.id:
 791                 cache.set(cache_key, tags, CACHE_FOREVER)
 792         return tags
 793
 794     def pretty_title(self, html_links=False):
 795         book = self
 796         names = list(book.tags.filter(category='author'))
 797
 798         books = []
 799         while book:
 800             books.append(book)
 801             book = book.parent
 802         names.extend(reversed(books))
 803
 804         if html_links:
 805             names = ['<a href="%s">%s</a>' % (tag.get_absolute_url(), tag.name) for tag in names]
 806         else:
 807             names = [tag.name for tag in names]
 808
 809         return ', '.join(names)
 810
 811     @classmethod
 812     def tagged_top_level(cls, tags):
 813         """ Returns top-level books tagged with `tags'.
 814
 815         It only returns those books which don't have ancestors which are
 816         also tagged with those tags.
 817
 818         """
 819         # get relevant books and their tags
 820         objects = cls.tagged.with_all(tags)
 821         # eliminate descendants
 822         l_tags = Tag.objects.filter(category='book', slug__in=[book.book_tag_slug() for book in objects])
 823         descendants_keys = [book.pk for book in cls.tagged.with_any(l_tags)]
 824         if descendants_keys:
 825             objects = objects.exclude(pk__in=descendants_keys)
 826
 827         return objects
 828
 829
 830 def _has_factory(ftype):
 831     has = lambda self: bool(getattr(self, "%s_file" % ftype))
 832     has.short_description = t.upper()
 833     has.boolean = True
 834     has.__name__ = "has_%s_file" % ftype
 835     return has
 836
 837
 838 # add the file fields
 839 for t in Book.file_types:
 840     field_name = "%s_file" % t
 841     models.FileField(_("%s file" % t.upper()),
 842             upload_to=book_upload_path(t),
 843             blank=True).contribute_to_class(Book, field_name)
 844
 845     setattr(Book, "has_%s_file" % t, _has_factory(t))
 846
 847
 848 class Fragment(models.Model):
 849     text = models.TextField()
 850     short_text = models.TextField(editable=False)
 851     anchor = models.CharField(max_length=120)
 852     book = models.ForeignKey(Book, related_name='fragments')
 853
 854     objects = models.Manager()
 855     tagged = managers.ModelTaggedItemManager(Tag)
 856     tags = managers.TagDescriptor(Tag)
 857
 858     class Meta:
 859         ordering = ('book', 'anchor',)
 860         verbose_name = _('fragment')
 861         verbose_name_plural = _('fragments')
 862
 863     def get_absolute_url(self):
 864         return '%s#m%s' % (reverse('book_text', kwargs={'slug': self.book.slug}), self.anchor)
 865
 866     def reset_short_html(self):
 867         if self.id is None:
 868             return
 869
 870         cache_key = "Fragment.short_html/%d/%s"
 871         for lang, langname in settings.LANGUAGES:
 872             cache.delete(cache_key % (self.id, lang))
 873
 874     def short_html(self):
 875         if self.id:
 876             cache_key = "Fragment.short_html/%d/%s" % (self.id, get_language())
 877             short_html = cache.get(cache_key)
 878         else:
 879             short_html = None
 880
 881         if short_html is not None:
 882             return mark_safe(short_html)
 883         else:
 884             short_html = unicode(render_to_string('catalogue/fragment_short.html',
 885                 {'fragment': self}))
 886             if self.id:
 887                 cache.set(cache_key, short_html, CACHE_FOREVER)
 888             return mark_safe(short_html)
 889
 890
 891 class FileRecord(models.Model):
 892     slug = models.SlugField(_('slug'), max_length=120, db_index=True)
 893     type = models.CharField(_('type'), max_length=20, db_index=True)
 894     sha1 = models.CharField(_('sha-1 hash'), max_length=40)
 895     time = models.DateTimeField(_('time'), auto_now_add=True)
 896
 897     class Meta:
 898         ordering = ('-time','-slug', '-type')
 899         verbose_name = _('file record')
 900         verbose_name_plural = _('file records')
 901
 902     def __unicode__(self):
 903         return "%s %s.%s" % (self.sha1,  self.slug, self.type)
 904
 905 ###########
 906 #
 907 # SIGNALS
 908 #
 909 ###########
 910
 911
 912 def _tags_updated_handler(sender, affected_tags, **kwargs):
 913     # reset tag global counter
 914     # we want Tag.changed_at updated for API to know the tag was touched
 915     Tag.objects.filter(pk__in=[tag.pk for tag in affected_tags]).update(book_count=None, changed_at=datetime.now())
 916
 917     # if book tags changed, reset book tag counter
 918     if isinstance(sender, Book) and \
 919                 Tag.objects.filter(pk__in=(tag.pk for tag in affected_tags)).\
 920                     exclude(category__in=('book', 'theme', 'set')).count():
 921         sender.reset_tag_counter()
 922     # if fragment theme changed, reset book theme counter
 923     elif isinstance(sender, Fragment) and \
 924                 Tag.objects.filter(pk__in=(tag.pk for tag in affected_tags)).\
 925                     filter(category='theme').count():
 926         sender.book.reset_theme_counter()
 927 tags_updated.connect(_tags_updated_handler)
 928
 929
 930 def _pre_delete_handler(sender, instance, **kwargs):
 931     """ refresh Book on BookMedia delete """
 932     if sender == BookMedia:
 933         instance.book.save()
 934 pre_delete.connect(_pre_delete_handler)
 935
 936 def _post_save_handler(sender, instance, **kwargs):
 937     """ refresh all the short_html stuff on BookMedia update """
 938     if sender == BookMedia:
 939         instance.book.save()
 940 post_save.connect(_post_save_handler)