apps/catalogue/models.py

   1 # -*- coding: utf-8 -*-
   2 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
   3 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
   4 #
   5 from datetime import datetime
   6
   7 from django.db import models
   8 from django.db.models import permalink, Q
   9 import django.dispatch
  10 from django.core.cache import cache
  11 from django.utils.translation import ugettext_lazy as _
  12 from django.contrib.auth.models import User
  13 from django.core.files import File
  14 from django.template.loader import render_to_string
  15 from django.utils.datastructures import SortedDict
  16 from django.utils.safestring import mark_safe
  17 from django.utils.translation import get_language
  18 from django.core.urlresolvers import reverse
  19 from django.db.models.signals import post_save, m2m_changed, pre_delete
  20
  21 from django.conf import settings
  22
  23 from newtagging.models import TagBase, tags_updated
  24 from newtagging import managers
  25 from catalogue.fields import JSONField, OverwritingFileField
  26 from catalogue.utils import ExistingFile, ORMDocProvider, create_zip, remove_zip
  27
  28 from librarian import dcparser, html, epub, NoDublinCore
  29 import mutagen
  30 from mutagen import id3
  31 from slughifi import slughifi
  32 from sortify import sortify
  33 from os import unlink
  34
  35 TAG_CATEGORIES = (
  36     ('author', _('author')),
  37     ('epoch', _('epoch')),
  38     ('kind', _('kind')),
  39     ('genre', _('genre')),
  40     ('theme', _('theme')),
  41     ('set', _('set')),
  42     ('book', _('book')),
  43 )
  44
  45 MEDIA_FORMATS = (
  46     ('odt', _('ODT file')),
  47     ('mp3', _('MP3 file')),
  48     ('ogg', _('OGG file')),
  49     ('daisy', _('DAISY file')),
  50 )
  51
  52 # not quite, but Django wants you to set a timeout
  53 CACHE_FOREVER = 2419200  # 28 days
  54
  55
  56 class TagSubcategoryManager(models.Manager):
  57     def __init__(self, subcategory):
  58         super(TagSubcategoryManager, self).__init__()
  59         self.subcategory = subcategory
  60
  61     def get_query_set(self):
  62         return super(TagSubcategoryManager, self).get_query_set().filter(category=self.subcategory)
  63
  64
  65 class Tag(TagBase):
  66     name = models.CharField(_('name'), max_length=50, db_index=True)
  67     slug = models.SlugField(_('slug'), max_length=120, db_index=True)
  68     sort_key = models.CharField(_('sort key'), max_length=120, db_index=True)
  69     category = models.CharField(_('category'), max_length=50, blank=False, null=False,
  70         db_index=True, choices=TAG_CATEGORIES)
  71     description = models.TextField(_('description'), blank=True)
  72     main_page = models.BooleanField(_('main page'), default=False, db_index=True, help_text=_('Show tag on main page'))
  73
  74     user = models.ForeignKey(User, blank=True, null=True)
  75     book_count = models.IntegerField(_('book count'), blank=True, null=True)
  76     gazeta_link = models.CharField(blank=True, max_length=240)
  77     wiki_link = models.CharField(blank=True, max_length=240)
  78
  79     created_at    = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True)
  80     changed_at    = models.DateTimeField(_('creation date'), auto_now=True, db_index=True)
  81
  82     class UrlDeprecationWarning(DeprecationWarning):
  83         pass
  84
  85     categories_rev = {
  86         'autor': 'author',
  87         'epoka': 'epoch',
  88         'rodzaj': 'kind',
  89         'gatunek': 'genre',
  90         'motyw': 'theme',
  91         'polka': 'set',
  92     }
  93     categories_dict = dict((item[::-1] for item in categories_rev.iteritems()))
  94
  95     class Meta:
  96         ordering = ('sort_key',)
  97         verbose_name = _('tag')
  98         verbose_name_plural = _('tags')
  99         unique_together = (("slug", "category"),)
 100
 101     def __unicode__(self):
 102         return self.name
 103
 104     def __repr__(self):
 105         return "Tag(slug=%r)" % self.slug
 106
 107     @permalink
 108     def get_absolute_url(self):
 109         return ('catalogue.views.tagged_object_list', [self.url_chunk])
 110
 111     def has_description(self):
 112         return len(self.description) > 0
 113     has_description.short_description = _('description')
 114     has_description.boolean = True
 115
 116     def get_count(self):
 117         """ returns global book count for book tags, fragment count for themes """
 118
 119         if self.book_count is None:
 120             if self.category == 'book':
 121                 # never used
 122                 objects = Book.objects.none()
 123             elif self.category == 'theme':
 124                 objects = Fragment.tagged.with_all((self,))
 125             else:
 126                 objects = Book.tagged.with_all((self,)).order_by()
 127                 if self.category != 'set':
 128                     # eliminate descendants
 129                     l_tags = Tag.objects.filter(slug__in=[book.book_tag_slug() for book in objects])
 130                     descendants_keys = [book.pk for book in Book.tagged.with_any(l_tags)]
 131                     if descendants_keys:
 132                         objects = objects.exclude(pk__in=descendants_keys)
 133             self.book_count = objects.count()
 134             self.save()
 135         return self.book_count
 136
 137     @staticmethod
 138     def get_tag_list(tags):
 139         if isinstance(tags, basestring):
 140             real_tags = []
 141             ambiguous_slugs = []
 142             category = None
 143             deprecated = False
 144             tags_splitted = tags.split('/')
 145             for name in tags_splitted:
 146                 if category:
 147                     real_tags.append(Tag.objects.get(slug=name, category=category))
 148                     category = None
 149                 elif name in Tag.categories_rev:
 150                     category = Tag.categories_rev[name]
 151                 else:
 152                     try:
 153                         real_tags.append(Tag.objects.exclude(category='book').get(slug=name))
 154                         deprecated = True
 155                     except Tag.MultipleObjectsReturned, e:
 156                         ambiguous_slugs.append(name)
 157
 158             if category:
 159                 # something strange left off
 160                 raise Tag.DoesNotExist()
 161             if ambiguous_slugs:
 162                 # some tags should be qualified
 163                 e = Tag.MultipleObjectsReturned()
 164                 e.tags = real_tags
 165                 e.ambiguous_slugs = ambiguous_slugs
 166                 raise e
 167             if deprecated:
 168                 e = Tag.UrlDeprecationWarning()
 169                 e.tags = real_tags
 170                 raise e
 171             return real_tags
 172         else:
 173             return TagBase.get_tag_list(tags)
 174
 175     @property
 176     def url_chunk(self):
 177         return '/'.join((Tag.categories_dict[self.category], self.slug))
 178
 179
 180 # TODO: why is this hard-coded ?
 181 def book_upload_path(ext=None, maxlen=100):
 182     def get_dynamic_path(media, filename, ext=ext):
 183         # how to put related book's slug here?
 184         if not ext:
 185             if media.type == 'daisy':
 186                 ext = 'daisy.zip'
 187             else:
 188                 ext = media.type
 189         if not media.name:
 190             name = slughifi(filename.split(".")[0])
 191         else:
 192             name = slughifi(media.name)
 193         return 'book/%s/%s.%s' % (ext, name[:maxlen-len('book/%s/.%s' % (ext, ext))-4], ext)
 194     return get_dynamic_path
 195
 196
 197 class BookMedia(models.Model):
 198     type        = models.CharField(_('type'), choices=MEDIA_FORMATS, max_length="100")
 199     name        = models.CharField(_('name'), max_length="100")
 200     file        = OverwritingFileField(_('file'), upload_to=book_upload_path())
 201     uploaded_at = models.DateTimeField(_('creation date'), auto_now_add=True, editable=False)
 202     extra_info  = JSONField(_('extra information'), default='{}', editable=False)
 203     book = models.ForeignKey('Book', related_name='media')
 204     source_sha1 = models.CharField(null=True, blank=True, max_length=40, editable=False)
 205
 206     def __unicode__(self):
 207         return "%s (%s)" % (self.name, self.file.name.split("/")[-1])
 208
 209     class Meta:
 210         ordering            = ('type', 'name')
 211         verbose_name        = _('book media')
 212         verbose_name_plural = _('book media')
 213
 214     def save(self, *args, **kwargs):
 215         try:
 216             old = BookMedia.objects.get(pk=self.pk)
 217         except BookMedia.DoesNotExist, e:
 218             pass
 219         else:
 220             # if name changed, change the file name, too
 221             if slughifi(self.name) != slughifi(old.name):
 222                 self.file.save(None, ExistingFile(self.file.path), save=False, leave=True)
 223
 224         super(BookMedia, self).save(*args, **kwargs)
 225
 226         # remove the zip package for book with modified media
 227         remove_zip(self.book.slug)
 228
 229         extra_info = self.get_extra_info_value()
 230         extra_info.update(self.read_meta())
 231         self.set_extra_info_value(extra_info)
 232         self.source_sha1 = self.read_source_sha1(self.file.path, self.type)
 233         return super(BookMedia, self).save(*args, **kwargs)
 234
 235     def read_meta(self):
 236         """
 237             Reads some metadata from the audiobook.
 238         """
 239
 240         artist_name = director_name = project = funded_by = ''
 241         if self.type == 'mp3':
 242             try:
 243                 audio = id3.ID3(self.file.path)
 244                 artist_name = ', '.join(', '.join(tag.text) for tag in audio.getall('TPE1'))
 245                 director_name = ', '.join(', '.join(tag.text) for tag in audio.getall('TPE3'))
 246                 project = ", ".join([t.data for t in audio.getall('PRIV')
 247                         if t.owner=='wolnelektury.pl?project'])
 248                 funded_by = ", ".join([t.data for t in audio.getall('PRIV')
 249                         if t.owner=='wolnelektury.pl?funded_by'])
 250             except:
 251                 pass
 252         elif self.type == 'ogg':
 253             try:
 254                 audio = mutagen.File(self.file.path)
 255                 artist_name = ', '.join(audio.get('artist', []))
 256                 director_name = ', '.join(audio.get('conductor', []))
 257                 project = ", ".join(audio.get('project', []))
 258                 funded_by = ", ".join(audio.get('funded_by', []))
 259             except:
 260                 pass
 261         else:
 262             return {}
 263         return {'artist_name': artist_name, 'director_name': director_name,
 264                 'project': project, 'funded_by': funded_by}
 265
 266     @staticmethod
 267     def read_source_sha1(filepath, filetype):
 268         """
 269             Reads source file SHA1 from audiobok metadata.
 270         """
 271
 272         if filetype == 'mp3':
 273             try:
 274                 audio = id3.ID3(filepath)
 275                 return [t.data for t in audio.getall('PRIV')
 276                         if t.owner=='wolnelektury.pl?flac_sha1'][0]
 277             except:
 278                 return None
 279         elif filetype == 'ogg':
 280             try:
 281                 audio = mutagen.File(filepath)
 282                 return audio.get('flac_sha1', [None])[0]
 283             except:
 284                 return None
 285         else:
 286             return None
 287
 288
 289 class Book(models.Model):
 290     title         = models.CharField(_('title'), max_length=120)
 291     sort_key = models.CharField(_('sort key'), max_length=120, db_index=True, editable=False)
 292     slug          = models.SlugField(_('slug'), max_length=120, unique=True, db_index=True)
 293     description   = models.TextField(_('description'), blank=True)
 294     created_at    = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True)
 295     changed_at    = models.DateTimeField(_('creation date'), auto_now=True, db_index=True)
 296     parent_number = models.IntegerField(_('parent number'), default=0)
 297     extra_info    = JSONField(_('extra information'), default='{}')
 298     gazeta_link   = models.CharField(blank=True, max_length=240)
 299     wiki_link     = models.CharField(blank=True, max_length=240)
 300     # files generated during publication
 301
 302     file_types = ['epub', 'html', 'mobi', 'pdf', 'txt', 'xml']
 303
 304     parent        = models.ForeignKey('self', blank=True, null=True, related_name='children')
 305     objects  = models.Manager()
 306     tagged   = managers.ModelTaggedItemManager(Tag)
 307     tags     = managers.TagDescriptor(Tag)
 308
 309     html_built = django.dispatch.Signal()
 310     published = django.dispatch.Signal()
 311
 312     class AlreadyExists(Exception):
 313         pass
 314
 315     class Meta:
 316         ordering = ('sort_key',)
 317         verbose_name = _('book')
 318         verbose_name_plural = _('books')
 319
 320     def __unicode__(self):
 321         return self.title
 322
 323     def save(self, force_insert=False, force_update=False, reset_short_html=True, **kwargs):
 324         self.sort_key = sortify(self.title)
 325
 326         ret = super(Book, self).save(force_insert, force_update)
 327
 328         if reset_short_html:
 329             self.reset_short_html()
 330
 331         return ret
 332
 333     @permalink
 334     def get_absolute_url(self):
 335         return ('catalogue.views.book_detail', [self.slug])
 336
 337     @property
 338     def name(self):
 339         return self.title
 340
 341     def book_tag_slug(self):
 342         return ('l-' + self.slug)[:120]
 343
 344     def book_tag(self):
 345         slug = self.book_tag_slug()
 346         book_tag, created = Tag.objects.get_or_create(slug=slug, category='book')
 347         if created:
 348             book_tag.name = self.title[:50]
 349             book_tag.sort_key = self.title.lower()
 350             book_tag.save()
 351         return book_tag
 352
 353     def has_media(self, type):
 354         if type in Book.file_types:
 355             return bool(getattr(self, "%s_file" % type))
 356         else:
 357             return self.media.filter(type=type).exists()
 358
 359     def get_media(self, type):
 360         if self.has_media(type):
 361             if type in Book.file_types:
 362                 return getattr(self, "%s_file" % type)
 363             else:
 364                 return self.media.filter(type=type)
 365         else:
 366             return None
 367
 368     def get_mp3(self):
 369         return self.get_media("mp3")
 370     def get_odt(self):
 371         return self.get_media("odt")
 372     def get_ogg(self):
 373         return self.get_media("ogg")
 374     def get_daisy(self):
 375         return self.get_media("daisy")
 376
 377     def reset_short_html(self):
 378         if self.id is None:
 379             return
 380
 381         cache_key = "Book.short_html/%d/%s"
 382         for lang, langname in settings.LANGUAGES:
 383             cache.delete(cache_key % (self.id, lang))
 384         # Fragment.short_html relies on book's tags, so reset it here too
 385         for fragm in self.fragments.all():
 386             fragm.reset_short_html()
 387
 388     def short_html(self):
 389         if self.id:
 390             cache_key = "Book.short_html/%d/%s" % (self.id, get_language())
 391             short_html = cache.get(cache_key)
 392         else:
 393             short_html = None
 394
 395         if short_html is not None:
 396             return mark_safe(short_html)
 397         else:
 398             tags = self.tags.filter(~Q(category__in=('set', 'theme', 'book')))
 399             tags = [mark_safe(u'<a href="%s">%s</a>' % (tag.get_absolute_url(), tag.name)) for tag in tags]
 400
 401             formats = []
 402             # files generated during publication
 403             if self.has_media("html"):
 404                 formats.append(u'<a href="%s">%s</a>' % (reverse('book_text', kwargs={'slug': self.slug}), _('Read online')))
 405             if self.has_media("pdf"):
 406                 formats.append(u'<a href="%s">PDF</a>' % self.get_media('pdf').url)
 407             if self.has_media("mobi"):
 408                 formats.append(u'<a href="%s">MOBI</a>' % self.get_media('mobi').url)
 409             if self.root_ancestor.has_media("epub"):
 410                 formats.append(u'<a href="%s">EPUB</a>' % self.root_ancestor.get_media('epub').url)
 411             if self.has_media("txt"):
 412                 formats.append(u'<a href="%s">TXT</a>' % self.get_media('txt').url)
 413             # other files
 414             for m in self.media.order_by('type'):
 415                 formats.append(u'<a href="%s">%s</a>' % (m.file.url, m.type.upper()))
 416
 417             formats = [mark_safe(format) for format in formats]
 418
 419             short_html = unicode(render_to_string('catalogue/book_short.html',
 420                 {'book': self, 'tags': tags, 'formats': formats}))
 421
 422             if self.id:
 423                 cache.set(cache_key, short_html, CACHE_FOREVER)
 424             return mark_safe(short_html)
 425
 426     @property
 427     def root_ancestor(self):
 428         """ returns the oldest ancestor """
 429
 430         if not hasattr(self, '_root_ancestor'):
 431             book = self
 432             while book.parent:
 433                 book = book.parent
 434             self._root_ancestor = book
 435         return self._root_ancestor
 436
 437
 438     def has_description(self):
 439         return len(self.description) > 0
 440     has_description.short_description = _('description')
 441     has_description.boolean = True
 442
 443     # ugly ugly ugly
 444     def has_odt_file(self):
 445         return bool(self.has_media("odt"))
 446     has_odt_file.short_description = 'ODT'
 447     has_odt_file.boolean = True
 448
 449     def has_mp3_file(self):
 450         return bool(self.has_media("mp3"))
 451     has_mp3_file.short_description = 'MP3'
 452     has_mp3_file.boolean = True
 453
 454     def has_ogg_file(self):
 455         return bool(self.has_media("ogg"))
 456     has_ogg_file.short_description = 'OGG'
 457     has_ogg_file.boolean = True
 458
 459     def has_daisy_file(self):
 460         return bool(self.has_media("daisy"))
 461     has_daisy_file.short_description = 'DAISY'
 462     has_daisy_file.boolean = True
 463
 464     def build_pdf(self):
 465         """ (Re)builds the pdf file.
 466
 467         """
 468         from librarian import pdf
 469         from tempfile import NamedTemporaryFile
 470         import os
 471
 472         try:
 473             pdf_file = NamedTemporaryFile(delete=False)
 474             pdf.transform(ORMDocProvider(self),
 475                       file_path=str(self.xml_file.path),
 476                       output_file=pdf_file,
 477                       )
 478
 479             self.pdf_file.save('%s.pdf' % self.slug, File(open(pdf_file.name)))
 480         finally:
 481             unlink(pdf_file.name)
 482
 483         # remove zip with all pdf files
 484         remove_zip(settings.ALL_PDF_ZIP)
 485
 486     def build_mobi(self):
 487         """ (Re)builds the MOBI file.
 488
 489         """
 490         from librarian import mobi
 491         from tempfile import NamedTemporaryFile
 492         import os
 493
 494         try:
 495             mobi_file = NamedTemporaryFile(suffix='.mobi', delete=False)
 496             mobi.transform(ORMDocProvider(self), verbose=1,
 497                       file_path=str(self.xml_file.path),
 498                       output_file=mobi_file.name,
 499                       )
 500
 501             self.mobi_file.save('%s.mobi' % self.slug, File(open(mobi_file.name)))
 502         finally:
 503             unlink(mobi_file.name)
 504
 505         # remove zip with all mobi files
 506         remove_zip(settings.ALL_MOBI_ZIP)
 507
 508     def build_epub(self, remove_descendants=True):
 509         """ (Re)builds the epub file.
 510             If book has a parent, does nothing.
 511             Unless remove_descendants is False, descendants' epubs are removed.
 512         """
 513         from StringIO import StringIO
 514         from hashlib import sha1
 515         from django.core.files.base import ContentFile
 516
 517         if self.parent:
 518             # don't need an epub
 519             return
 520
 521         epub_file = StringIO()
 522         try:
 523             epub.transform(ORMDocProvider(self), self.slug, output_file=epub_file)
 524             self.epub_file.save('%s.epub' % self.slug, ContentFile(epub_file.getvalue()))
 525             FileRecord(slug=self.slug, type='epub', sha1=sha1(epub_file.getvalue()).hexdigest()).save()
 526         except NoDublinCore:
 527             pass
 528
 529         book_descendants = list(self.children.all())
 530         while len(book_descendants) > 0:
 531             child_book = book_descendants.pop(0)
 532             if remove_descendants and child_book.has_epub_file():
 533                 child_book.epub_file.delete()
 534             # save anyway, to refresh short_html
 535             child_book.save()
 536             book_descendants += list(child_book.children.all())
 537
 538         # remove zip package with all epub files
 539         remove_zip(settings.ALL_EPUB_ZIP)
 540
 541     def build_txt(self):
 542         from StringIO import StringIO
 543         from django.core.files.base import ContentFile
 544         from librarian import text
 545
 546         out = StringIO()
 547         text.transform(open(self.xml_file.path), out)
 548         self.txt_file.save('%s.txt' % self.slug, ContentFile(out.getvalue()))
 549
 550
 551     def build_html(self):
 552         from tempfile import NamedTemporaryFile
 553         from markupstring import MarkupString
 554
 555         meta_tags = list(self.tags.filter(
 556             category__in=('author', 'epoch', 'genre', 'kind')))
 557         book_tag = self.book_tag()
 558
 559         html_file = NamedTemporaryFile()
 560         if html.transform(self.xml_file.path, html_file, parse_dublincore=False):
 561             self.html_file.save('%s.html' % self.slug, File(html_file))
 562
 563             # get ancestor l-tags for adding to new fragments
 564             ancestor_tags = []
 565             p = self.parent
 566             while p:
 567                 ancestor_tags.append(p.book_tag())
 568                 p = p.parent
 569
 570             # Delete old fragments and create them from scratch
 571             self.fragments.all().delete()
 572             # Extract fragments
 573             closed_fragments, open_fragments = html.extract_fragments(self.html_file.path)
 574             for fragment in closed_fragments.values():
 575                 try:
 576                     theme_names = [s.strip() for s in fragment.themes.split(',')]
 577                 except AttributeError:
 578                     continue
 579                 themes = []
 580                 for theme_name in theme_names:
 581                     if not theme_name:
 582                         continue
 583                     tag, created = Tag.objects.get_or_create(slug=slughifi(theme_name), category='theme')
 584                     if created:
 585                         tag.name = theme_name
 586                         tag.sort_key = theme_name.lower()
 587                         tag.save()
 588                     themes.append(tag)
 589                 if not themes:
 590                     continue
 591
 592                 text = fragment.to_string()
 593                 short_text = ''
 594                 if (len(MarkupString(text)) > 240):
 595                     short_text = unicode(MarkupString(text)[:160])
 596                 new_fragment = Fragment.objects.create(anchor=fragment.id, book=self,
 597                     text=text, short_text=short_text)
 598
 599                 new_fragment.save()
 600                 new_fragment.tags = set(meta_tags + themes + [book_tag] + ancestor_tags)
 601             self.save()
 602             self.html_built.send(sender=self)
 603             return True
 604         return False
 605
 606     @staticmethod
 607     def zip_format(format_):
 608         def pretty_file_name(book):
 609             return "%s/%s.%s" % (
 610                 b.get_extra_info_value()['author'],
 611                 b.slug,
 612                 format_)
 613
 614         field_name = "%s_file" % format_
 615         books = Book.objects.filter(parent=None).exclude(**{field_name: ""})
 616         paths = [(pretty_file_name(b), getattr(b, field_name).path)
 617                     for b in books]
 618         result = create_zip.delay(paths,
 619                     getattr(settings, "ALL_%s_ZIP" % format_.upper()))
 620         return result.wait()
 621
 622     def zip_audiobooks(self):
 623         bm = BookMedia.objects.filter(book=self, type='mp3')
 624         paths = map(lambda bm: (None, bm.file.path), bm)
 625         result = create_zip.delay(paths, self.slug)
 626         return result.wait()
 627
 628     @classmethod
 629     def from_xml_file(cls, xml_file, **kwargs):
 630         # use librarian to parse meta-data
 631         book_info = dcparser.parse(xml_file)
 632
 633         if not isinstance(xml_file, File):
 634             xml_file = File(open(xml_file))
 635
 636         try:
 637             return cls.from_text_and_meta(xml_file, book_info, **kwargs)
 638         finally:
 639             xml_file.close()
 640
 641     @classmethod
 642     def from_text_and_meta(cls, raw_file, book_info, overwrite=False,
 643             build_epub=True, build_txt=True, build_pdf=True, build_mobi=True):
 644         import re
 645
 646         # check for parts before we do anything
 647         children = []
 648         if hasattr(book_info, 'parts'):
 649             for part_url in book_info.parts:
 650                 base, slug = part_url.rsplit('/', 1)
 651                 try:
 652                     children.append(Book.objects.get(slug=slug))
 653                 except Book.DoesNotExist, e:
 654                     raise Book.DoesNotExist(_('Book with slug = "%s" does not exist.') % slug)
 655
 656
 657         # Read book metadata
 658         book_base, book_slug = book_info.url.rsplit('/', 1)
 659         if re.search(r'[^a-zA-Z0-9-]', book_slug):
 660             raise ValueError('Invalid characters in slug')
 661         book, created = Book.objects.get_or_create(slug=book_slug)
 662
 663         if created:
 664             book_shelves = []
 665         else:
 666             if not overwrite:
 667                 raise Book.AlreadyExists(_('Book %s already exists') % book_slug)
 668             # Save shelves for this book
 669             book_shelves = list(book.tags.filter(category='set'))
 670
 671         book.title = book_info.title
 672         book.set_extra_info_value(book_info.to_dict())
 673         book.save()
 674
 675         meta_tags = []
 676         categories = (('kinds', 'kind'), ('genres', 'genre'), ('authors', 'author'), ('epochs', 'epoch'))
 677         for field_name, category in categories:
 678             try:
 679                 tag_names = getattr(book_info, field_name)
 680             except:
 681                 tag_names = [getattr(book_info, category)]
 682             for tag_name in tag_names:
 683                 tag_sort_key = tag_name
 684                 if category == 'author':
 685                     tag_sort_key = tag_name.last_name
 686                     tag_name = ' '.join(tag_name.first_names) + ' ' + tag_name.last_name
 687                 tag, created = Tag.objects.get_or_create(slug=slughifi(tag_name), category=category)
 688                 if created:
 689                     tag.name = tag_name
 690                     tag.sort_key = sortify(tag_sort_key.lower())
 691                     tag.save()
 692                 meta_tags.append(tag)
 693
 694         book.tags = set(meta_tags + book_shelves)
 695
 696         book_tag = book.book_tag()
 697
 698         for n, child_book in enumerate(children):
 699             child_book.parent = book
 700             child_book.parent_number = n
 701             child_book.save()
 702
 703         # Save XML and HTML files
 704         book.xml_file.save('%s.xml' % book.slug, raw_file, save=False)
 705
 706         # delete old fragments when overwriting
 707         book.fragments.all().delete()
 708
 709         if book.build_html():
 710             if not settings.NO_BUILD_TXT and build_txt:
 711                 book.build_txt()
 712
 713         if not settings.NO_BUILD_EPUB and build_epub:
 714             book.root_ancestor.build_epub()
 715
 716         if not settings.NO_BUILD_PDF and build_pdf:
 717             book.root_ancestor.build_pdf()
 718
 719         if not settings.NO_BUILD_MOBI and build_mobi:
 720             book.build_mobi()
 721
 722         book_descendants = list(book.children.all())
 723         # add l-tag to descendants and their fragments
 724         # delete unnecessary EPUB files
 725         while len(book_descendants) > 0:
 726             child_book = book_descendants.pop(0)
 727             child_book.tags = list(child_book.tags) + [book_tag]
 728             child_book.save()
 729             for fragment in child_book.fragments.all():
 730                 fragment.tags = set(list(fragment.tags) + [book_tag])
 731             book_descendants += list(child_book.children.all())
 732
 733         book.save()
 734
 735         # refresh cache
 736         book.reset_tag_counter()
 737         book.reset_theme_counter()
 738
 739         cls.published.send(sender=book)
 740         return book
 741
 742     def reset_tag_counter(self):
 743         if self.id is None:
 744             return
 745
 746         cache_key = "Book.tag_counter/%d" % self.id
 747         cache.delete(cache_key)
 748         if self.parent:
 749             self.parent.reset_tag_counter()
 750
 751     @property
 752     def tag_counter(self):
 753         if self.id:
 754             cache_key = "Book.tag_counter/%d" % self.id
 755             tags = cache.get(cache_key)
 756         else:
 757             tags = None
 758
 759         if tags is None:
 760             tags = {}
 761             for child in self.children.all().order_by():
 762                 for tag_pk, value in child.tag_counter.iteritems():
 763                     tags[tag_pk] = tags.get(tag_pk, 0) + value
 764             for tag in self.tags.exclude(category__in=('book', 'theme', 'set')).order_by():
 765                 tags[tag.pk] = 1
 766
 767             if self.id:
 768                 cache.set(cache_key, tags, CACHE_FOREVER)
 769         return tags
 770
 771     def reset_theme_counter(self):
 772         if self.id is None:
 773             return
 774
 775         cache_key = "Book.theme_counter/%d" % self.id
 776         cache.delete(cache_key)
 777         if self.parent:
 778             self.parent.reset_theme_counter()
 779
 780     @property
 781     def theme_counter(self):
 782         if self.id:
 783             cache_key = "Book.theme_counter/%d" % self.id
 784             tags = cache.get(cache_key)
 785         else:
 786             tags = None
 787
 788         if tags is None:
 789             tags = {}
 790             for fragment in Fragment.tagged.with_any([self.book_tag()]).order_by():
 791                 for tag in fragment.tags.filter(category='theme').order_by():
 792                     tags[tag.pk] = tags.get(tag.pk, 0) + 1
 793
 794             if self.id:
 795                 cache.set(cache_key, tags, CACHE_FOREVER)
 796         return tags
 797
 798     def pretty_title(self, html_links=False):
 799         book = self
 800         names = list(book.tags.filter(category='author'))
 801
 802         books = []
 803         while book:
 804             books.append(book)
 805             book = book.parent
 806         names.extend(reversed(books))
 807
 808         if html_links:
 809             names = ['<a href="%s">%s</a>' % (tag.get_absolute_url(), tag.name) for tag in names]
 810         else:
 811             names = [tag.name for tag in names]
 812
 813         return ', '.join(names)
 814
 815     @classmethod
 816     def tagged_top_level(cls, tags):
 817         """ Returns top-level books tagged with `tags'.
 818
 819         It only returns those books which don't have ancestors which are
 820         also tagged with those tags.
 821
 822         """
 823         # get relevant books and their tags
 824         objects = cls.tagged.with_all(tags)
 825         # eliminate descendants
 826         l_tags = Tag.objects.filter(category='book', slug__in=[book.book_tag_slug() for book in objects])
 827         descendants_keys = [book.pk for book in cls.tagged.with_any(l_tags)]
 828         if descendants_keys:
 829             objects = objects.exclude(pk__in=descendants_keys)
 830
 831         return objects
 832
 833     @classmethod
 834     def book_list(cls, filter=None):
 835         """Generates a hierarchical listing of all books.
 836
 837         Books are optionally filtered with a test function.
 838
 839         """
 840
 841         books_by_parent = {}
 842         books = cls.objects.all().order_by('parent_number', 'sort_key').only('title', 'parent', 'slug')
 843         if filter:
 844             books = books.filter(filter).distinct()
 845             book_ids = set((book.pk for book in books))
 846             for book in books:
 847                 parent = book.parent_id
 848                 if parent not in book_ids:
 849                     parent = None
 850                 books_by_parent.setdefault(parent, []).append(book)
 851         else:
 852             for book in books:
 853                 books_by_parent.setdefault(book.parent_id, []).append(book)
 854
 855         orphans = []
 856         books_by_author = SortedDict()
 857         for tag in Tag.objects.filter(category='author'):
 858             books_by_author[tag] = []
 859
 860         for book in books_by_parent.get(None,()):
 861             authors = list(book.tags.filter(category='author'))
 862             if authors:
 863                 for author in authors:
 864                     books_by_author[author].append(book)
 865             else:
 866                 orphans.append(book)
 867
 868         return books_by_author, orphans, books_by_parent
 869
 870
 871 def _has_factory(ftype):
 872     has = lambda self: bool(getattr(self, "%s_file" % ftype))
 873     has.short_description = t.upper()
 874     has.boolean = True
 875     has.__name__ = "has_%s_file" % ftype
 876     return has
 877
 878
 879 # add the file fields
 880 for t in Book.file_types:
 881     field_name = "%s_file" % t
 882     models.FileField(_("%s file" % t.upper()),
 883             upload_to=book_upload_path(t),
 884             blank=True).contribute_to_class(Book, field_name)
 885
 886     setattr(Book, "has_%s_file" % t, _has_factory(t))
 887
 888
 889 class Fragment(models.Model):
 890     text = models.TextField()
 891     short_text = models.TextField(editable=False)
 892     anchor = models.CharField(max_length=120)
 893     book = models.ForeignKey(Book, related_name='fragments')
 894
 895     objects = models.Manager()
 896     tagged = managers.ModelTaggedItemManager(Tag)
 897     tags = managers.TagDescriptor(Tag)
 898
 899     class Meta:
 900         ordering = ('book', 'anchor',)
 901         verbose_name = _('fragment')
 902         verbose_name_plural = _('fragments')
 903
 904     def get_absolute_url(self):
 905         return '%s#m%s' % (reverse('book_text', kwargs={'slug': self.book.slug}), self.anchor)
 906
 907     def reset_short_html(self):
 908         if self.id is None:
 909             return
 910
 911         cache_key = "Fragment.short_html/%d/%s"
 912         for lang, langname in settings.LANGUAGES:
 913             cache.delete(cache_key % (self.id, lang))
 914
 915     def short_html(self):
 916         if self.id:
 917             cache_key = "Fragment.short_html/%d/%s" % (self.id, get_language())
 918             short_html = cache.get(cache_key)
 919         else:
 920             short_html = None
 921
 922         if short_html is not None:
 923             return mark_safe(short_html)
 924         else:
 925             short_html = unicode(render_to_string('catalogue/fragment_short.html',
 926                 {'fragment': self}))
 927             if self.id:
 928                 cache.set(cache_key, short_html, CACHE_FOREVER)
 929             return mark_safe(short_html)
 930
 931
 932 class FileRecord(models.Model):
 933     slug = models.SlugField(_('slug'), max_length=120, db_index=True)
 934     type = models.CharField(_('type'), max_length=20, db_index=True)
 935     sha1 = models.CharField(_('sha-1 hash'), max_length=40)
 936     time = models.DateTimeField(_('time'), auto_now_add=True)
 937
 938     class Meta:
 939         ordering = ('-time','-slug', '-type')
 940         verbose_name = _('file record')
 941         verbose_name_plural = _('file records')
 942
 943     def __unicode__(self):
 944         return "%s %s.%s" % (self.sha1,  self.slug, self.type)
 945
 946 ###########
 947 #
 948 # SIGNALS
 949 #
 950 ###########
 951
 952
 953 def _tags_updated_handler(sender, affected_tags, **kwargs):
 954     # reset tag global counter
 955     # we want Tag.changed_at updated for API to know the tag was touched
 956     Tag.objects.filter(pk__in=[tag.pk for tag in affected_tags]).update(book_count=None, changed_at=datetime.now())
 957
 958     # if book tags changed, reset book tag counter
 959     if isinstance(sender, Book) and \
 960                 Tag.objects.filter(pk__in=(tag.pk for tag in affected_tags)).\
 961                     exclude(category__in=('book', 'theme', 'set')).count():
 962         sender.reset_tag_counter()
 963     # if fragment theme changed, reset book theme counter
 964     elif isinstance(sender, Fragment) and \
 965                 Tag.objects.filter(pk__in=(tag.pk for tag in affected_tags)).\
 966                     filter(category='theme').count():
 967         sender.book.reset_theme_counter()
 968 tags_updated.connect(_tags_updated_handler)
 969
 970
 971 def _pre_delete_handler(sender, instance, **kwargs):
 972     """ refresh Book on BookMedia delete """
 973     if sender == BookMedia:
 974         instance.book.save()
 975 pre_delete.connect(_pre_delete_handler)
 976
 977 def _post_save_handler(sender, instance, **kwargs):
 978     """ refresh all the short_html stuff on BookMedia update """
 979     if sender == BookMedia:
 980         instance.book.save()
 981 post_save.connect(_post_save_handler)