apps/catalogue/models.py

   1 # -*- coding: utf-8 -*-
   2 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
   3 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
   4 #
   5 from datetime import datetime
   6
   7 from django.db import models
   8 from django.db.models import permalink, Q
   9 from django.core.cache import cache
  10 from django.utils.translation import ugettext_lazy as _
  11 from django.contrib.auth.models import User
  12 from django.core.files import File
  13 from django.template.loader import render_to_string
  14 from django.utils.safestring import mark_safe
  15 from django.utils.translation import get_language
  16 from django.core.urlresolvers import reverse
  17 from django.db.models.signals import post_save, m2m_changed, pre_delete
  18
  19 from django.conf import settings
  20
  21 from newtagging.models import TagBase, tags_updated
  22 from newtagging import managers
  23 from catalogue.fields import JSONField, OverwritingFileField
  24 from catalogue.utils import ExistingFile
  25
  26 from librarian import dcparser, html, epub, NoDublinCore
  27 import mutagen
  28 from mutagen import id3
  29 from slughifi import slughifi
  30 from sortify import sortify
  31
  32
  33 TAG_CATEGORIES = (
  34     ('author', _('author')),
  35     ('epoch', _('epoch')),
  36     ('kind', _('kind')),
  37     ('genre', _('genre')),
  38     ('theme', _('theme')),
  39     ('set', _('set')),
  40     ('book', _('book')),
  41 )
  42
  43 MEDIA_FORMATS = (
  44     ('odt', _('ODT file')),
  45     ('mp3', _('MP3 file')),
  46     ('ogg', _('OGG file')),
  47     ('daisy', _('DAISY file')),
  48 )
  49
  50 class TagSubcategoryManager(models.Manager):
  51     def __init__(self, subcategory):
  52         super(TagSubcategoryManager, self).__init__()
  53         self.subcategory = subcategory
  54
  55     def get_query_set(self):
  56         return super(TagSubcategoryManager, self).get_query_set().filter(category=self.subcategory)
  57
  58
  59 class Tag(TagBase):
  60     name = models.CharField(_('name'), max_length=50, db_index=True)
  61     slug = models.SlugField(_('slug'), max_length=120, db_index=True)
  62     sort_key = models.CharField(_('sort key'), max_length=120, db_index=True)
  63     category = models.CharField(_('category'), max_length=50, blank=False, null=False,
  64         db_index=True, choices=TAG_CATEGORIES)
  65     description = models.TextField(_('description'), blank=True)
  66     main_page = models.BooleanField(_('main page'), default=False, db_index=True, help_text=_('Show tag on main page'))
  67
  68     user = models.ForeignKey(User, blank=True, null=True)
  69     book_count = models.IntegerField(_('book count'), blank=True, null=True)
  70     gazeta_link = models.CharField(blank=True, max_length=240)
  71     wiki_link = models.CharField(blank=True, max_length=240)
  72
  73     created_at    = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True)
  74     changed_at    = models.DateTimeField(_('creation date'), auto_now=True, db_index=True)
  75
  76     class UrlDeprecationWarning(DeprecationWarning):
  77         pass
  78
  79     categories_rev = {
  80         'autor': 'author',
  81         'epoka': 'epoch',
  82         'rodzaj': 'kind',
  83         'gatunek': 'genre',
  84         'motyw': 'theme',
  85         'polka': 'set',
  86     }
  87     categories_dict = dict((item[::-1] for item in categories_rev.iteritems()))
  88
  89     class Meta:
  90         ordering = ('sort_key',)
  91         verbose_name = _('tag')
  92         verbose_name_plural = _('tags')
  93         unique_together = (("slug", "category"),)
  94
  95     def __unicode__(self):
  96         return self.name
  97
  98     def __repr__(self):
  99         return "Tag(slug=%r)" % self.slug
 100
 101     @permalink
 102     def get_absolute_url(self):
 103         return ('catalogue.views.tagged_object_list', [self.url_chunk])
 104
 105     def has_description(self):
 106         return len(self.description) > 0
 107     has_description.short_description = _('description')
 108     has_description.boolean = True
 109
 110     def get_count(self):
 111         """ returns global book count for book tags, fragment count for themes """
 112
 113         if self.book_count is None:
 114             if self.category == 'book':
 115                 # never used
 116                 objects = Book.objects.none()
 117             elif self.category == 'theme':
 118                 objects = Fragment.tagged.with_all((self,))
 119             else:
 120                 objects = Book.tagged.with_all((self,)).order_by()
 121                 if self.category != 'set':
 122                     # eliminate descendants
 123                     l_tags = Tag.objects.filter(slug__in=[book.book_tag_slug() for book in objects])
 124                     descendants_keys = [book.pk for book in Book.tagged.with_any(l_tags)]
 125                     if descendants_keys:
 126                         objects = objects.exclude(pk__in=descendants_keys)
 127             self.book_count = objects.count()
 128             self.save()
 129         return self.book_count
 130
 131     @staticmethod
 132     def get_tag_list(tags):
 133         if isinstance(tags, basestring):
 134             real_tags = []
 135             ambiguous_slugs = []
 136             category = None
 137             deprecated = False
 138             tags_splitted = tags.split('/')
 139             for name in tags_splitted:
 140                 if category:
 141                     real_tags.append(Tag.objects.get(slug=name, category=category))
 142                     category = None
 143                 elif name in Tag.categories_rev:
 144                     category = Tag.categories_rev[name]
 145                 else:
 146                     try:
 147                         real_tags.append(Tag.objects.exclude(category='book').get(slug=name))
 148                         deprecated = True
 149                     except Tag.MultipleObjectsReturned, e:
 150                         ambiguous_slugs.append(name)
 151
 152             if category:
 153                 # something strange left off
 154                 raise Tag.DoesNotExist()
 155             if ambiguous_slugs:
 156                 # some tags should be qualified
 157                 e = Tag.MultipleObjectsReturned()
 158                 e.tags = real_tags
 159                 e.ambiguous_slugs = ambiguous_slugs
 160                 raise e
 161             if deprecated:
 162                 e = Tag.UrlDeprecationWarning()
 163                 e.tags = real_tags
 164                 raise e
 165             return real_tags
 166         else:
 167             return TagBase.get_tag_list(tags)
 168
 169     @property
 170     def url_chunk(self):
 171         return '/'.join((Tag.categories_dict[self.category], self.slug))
 172
 173
 174 # TODO: why is this hard-coded ?
 175 def book_upload_path(ext=None, maxlen=100):
 176     def get_dynamic_path(media, filename, ext=ext):
 177         # how to put related book's slug here?
 178         if not ext:
 179             if media.type == 'daisy':
 180                 ext = 'daisy.zip'
 181             else:
 182                 ext = media.type
 183         if not media.name:
 184             name = slughifi(filename.split(".")[0])
 185         else:
 186             name = slughifi(media.name)
 187         return 'book/%s/%s.%s' % (ext, name[:maxlen-len('book/%s/.%s' % (ext, ext))-4], ext)
 188     return get_dynamic_path
 189
 190
 191 class BookMedia(models.Model):
 192     type        = models.CharField(_('type'), choices=MEDIA_FORMATS, max_length="100")
 193     name        = models.CharField(_('name'), max_length="100")
 194     file        = OverwritingFileField(_('file'), upload_to=book_upload_path())
 195     uploaded_at = models.DateTimeField(_('creation date'), auto_now_add=True, editable=False)
 196     extra_info  = JSONField(_('extra information'), default='{}', editable=False)
 197     book = models.ForeignKey('Book', related_name='media')
 198     source_sha1 = models.CharField(null=True, blank=True, max_length=40, editable=False)
 199
 200     def __unicode__(self):
 201         return "%s (%s)" % (self.name, self.file.name.split("/")[-1])
 202
 203     class Meta:
 204         ordering            = ('type', 'name')
 205         verbose_name        = _('book media')
 206         verbose_name_plural = _('book media')
 207
 208     def save(self, *args, **kwargs):
 209         try:
 210             old = BookMedia.objects.get(pk=self.pk)
 211         except BookMedia.DoesNotExist, e:
 212             pass
 213         else:
 214             # if name changed, change the file name, too
 215             if slughifi(self.name) != slughifi(old.name):
 216                 self.file.save(None, ExistingFile(self.file.path), save=False, leave=True)
 217
 218         super(BookMedia, self).save(*args, **kwargs)
 219         extra_info = self.get_extra_info_value()
 220         extra_info.update(self.read_meta())
 221         self.set_extra_info_value(extra_info)
 222         self.source_sha1 = self.read_source_sha1(self.file.path, self.type)
 223         return super(BookMedia, self).save(*args, **kwargs)
 224
 225     def read_meta(self):
 226         """
 227             Reads some metadata from the audiobook.
 228         """
 229
 230         artist_name = director_name = project = funded_by = ''
 231         if self.type == 'mp3':
 232             try:
 233                 audio = id3.ID3(self.file.path)
 234                 artist_name = ', '.join(', '.join(tag.text) for tag in audio.getall('TPE1'))
 235                 director_name = ', '.join(', '.join(tag.text) for tag in audio.getall('TPE3'))
 236                 project = ", ".join([t.data for t in audio.getall('PRIV')
 237                         if t.owner=='wolnelektury.pl?project'])
 238                 funded_by = ", ".join([t.data for t in audio.getall('PRIV')
 239                         if t.owner=='wolnelektury.pl?funded_by'])
 240             except:
 241                 pass
 242         elif self.type == 'ogg':
 243             try:
 244                 audio = mutagen.File(self.file.path)
 245                 artist_name = ', '.join(audio.get('artist', []))
 246                 director_name = ', '.join(audio.get('conductor', []))
 247                 project = ", ".join(audio.get('project', []))
 248                 funded_by = ", ".join(audio.get('funded_by', []))
 249             except:
 250                 pass
 251         else:
 252             return {}
 253         return {'artist_name': artist_name, 'director_name': director_name,
 254                 'project': project, 'funded_by': funded_by}
 255
 256     @staticmethod
 257     def read_source_sha1(filepath, filetype):
 258         """
 259             Reads source file SHA1 from audiobok metadata.
 260         """
 261
 262         if filetype == 'mp3':
 263             try:
 264                 audio = id3.ID3(filepath)
 265                 return [t.data for t in audio.getall('PRIV')
 266                         if t.owner=='wolnelektury.pl?flac_sha1'][0]
 267             except:
 268                 return None
 269         elif filetype == 'ogg':
 270             try:
 271                 audio = mutagen.File(filepath)
 272                 return audio.get('flac_sha1', [None])[0]
 273             except:
 274                 return None
 275         else:
 276             return None
 277
 278
 279 class Book(models.Model):
 280     title         = models.CharField(_('title'), max_length=120)
 281     sort_key = models.CharField(_('sort_key'), max_length=120, db_index=True, editable=False)
 282     slug          = models.SlugField(_('slug'), max_length=120, unique=True, db_index=True)
 283     description   = models.TextField(_('description'), blank=True)
 284     created_at    = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True)
 285     changed_at    = models.DateTimeField(_('creation date'), auto_now=True, db_index=True)
 286     parent_number = models.IntegerField(_('parent number'), default=0)
 287     extra_info    = JSONField(_('extra information'), default='{}')
 288     gazeta_link   = models.CharField(blank=True, max_length=240)
 289     wiki_link     = models.CharField(blank=True, max_length=240)
 290     # files generated during publication
 291     xml_file      = models.FileField(_('XML file'), upload_to=book_upload_path('xml'), blank=True)
 292     html_file     = models.FileField(_('HTML file'), upload_to=book_upload_path('html'), blank=True)
 293     pdf_file      = models.FileField(_('PDF file'), upload_to=book_upload_path('pdf'), blank=True)
 294     epub_file     = models.FileField(_('EPUB file'), upload_to=book_upload_path('epub'), blank=True)
 295     txt_file      = models.FileField(_('TXT file'), upload_to=book_upload_path('txt'), blank=True)
 296
 297     parent        = models.ForeignKey('self', blank=True, null=True, related_name='children')
 298     objects  = models.Manager()
 299     tagged   = managers.ModelTaggedItemManager(Tag)
 300     tags     = managers.TagDescriptor(Tag)
 301
 302     class AlreadyExists(Exception):
 303         pass
 304
 305     class Meta:
 306         ordering = ('sort_key',)
 307         verbose_name = _('book')
 308         verbose_name_plural = _('books')
 309
 310     def __unicode__(self):
 311         return self.title
 312
 313     def save(self, force_insert=False, force_update=False, reset_short_html=True, **kwargs):
 314         self.sort_key = sortify(self.title)
 315
 316         if reset_short_html:
 317             self.reset_short_html()
 318
 319         return super(Book, self).save(force_insert, force_update)
 320
 321     @permalink
 322     def get_absolute_url(self):
 323         return ('catalogue.views.book_detail', [self.slug])
 324
 325     @property
 326     def name(self):
 327         return self.title
 328
 329     def book_tag_slug(self):
 330         return ('l-' + self.slug)[:120]
 331
 332     def book_tag(self):
 333         slug = self.book_tag_slug()
 334         book_tag, created = Tag.objects.get_or_create(slug=slug, category='book')
 335         if created:
 336             book_tag.name = self.title[:50]
 337             book_tag.sort_key = self.title.lower()
 338             book_tag.save()
 339         return book_tag
 340
 341     def has_media(self, type):
 342         if   type == 'xml':
 343             if self.xml_file:
 344                 return True
 345             else:
 346                 return False
 347         elif type == 'html':
 348             if self.html_file:
 349                 return True
 350             else:
 351                 return False
 352         elif type == 'txt':
 353             if self.txt_file:
 354                 return True
 355             else:
 356                 return False
 357         elif type == 'pdf':
 358             if self.pdf_file:
 359                 return True
 360             else:
 361                 return False
 362         elif type == 'epub':
 363             if self.epub_file:
 364                 return True
 365             else:
 366                 return False
 367         else:
 368             if self.media.filter(type=type).exists():
 369                 return True
 370             else:
 371                 return False
 372
 373     def get_media(self, type):
 374         if self.has_media(type):
 375             if   type == "xml":
 376                 return self.xml_file
 377             elif type == "html":
 378                 return self.html_file
 379             elif type == "epub":
 380                 return self.epub_file
 381             elif type == "txt":
 382                 return self.txt_file
 383             elif type == "pdf":
 384                 return self.pdf_file
 385             else:
 386                 return self.media.filter(type=type)
 387         else:
 388             return None
 389
 390     def get_mp3(self):
 391         return self.get_media("mp3")
 392     def get_odt(self):
 393         return self.get_media("odt")
 394     def get_ogg(self):
 395         return self.get_media("ogg")
 396     def get_daisy(self):
 397         return self.get_media("daisy")
 398
 399     def reset_short_html(self):
 400         if self.id is None:
 401             return
 402
 403         cache_key = "Book.short_html/%d/%s"
 404         for lang, langname in settings.LANGUAGES:
 405             cache.delete(cache_key % (self.id, lang))
 406         # Fragment.short_html relies on book's tags, so reset it here too
 407         for fragm in self.fragments.all():
 408             fragm.reset_short_html()
 409
 410     def short_html(self):
 411         if self.id:
 412             cache_key = "Book.short_html/%d/%s" % (self.id, get_language())
 413             short_html = cache.get(cache_key)
 414         else:
 415             short_html = None
 416
 417         if short_html is not None:
 418             return mark_safe(short_html)
 419         else:
 420             tags = self.tags.filter(~Q(category__in=('set', 'theme', 'book')))
 421             tags = [mark_safe(u'<a href="%s">%s</a>' % (tag.get_absolute_url(), tag.name)) for tag in tags]
 422
 423             formats = []
 424             # files generated during publication
 425             if self.has_media("html"):
 426                 formats.append(u'<a href="%s">%s</a>' % (reverse('book_text', kwargs={'slug': self.slug}), _('Read online')))
 427             if self.has_media("pdf"):
 428                 formats.append(u'<a href="%s">PDF</a>' % self.get_media('pdf').url)
 429             if self.root_ancestor.has_media("epub"):
 430                 formats.append(u'<a href="%s">EPUB</a>' % self.root_ancestor.get_media('epub').url)
 431             if self.has_media("txt"):
 432                 formats.append(u'<a href="%s">TXT</a>' % self.get_media('txt').url)
 433             # other files
 434             for m in self.media.order_by('type'):
 435                 formats.append(u'<a href="%s">%s</a>' % (m.file.url, m.type.upper()))
 436
 437             formats = [mark_safe(format) for format in formats]
 438
 439             short_html = unicode(render_to_string('catalogue/book_short.html',
 440                 {'book': self, 'tags': tags, 'formats': formats}))
 441
 442             if self.id:
 443                 cache.set(cache_key, short_html)
 444             return mark_safe(short_html)
 445
 446     @property
 447     def root_ancestor(self):
 448         """ returns the oldest ancestor """
 449
 450         if not hasattr(self, '_root_ancestor'):
 451             book = self
 452             while book.parent:
 453                 book = book.parent
 454             self._root_ancestor = book
 455         return self._root_ancestor
 456
 457
 458     def has_description(self):
 459         return len(self.description) > 0
 460     has_description.short_description = _('description')
 461     has_description.boolean = True
 462
 463     # ugly ugly ugly
 464     def has_pdf_file(self):
 465         return bool(self.pdf_file)
 466     has_pdf_file.short_description = 'PDF'
 467     has_pdf_file.boolean = True
 468
 469     def has_epub_file(self):
 470         return bool(self.epub_file)
 471     has_epub_file.short_description = 'EPUB'
 472     has_epub_file.boolean = True
 473
 474     def has_txt_file(self):
 475         return bool(self.txt_file)
 476     has_txt_file.short_description = 'HTML'
 477     has_txt_file.boolean = True
 478
 479     def has_html_file(self):
 480         return bool(self.html_file)
 481     has_html_file.short_description = 'HTML'
 482     has_html_file.boolean = True
 483
 484     def has_odt_file(self):
 485         return bool(self.has_media("odt"))
 486     has_odt_file.short_description = 'ODT'
 487     has_odt_file.boolean = True
 488
 489     def has_mp3_file(self):
 490         return bool(self.has_media("mp3"))
 491     has_mp3_file.short_description = 'MP3'
 492     has_mp3_file.boolean = True
 493
 494     def has_ogg_file(self):
 495         return bool(self.has_media("ogg"))
 496     has_ogg_file.short_description = 'OGG'
 497     has_ogg_file.boolean = True
 498
 499     def has_daisy_file(self):
 500         return bool(self.has_media("daisy"))
 501     has_daisy_file.short_description = 'DAISY'
 502     has_daisy_file.boolean = True
 503
 504     def build_epub(self, remove_descendants=True):
 505         """ (Re)builds the epub file.
 506             If book has a parent, does nothing.
 507             Unless remove_descendants is False, descendants' epubs are removed.
 508         """
 509
 510         from StringIO import StringIO
 511         from hashlib import sha1
 512         from django.core.files.base import ContentFile
 513         from librarian import DocProvider
 514
 515         class BookImportDocProvider(DocProvider):
 516             """ used for joined EPUBs """
 517
 518             def __init__(self, book):
 519                 self.book = book
 520
 521             def by_slug(self, slug):
 522                 if slug == self.book.slug:
 523                     return self.book.xml_file
 524                 else:
 525                     return Book.objects.get(slug=slug).xml_file
 526
 527         if self.parent:
 528             # don't need an epub
 529             return
 530
 531         epub_file = StringIO()
 532         try:
 533             epub.transform(BookImportDocProvider(self), self.slug, output_file=epub_file)
 534             self.epub_file.save('%s.epub' % self.slug, ContentFile(epub_file.getvalue()))
 535             FileRecord(slug=self.slug, type='epub', sha1=sha1(epub_file.getvalue()).hexdigest()).save()
 536         except NoDublinCore:
 537             pass
 538
 539         book_descendants = list(self.children.all())
 540         while len(book_descendants) > 0:
 541             child_book = book_descendants.pop(0)
 542             if remove_descendants and child_book.has_epub_file():
 543                 child_book.epub_file.delete()
 544             # save anyway, to refresh short_html
 545             child_book.save()
 546             book_descendants += list(child_book.children.all())
 547
 548     def build_txt(self):
 549         from StringIO import StringIO
 550         from django.core.files.base import ContentFile
 551         from librarian import text
 552
 553         out = StringIO()
 554         text.transform(open(self.xml_file.path), out)
 555         self.txt_file.save('%s.txt' % self.slug, ContentFile(out.getvalue()))
 556
 557
 558     def build_html(self):
 559         from tempfile import NamedTemporaryFile
 560         from markupstring import MarkupString
 561
 562         meta_tags = list(self.tags.filter(
 563             category__in=('author', 'epoch', 'genre', 'kind')))
 564         book_tag = self.book_tag()
 565
 566         html_file = NamedTemporaryFile()
 567         if html.transform(self.xml_file.path, html_file, parse_dublincore=False):
 568             self.html_file.save('%s.html' % self.slug, File(html_file))
 569
 570             # get ancestor l-tags for adding to new fragments
 571             ancestor_tags = []
 572             p = self.parent
 573             while p:
 574                 ancestor_tags.append(p.book_tag())
 575                 p = p.parent
 576
 577             # Delete old fragments and create them from scratch
 578             self.fragments.all().delete()
 579             # Extract fragments
 580             closed_fragments, open_fragments = html.extract_fragments(self.html_file.path)
 581             for fragment in closed_fragments.values():
 582                 try:
 583                     theme_names = [s.strip() for s in fragment.themes.split(',')]
 584                 except AttributeError:
 585                     continue
 586                 themes = []
 587                 for theme_name in theme_names:
 588                     if not theme_name:
 589                         continue
 590                     tag, created = Tag.objects.get_or_create(slug=slughifi(theme_name), category='theme')
 591                     if created:
 592                         tag.name = theme_name
 593                         tag.sort_key = theme_name.lower()
 594                         tag.save()
 595                     themes.append(tag)
 596                 if not themes:
 597                     continue
 598
 599                 text = fragment.to_string()
 600                 short_text = ''
 601                 if (len(MarkupString(text)) > 240):
 602                     short_text = unicode(MarkupString(text)[:160])
 603                 new_fragment = Fragment.objects.create(anchor=fragment.id, book=self,
 604                     text=text, short_text=short_text)
 605
 606                 new_fragment.save()
 607                 new_fragment.tags = set(meta_tags + themes + [book_tag] + ancestor_tags)
 608             self.save()
 609             return True
 610         return False
 611
 612
 613     @classmethod
 614     def from_xml_file(cls, xml_file, **kwargs):
 615         # use librarian to parse meta-data
 616         book_info = dcparser.parse(xml_file)
 617
 618         if not isinstance(xml_file, File):
 619             xml_file = File(open(xml_file))
 620
 621         try:
 622             return cls.from_text_and_meta(xml_file, book_info, **kwargs)
 623         finally:
 624             xml_file.close()
 625
 626     @classmethod
 627     def from_text_and_meta(cls, raw_file, book_info, overwrite=False, build_epub=True, build_txt=True):
 628         import re
 629
 630         # check for parts before we do anything
 631         children = []
 632         if hasattr(book_info, 'parts'):
 633             for part_url in book_info.parts:
 634                 base, slug = part_url.rsplit('/', 1)
 635                 try:
 636                     children.append(Book.objects.get(slug=slug))
 637                 except Book.DoesNotExist, e:
 638                     raise Book.DoesNotExist(_('Book with slug = "%s" does not exist.') % slug)
 639
 640
 641         # Read book metadata
 642         book_base, book_slug = book_info.url.rsplit('/', 1)
 643         if re.search(r'[^a-zA-Z0-9-]', book_slug):
 644             raise ValueError('Invalid characters in slug')
 645         book, created = Book.objects.get_or_create(slug=book_slug)
 646
 647         if created:
 648             book_shelves = []
 649         else:
 650             if not overwrite:
 651                 raise Book.AlreadyExists(_('Book %s already exists') % book_slug)
 652             # Save shelves for this book
 653             book_shelves = list(book.tags.filter(category='set'))
 654
 655         book.title = book_info.title
 656         book.set_extra_info_value(book_info.to_dict())
 657         book.save()
 658
 659         meta_tags = []
 660         categories = (('kinds', 'kind'), ('genres', 'genre'), ('authors', 'author'), ('epochs', 'epoch'))
 661         for field_name, category in categories:
 662             try:
 663                 tag_names = getattr(book_info, field_name)
 664             except:
 665                 tag_names = [getattr(book_info, category)]
 666             for tag_name in tag_names:
 667                 tag_sort_key = tag_name
 668                 if category == 'author':
 669                     tag_sort_key = tag_name.last_name
 670                     tag_name = ' '.join(tag_name.first_names) + ' ' + tag_name.last_name
 671                 tag, created = Tag.objects.get_or_create(slug=slughifi(tag_name), category=category)
 672                 if created:
 673                     tag.name = tag_name
 674                     tag.sort_key = sortify(tag_sort_key.lower())
 675                     tag.save()
 676                 meta_tags.append(tag)
 677
 678         book.tags = set(meta_tags + book_shelves)
 679
 680         book_tag = book.book_tag()
 681
 682         for n, child_book in enumerate(children):
 683             child_book.parent = book
 684             child_book.parent_number = n
 685             child_book.save()
 686
 687         # Save XML and HTML files
 688         book.xml_file.save('%s.xml' % book.slug, raw_file, save=False)
 689
 690         # delete old fragments when overwriting
 691         book.fragments.all().delete()
 692
 693         if book.build_html():
 694             if not settings.NO_BUILD_TXT and build_txt:
 695                 book.build_txt()
 696
 697         if not settings.NO_BUILD_EPUB and build_epub:
 698             book.root_ancestor.build_epub()
 699
 700         book_descendants = list(book.children.all())
 701         # add l-tag to descendants and their fragments
 702         # delete unnecessary EPUB files
 703         while len(book_descendants) > 0:
 704             child_book = book_descendants.pop(0)
 705             child_book.tags = list(child_book.tags) + [book_tag]
 706             child_book.save()
 707             for fragment in child_book.fragments.all():
 708                 fragment.tags = set(list(fragment.tags) + [book_tag])
 709             book_descendants += list(child_book.children.all())
 710
 711         # refresh cache
 712         book.reset_tag_counter()
 713         book.reset_theme_counter()
 714
 715         book.save()
 716         return book
 717
 718     def reset_tag_counter(self):
 719         if self.id is None:
 720             return
 721
 722         cache_key = "Book.tag_counter/%d" % self.id
 723         cache.delete(cache_key)
 724         if self.parent:
 725             self.parent.reset_tag_counter()
 726
 727     @property
 728     def tag_counter(self):
 729         if self.id:
 730             cache_key = "Book.tag_counter/%d" % self.id
 731             tags = cache.get(cache_key)
 732         else:
 733             tags = None
 734
 735         if tags is None:
 736             tags = {}
 737             for child in self.children.all().order_by():
 738                 for tag_pk, value in child.tag_counter.iteritems():
 739                     tags[tag_pk] = tags.get(tag_pk, 0) + value
 740             for tag in self.tags.exclude(category__in=('book', 'theme', 'set')).order_by():
 741                 tags[tag.pk] = 1
 742
 743             if self.id:
 744                 cache.set(cache_key, tags)
 745         return tags
 746
 747     def reset_theme_counter(self):
 748         if self.id is None:
 749             return
 750
 751         cache_key = "Book.theme_counter/%d" % self.id
 752         cache.delete(cache_key)
 753         if self.parent:
 754             self.parent.reset_theme_counter()
 755
 756     @property
 757     def theme_counter(self):
 758         if self.id:
 759             cache_key = "Book.theme_counter/%d" % self.id
 760             tags = cache.get(cache_key)
 761         else:
 762             tags = None
 763
 764         if tags is None:
 765             tags = {}
 766             for fragment in Fragment.tagged.with_any([self.book_tag()]).order_by():
 767                 for tag in fragment.tags.filter(category='theme').order_by():
 768                     tags[tag.pk] = tags.get(tag.pk, 0) + 1
 769
 770             if self.id:
 771                 cache.set(cache_key, tags)
 772         return tags
 773
 774     def pretty_title(self, html_links=False):
 775         book = self
 776         names = list(book.tags.filter(category='author'))
 777
 778         books = []
 779         while book:
 780             books.append(book)
 781             book = book.parent
 782         names.extend(reversed(books))
 783
 784         if html_links:
 785             names = ['<a href="%s">%s</a>' % (tag.get_absolute_url(), tag.name) for tag in names]
 786         else:
 787             names = [tag.name for tag in names]
 788
 789         return ', '.join(names)
 790
 791     @classmethod
 792     def tagged_top_level(cls, tags):
 793         """ Returns top-level books tagged with `tags'.
 794
 795         It only returns those books which don't have ancestors which are
 796         also tagged with those tags.
 797
 798         """
 799         # get relevant books and their tags
 800         objects = cls.tagged.with_all(tags)
 801         # eliminate descendants
 802         l_tags = Tag.objects.filter(category='book', slug__in=[book.book_tag_slug() for book in objects])
 803         descendants_keys = [book.pk for book in cls.tagged.with_any(l_tags)]
 804         if descendants_keys:
 805             objects = objects.exclude(pk__in=descendants_keys)
 806
 807         return objects
 808
 809
 810 class Fragment(models.Model):
 811     text = models.TextField()
 812     short_text = models.TextField(editable=False)
 813     anchor = models.CharField(max_length=120)
 814     book = models.ForeignKey(Book, related_name='fragments')
 815
 816     objects = models.Manager()
 817     tagged = managers.ModelTaggedItemManager(Tag)
 818     tags = managers.TagDescriptor(Tag)
 819
 820     class Meta:
 821         ordering = ('book', 'anchor',)
 822         verbose_name = _('fragment')
 823         verbose_name_plural = _('fragments')
 824
 825     def get_absolute_url(self):
 826         return '%s#m%s' % (reverse('book_text', kwargs={'slug': self.book.slug}), self.anchor)
 827
 828     def reset_short_html(self):
 829         if self.id is None:
 830             return
 831
 832         cache_key = "Fragment.short_html/%d/%s"
 833         for lang, langname in settings.LANGUAGES:
 834             cache.delete(cache_key % (self.id, lang))
 835
 836     def short_html(self):
 837         if self.id:
 838             cache_key = "Fragment.short_html/%d/%s" % (self.id, get_language())
 839             short_html = cache.get(cache_key)
 840         else:
 841             short_html = None
 842
 843         if short_html is not None:
 844             return mark_safe(short_html)
 845         else:
 846             short_html = unicode(render_to_string('catalogue/fragment_short.html',
 847                 {'fragment': self}))
 848             if self.id:
 849                 cache.set(cache_key, short_html)
 850             return mark_safe(short_html)
 851
 852
 853 class FileRecord(models.Model):
 854     slug = models.SlugField(_('slug'), max_length=120, db_index=True)
 855     type = models.CharField(_('type'), max_length=20, db_index=True)
 856     sha1 = models.CharField(_('sha-1 hash'), max_length=40)
 857     time = models.DateTimeField(_('time'), auto_now_add=True)
 858
 859     class Meta:
 860         ordering = ('-time','-slug', '-type')
 861         verbose_name = _('file record')
 862         verbose_name_plural = _('file records')
 863
 864     def __unicode__(self):
 865         return "%s %s.%s" % (self.sha1,  self.slug, self.type)
 866
 867 ###########
 868 #
 869 # SIGNALS
 870 #
 871 ###########
 872
 873
 874 def _tags_updated_handler(sender, affected_tags, **kwargs):
 875     # reset tag global counter
 876     # we want Tag.changed_at updated for API to know the tag was touched
 877     Tag.objects.filter(pk__in=[tag.pk for tag in affected_tags]).update(book_count=None, changed_at=datetime.now())
 878
 879     # if book tags changed, reset book tag counter
 880     if isinstance(sender, Book) and \
 881                 Tag.objects.filter(pk__in=(tag.pk for tag in affected_tags)).\
 882                     exclude(category__in=('book', 'theme', 'set')).count():
 883         sender.reset_tag_counter()
 884     # if fragment theme changed, reset book theme counter
 885     elif isinstance(sender, Fragment) and \
 886                 Tag.objects.filter(pk__in=(tag.pk for tag in affected_tags)).\
 887                     filter(category='theme').count():
 888         sender.book.reset_theme_counter()
 889 tags_updated.connect(_tags_updated_handler)
 890
 891
 892 def _pre_delete_handler(sender, instance, **kwargs):
 893     """ refresh Book on BookMedia delete """
 894     if sender == BookMedia:
 895         instance.book.save()
 896 pre_delete.connect(_pre_delete_handler)
 897
 898 def _post_save_handler(sender, instance, **kwargs):
 899     """ refresh all the short_html stuff on BookMedia update """
 900     if sender == BookMedia:
 901         instance.book.save()
 902 post_save.connect(_post_save_handler)