apps/catalogue/models.py

   1 # -*- coding: utf-8 -*-
   2 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
   3 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
   4 #
   5 from datetime import datetime
   6
   7 from django.db import models
   8 from django.db.models import permalink, Q
   9 from django.utils.translation import ugettext_lazy as _
  10 from django.contrib.auth.models import User
  11 from django.core.files import File
  12 from django.template.loader import render_to_string
  13 from django.utils.safestring import mark_safe
  14 from django.utils.translation import get_language
  15 from django.core.urlresolvers import reverse
  16 from django.db.models.signals import post_save, m2m_changed, pre_delete
  17
  18 from django.conf import settings
  19
  20 from newtagging.models import TagBase, tags_updated
  21 from newtagging import managers
  22 from catalogue.fields import JSONField, OverwritingFileField
  23 from catalogue.utils import ExistingFile
  24
  25 from librarian import dcparser, html, epub, NoDublinCore
  26 import mutagen
  27 from mutagen import id3
  28 from slughifi import slughifi
  29 from sortify import sortify
  30
  31
  32 TAG_CATEGORIES = (
  33     ('author', _('author')),
  34     ('epoch', _('epoch')),
  35     ('kind', _('kind')),
  36     ('genre', _('genre')),
  37     ('theme', _('theme')),
  38     ('set', _('set')),
  39     ('book', _('book')),
  40 )
  41
  42 MEDIA_FORMATS = (
  43     ('odt', _('ODT file')),
  44     ('mp3', _('MP3 file')),
  45     ('ogg', _('OGG file')),
  46     ('daisy', _('DAISY file')),
  47 )
  48
  49 class TagSubcategoryManager(models.Manager):
  50     def __init__(self, subcategory):
  51         super(TagSubcategoryManager, self).__init__()
  52         self.subcategory = subcategory
  53
  54     def get_query_set(self):
  55         return super(TagSubcategoryManager, self).get_query_set().filter(category=self.subcategory)
  56
  57
  58 class Tag(TagBase):
  59     name = models.CharField(_('name'), max_length=50, db_index=True)
  60     slug = models.SlugField(_('slug'), max_length=120, db_index=True)
  61     sort_key = models.CharField(_('sort key'), max_length=120, db_index=True)
  62     category = models.CharField(_('category'), max_length=50, blank=False, null=False,
  63         db_index=True, choices=TAG_CATEGORIES)
  64     description = models.TextField(_('description'), blank=True)
  65     main_page = models.BooleanField(_('main page'), default=False, db_index=True, help_text=_('Show tag on main page'))
  66
  67     user = models.ForeignKey(User, blank=True, null=True)
  68     book_count = models.IntegerField(_('book count'), blank=True, null=True)
  69     gazeta_link = models.CharField(blank=True, max_length=240)
  70     wiki_link = models.CharField(blank=True, max_length=240)
  71
  72     created_at    = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True)
  73     changed_at    = models.DateTimeField(_('creation date'), auto_now=True, db_index=True)
  74
  75     class UrlDeprecationWarning(DeprecationWarning):
  76         pass
  77
  78     categories_rev = {
  79         'autor': 'author',
  80         'epoka': 'epoch',
  81         'rodzaj': 'kind',
  82         'gatunek': 'genre',
  83         'motyw': 'theme',
  84         'polka': 'set',
  85     }
  86     categories_dict = dict((item[::-1] for item in categories_rev.iteritems()))
  87
  88     class Meta:
  89         ordering = ('sort_key',)
  90         verbose_name = _('tag')
  91         verbose_name_plural = _('tags')
  92         unique_together = (("slug", "category"),)
  93
  94     def __unicode__(self):
  95         return self.name
  96
  97     def __repr__(self):
  98         return "Tag(slug=%r)" % self.slug
  99
 100     @permalink
 101     def get_absolute_url(self):
 102         return ('catalogue.views.tagged_object_list', [self.url_chunk])
 103
 104     def has_description(self):
 105         return len(self.description) > 0
 106     has_description.short_description = _('description')
 107     has_description.boolean = True
 108
 109     def get_count(self):
 110         """ returns global book count for book tags, fragment count for themes """
 111
 112         if self.book_count is None:
 113             if self.category == 'book':
 114                 # never used
 115                 objects = Book.objects.none()
 116             elif self.category == 'theme':
 117                 objects = Fragment.tagged.with_all((self,))
 118             else:
 119                 objects = Book.tagged.with_all((self,)).order_by()
 120                 if self.category != 'set':
 121                     # eliminate descendants
 122                     l_tags = Tag.objects.filter(slug__in=[book.book_tag_slug() for book in objects])
 123                     descendants_keys = [book.pk for book in Book.tagged.with_any(l_tags)]
 124                     if descendants_keys:
 125                         objects = objects.exclude(pk__in=descendants_keys)
 126             self.book_count = objects.count()
 127             self.save()
 128         return self.book_count
 129
 130     @staticmethod
 131     def get_tag_list(tags):
 132         if isinstance(tags, basestring):
 133             real_tags = []
 134             ambiguous_slugs = []
 135             category = None
 136             deprecated = False
 137             tags_splitted = tags.split('/')
 138             for name in tags_splitted:
 139                 if category:
 140                     real_tags.append(Tag.objects.get(slug=name, category=category))
 141                     category = None
 142                 elif name in Tag.categories_rev:
 143                     category = Tag.categories_rev[name]
 144                 else:
 145                     try:
 146                         real_tags.append(Tag.objects.exclude(category='book').get(slug=name))
 147                         deprecated = True
 148                     except Tag.MultipleObjectsReturned, e:
 149                         ambiguous_slugs.append(name)
 150
 151             if category:
 152                 # something strange left off
 153                 raise Tag.DoesNotExist()
 154             if ambiguous_slugs:
 155                 # some tags should be qualified
 156                 e = Tag.MultipleObjectsReturned()
 157                 e.tags = real_tags
 158                 e.ambiguous_slugs = ambiguous_slugs
 159                 raise e
 160             if deprecated:
 161                 e = Tag.UrlDeprecationWarning()
 162                 e.tags = real_tags
 163                 raise e
 164             return real_tags
 165         else:
 166             return TagBase.get_tag_list(tags)
 167
 168     @property
 169     def url_chunk(self):
 170         return '/'.join((Tag.categories_dict[self.category], self.slug))
 171
 172
 173 # TODO: why is this hard-coded ?
 174 def book_upload_path(ext=None, maxlen=100):
 175     def get_dynamic_path(media, filename, ext=ext):
 176         # how to put related book's slug here?
 177         if not ext:
 178             if media.type == 'daisy':
 179                 ext = 'daisy.zip'
 180             else:
 181                 ext = media.type
 182         if not media.name:
 183             name = slughifi(filename.split(".")[0])
 184         else:
 185             name = slughifi(media.name)
 186         return 'book/%s/%s.%s' % (ext, name[:maxlen-len('book/%s/.%s' % (ext, ext))-4], ext)
 187     return get_dynamic_path
 188
 189
 190 class BookMedia(models.Model):
 191     type        = models.CharField(_('type'), choices=MEDIA_FORMATS, max_length="100")
 192     name        = models.CharField(_('name'), max_length="100")
 193     file        = OverwritingFileField(_('file'), upload_to=book_upload_path())
 194     uploaded_at = models.DateTimeField(_('creation date'), auto_now_add=True, editable=False)
 195     extra_info  = JSONField(_('extra information'), default='{}', editable=False)
 196     book = models.ForeignKey('Book', related_name='media')
 197     source_sha1 = models.CharField(null=True, blank=True, max_length=40, editable=False)
 198
 199     def __unicode__(self):
 200         return "%s (%s)" % (self.name, self.file.name.split("/")[-1])
 201
 202     class Meta:
 203         ordering            = ('type', 'name')
 204         verbose_name        = _('book media')
 205         verbose_name_plural = _('book media')
 206
 207     def save(self, *args, **kwargs):
 208         try:
 209             old = BookMedia.objects.get(pk=self.pk)
 210         except BookMedia.DoesNotExist, e:
 211             pass
 212         else:
 213             # if name changed, change the file name, too
 214             if slughifi(self.name) != slughifi(old.name):
 215                 self.file.save(None, ExistingFile(self.file.path), save=False, leave=True)
 216
 217         super(BookMedia, self).save(*args, **kwargs)
 218         extra_info = self.get_extra_info_value()
 219         extra_info.update(self.read_meta())
 220         self.set_extra_info_value(extra_info)
 221         self.source_sha1 = self.read_source_sha1(self.file.path, self.type)
 222         return super(BookMedia, self).save(*args, **kwargs)
 223
 224     def read_meta(self):
 225         """
 226             Reads some metadata from the audiobook.
 227         """
 228
 229         artist_name = director_name = project = funded_by = ''
 230         if self.type == 'mp3':
 231             try:
 232                 audio = id3.ID3(self.file.path)
 233                 artist_name = ', '.join(', '.join(tag.text) for tag in audio.getall('TPE1'))
 234                 director_name = ', '.join(', '.join(tag.text) for tag in audio.getall('TPE3'))
 235                 project = ", ".join([t.data for t in audio.getall('PRIV')
 236                         if t.owner=='wolnelektury.pl?project'])
 237                 funded_by = ", ".join([t.data for t in audio.getall('PRIV')
 238                         if t.owner=='wolnelektury.pl?funded_by'])
 239             except:
 240                 pass
 241         elif self.type == 'ogg':
 242             try:
 243                 audio = mutagen.File(self.file.path)
 244                 artist_name = ', '.join(audio.get('artist', []))
 245                 director_name = ', '.join(audio.get('conductor', []))
 246                 project = ", ".join(audio.get('project', []))
 247                 funded_by = ", ".join(audio.get('funded_by', []))
 248             except:
 249                 pass
 250         else:
 251             return {}
 252         return {'artist_name': artist_name, 'director_name': director_name,
 253                 'project': project, 'funded_by': funded_by}
 254
 255     @staticmethod
 256     def read_source_sha1(filepath, filetype):
 257         """
 258             Reads source file SHA1 from audiobok metadata.
 259         """
 260
 261         if filetype == 'mp3':
 262             try:
 263                 audio = id3.ID3(filepath)
 264                 return [t.data for t in audio.getall('PRIV')
 265                         if t.owner=='wolnelektury.pl?flac_sha1'][0]
 266             except:
 267                 return None
 268         elif filetype == 'ogg':
 269             try:
 270                 audio = mutagen.File(filepath)
 271                 return audio.get('flac_sha1', [None])[0]
 272             except:
 273                 return None
 274         else:
 275             return None
 276
 277
 278 class Book(models.Model):
 279     title         = models.CharField(_('title'), max_length=120)
 280     sort_key = models.CharField(_('sort_key'), max_length=120, db_index=True, editable=False)
 281     slug          = models.SlugField(_('slug'), max_length=120, unique=True, db_index=True)
 282     description   = models.TextField(_('description'), blank=True)
 283     created_at    = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True)
 284     changed_at    = models.DateTimeField(_('creation date'), auto_now=True, db_index=True)
 285     _short_html   = models.TextField(_('short HTML'), editable=False)
 286     parent_number = models.IntegerField(_('parent number'), default=0)
 287     extra_info    = JSONField(_('extra information'), default='{}')
 288     gazeta_link   = models.CharField(blank=True, max_length=240)
 289     wiki_link     = models.CharField(blank=True, max_length=240)
 290     # files generated during publication
 291     xml_file      = models.FileField(_('XML file'), upload_to=book_upload_path('xml'), blank=True)
 292     html_file     = models.FileField(_('HTML file'), upload_to=book_upload_path('html'), blank=True)
 293     pdf_file      = models.FileField(_('PDF file'), upload_to=book_upload_path('pdf'), blank=True)
 294     epub_file     = models.FileField(_('EPUB file'), upload_to=book_upload_path('epub'), blank=True)
 295     txt_file      = models.FileField(_('TXT file'), upload_to=book_upload_path('txt'), blank=True)
 296
 297     parent        = models.ForeignKey('self', blank=True, null=True, related_name='children')
 298     objects  = models.Manager()
 299     tagged   = managers.ModelTaggedItemManager(Tag)
 300     tags     = managers.TagDescriptor(Tag)
 301
 302     _tag_counter = JSONField(null=True, editable=False)
 303     _theme_counter = JSONField(null=True, editable=False)
 304
 305     class AlreadyExists(Exception):
 306         pass
 307
 308     class Meta:
 309         ordering = ('sort_key',)
 310         verbose_name = _('book')
 311         verbose_name_plural = _('books')
 312
 313     def __unicode__(self):
 314         return self.title
 315
 316     def save(self, force_insert=False, force_update=False, reset_short_html=True, **kwargs):
 317         self.sort_key = sortify(self.title)
 318
 319         if reset_short_html:
 320             # Reset _short_html during save
 321             update = {}
 322             for key in filter(lambda x: x.startswith('_short_html'), self.__dict__):
 323                 update[key] = ''
 324                 self.__setattr__(key, '')
 325             # Fragment.short_html relies on book's tags, so reset it here too
 326             self.fragments.all().update(**update)
 327
 328         return super(Book, self).save(force_insert, force_update)
 329
 330     @permalink
 331     def get_absolute_url(self):
 332         return ('catalogue.views.book_detail', [self.slug])
 333
 334     @property
 335     def name(self):
 336         return self.title
 337
 338     def book_tag_slug(self):
 339         return ('l-' + self.slug)[:120]
 340
 341     def book_tag(self):
 342         slug = self.book_tag_slug()
 343         book_tag, created = Tag.objects.get_or_create(slug=slug, category='book')
 344         if created:
 345             book_tag.name = self.title[:50]
 346             book_tag.sort_key = self.title.lower()
 347             book_tag.save()
 348         return book_tag
 349
 350     def has_media(self, type):
 351         if   type == 'xml':
 352             if self.xml_file:
 353                 return True
 354             else:
 355                 return False
 356         elif type == 'html':
 357             if self.html_file:
 358                 return True
 359             else:
 360                 return False
 361         elif type == 'txt':
 362             if self.txt_file:
 363                 return True
 364             else:
 365                 return False
 366         elif type == 'pdf':
 367             if self.pdf_file:
 368                 return True
 369             else:
 370                 return False
 371         elif type == 'epub':
 372             if self.epub_file:
 373                 return True
 374             else:
 375                 return False
 376         else:
 377             if self.media.filter(type=type).exists():
 378                 return True
 379             else:
 380                 return False
 381
 382     def get_media(self, type):
 383         if self.has_media(type):
 384             if   type == "xml":
 385                 return self.xml_file
 386             elif type == "html":
 387                 return self.html_file
 388             elif type == "epub":
 389                 return self.epub_file
 390             elif type == "txt":
 391                 return self.txt_file
 392             elif type == "pdf":
 393                 return self.pdf_file
 394             else:
 395                 return self.media.filter(type=type)
 396         else:
 397             return None
 398
 399     def get_mp3(self):
 400         return self.get_media("mp3")
 401     def get_odt(self):
 402         return self.get_media("odt")
 403     def get_ogg(self):
 404         return self.get_media("ogg")
 405     def get_daisy(self):
 406         return self.get_media("daisy")
 407
 408     def short_html(self):
 409         key = '_short_html_%s' % get_language()
 410         short_html = getattr(self, key)
 411
 412         if short_html and len(short_html):
 413             return mark_safe(short_html)
 414         else:
 415             tags = self.tags.filter(~Q(category__in=('set', 'theme', 'book')))
 416             tags = [mark_safe(u'<a href="%s">%s</a>' % (tag.get_absolute_url(), tag.name)) for tag in tags]
 417
 418             formats = []
 419             # files generated during publication
 420             if self.has_media("html"):
 421                 formats.append(u'<a href="%s">%s</a>' % (reverse('book_text', kwargs={'slug': self.slug}), _('Read online')))
 422             if self.has_media("pdf"):
 423                 formats.append(u'<a href="%s">PDF</a>' % self.get_media('pdf').url)
 424             if self.root_ancestor.has_media("epub"):
 425                 formats.append(u'<a href="%s">EPUB</a>' % self.root_ancestor.get_media('epub').url)
 426             if self.has_media("txt"):
 427                 formats.append(u'<a href="%s">TXT</a>' % self.get_media('txt').url)
 428             # other files
 429             for m in self.media.order_by('type'):
 430                 formats.append(u'<a href="%s">%s</a>' % (m.file.url, m.type.upper()))
 431
 432             formats = [mark_safe(format) for format in formats]
 433
 434             setattr(self, key, unicode(render_to_string('catalogue/book_short.html',
 435                 {'book': self, 'tags': tags, 'formats': formats})))
 436             self.save(reset_short_html=False)
 437             return mark_safe(getattr(self, key))
 438
 439
 440     @property
 441     def root_ancestor(self):
 442         """ returns the oldest ancestor """
 443
 444         if not hasattr(self, '_root_ancestor'):
 445             book = self
 446             while book.parent:
 447                 book = book.parent
 448             self._root_ancestor = book
 449         return self._root_ancestor
 450
 451
 452     def has_description(self):
 453         return len(self.description) > 0
 454     has_description.short_description = _('description')
 455     has_description.boolean = True
 456
 457     # ugly ugly ugly
 458     def has_pdf_file(self):
 459         return bool(self.pdf_file)
 460     has_pdf_file.short_description = 'PDF'
 461     has_pdf_file.boolean = True
 462
 463     def has_epub_file(self):
 464         return bool(self.epub_file)
 465     has_epub_file.short_description = 'EPUB'
 466     has_epub_file.boolean = True
 467
 468     def has_txt_file(self):
 469         return bool(self.txt_file)
 470     has_txt_file.short_description = 'HTML'
 471     has_txt_file.boolean = True
 472
 473     def has_html_file(self):
 474         return bool(self.html_file)
 475     has_html_file.short_description = 'HTML'
 476     has_html_file.boolean = True
 477
 478     def has_odt_file(self):
 479         return bool(self.has_media("odt"))
 480     has_odt_file.short_description = 'ODT'
 481     has_odt_file.boolean = True
 482
 483     def has_mp3_file(self):
 484         return bool(self.has_media("mp3"))
 485     has_mp3_file.short_description = 'MP3'
 486     has_mp3_file.boolean = True
 487
 488     def has_ogg_file(self):
 489         return bool(self.has_media("ogg"))
 490     has_ogg_file.short_description = 'OGG'
 491     has_ogg_file.boolean = True
 492
 493     def has_daisy_file(self):
 494         return bool(self.has_media("daisy"))
 495     has_daisy_file.short_description = 'DAISY'
 496     has_daisy_file.boolean = True
 497
 498     def build_epub(self, remove_descendants=True):
 499         """ (Re)builds the epub file.
 500             If book has a parent, does nothing.
 501             Unless remove_descendants is False, descendants' epubs are removed.
 502         """
 503
 504         from StringIO import StringIO
 505         from hashlib import sha1
 506         from django.core.files.base import ContentFile
 507         from librarian import DocProvider
 508
 509         class BookImportDocProvider(DocProvider):
 510             """ used for joined EPUBs """
 511
 512             def __init__(self, book):
 513                 self.book = book
 514
 515             def by_slug(self, slug):
 516                 if slug == self.book.slug:
 517                     return self.book.xml_file
 518                 else:
 519                     return Book.objects.get(slug=slug).xml_file
 520
 521         if self.parent:
 522             # don't need an epub
 523             return
 524
 525         epub_file = StringIO()
 526         try:
 527             epub.transform(BookImportDocProvider(self), self.slug, output_file=epub_file)
 528             self.epub_file.save('%s.epub' % self.slug, ContentFile(epub_file.getvalue()))
 529             FileRecord(slug=self.slug, type='epub', sha1=sha1(epub_file.getvalue()).hexdigest()).save()
 530         except NoDublinCore:
 531             pass
 532
 533         book_descendants = list(self.children.all())
 534         while len(book_descendants) > 0:
 535             child_book = book_descendants.pop(0)
 536             if remove_descendants and child_book.has_epub_file():
 537                 child_book.epub_file.delete()
 538             # save anyway, to refresh short_html
 539             child_book.save()
 540             book_descendants += list(child_book.children.all())
 541
 542     def build_txt(self):
 543         from StringIO import StringIO
 544         from django.core.files.base import ContentFile
 545         from librarian import text
 546
 547         out = StringIO()
 548         text.transform(open(self.xml_file.path), out)
 549         self.txt_file.save('%s.txt' % self.slug, ContentFile(out.getvalue()))
 550
 551
 552     def build_html(self):
 553         from tempfile import NamedTemporaryFile
 554         from markupstring import MarkupString
 555
 556         meta_tags = list(self.tags.filter(
 557             category__in=('author', 'epoch', 'genre', 'kind')))
 558         book_tag = self.book_tag()
 559
 560         html_file = NamedTemporaryFile()
 561         if html.transform(self.xml_file.path, html_file, parse_dublincore=False):
 562             self.html_file.save('%s.html' % self.slug, File(html_file))
 563
 564             # get ancestor l-tags for adding to new fragments
 565             ancestor_tags = []
 566             p = self.parent
 567             while p:
 568                 ancestor_tags.append(p.book_tag())
 569                 p = p.parent
 570
 571             # Delete old fragments and create them from scratch
 572             self.fragments.all().delete()
 573             # Extract fragments
 574             closed_fragments, open_fragments = html.extract_fragments(self.html_file.path)
 575             for fragment in closed_fragments.values():
 576                 try:
 577                     theme_names = [s.strip() for s in fragment.themes.split(',')]
 578                 except AttributeError:
 579                     continue
 580                 themes = []
 581                 for theme_name in theme_names:
 582                     if not theme_name:
 583                         continue
 584                     tag, created = Tag.objects.get_or_create(slug=slughifi(theme_name), category='theme')
 585                     if created:
 586                         tag.name = theme_name
 587                         tag.sort_key = theme_name.lower()
 588                         tag.save()
 589                     themes.append(tag)
 590                 if not themes:
 591                     continue
 592
 593                 text = fragment.to_string()
 594                 short_text = ''
 595                 if (len(MarkupString(text)) > 240):
 596                     short_text = unicode(MarkupString(text)[:160])
 597                 new_fragment = Fragment.objects.create(anchor=fragment.id, book=self,
 598                     text=text, short_text=short_text)
 599
 600                 new_fragment.save()
 601                 new_fragment.tags = set(meta_tags + themes + [book_tag] + ancestor_tags)
 602             self.save()
 603             return True
 604         return False
 605
 606
 607     @classmethod
 608     def from_xml_file(cls, xml_file, **kwargs):
 609         # use librarian to parse meta-data
 610         book_info = dcparser.parse(xml_file)
 611
 612         if not isinstance(xml_file, File):
 613             xml_file = File(open(xml_file))
 614
 615         try:
 616             return cls.from_text_and_meta(xml_file, book_info, **kwargs)
 617         finally:
 618             xml_file.close()
 619
 620     @classmethod
 621     def from_text_and_meta(cls, raw_file, book_info, overwrite=False, build_epub=True, build_txt=True):
 622         import re
 623
 624         # check for parts before we do anything
 625         children = []
 626         if hasattr(book_info, 'parts'):
 627             for part_url in book_info.parts:
 628                 base, slug = part_url.rsplit('/', 1)
 629                 try:
 630                     children.append(Book.objects.get(slug=slug))
 631                 except Book.DoesNotExist, e:
 632                     raise Book.DoesNotExist(_('Book with slug = "%s" does not exist.') % slug)
 633
 634
 635         # Read book metadata
 636         book_base, book_slug = book_info.url.rsplit('/', 1)
 637         if re.search(r'[^a-zA-Z0-9-]', book_slug):
 638             raise ValueError('Invalid characters in slug')
 639         book, created = Book.objects.get_or_create(slug=book_slug)
 640
 641         if created:
 642             book_shelves = []
 643         else:
 644             if not overwrite:
 645                 raise Book.AlreadyExists(_('Book %s already exists') % book_slug)
 646             # Save shelves for this book
 647             book_shelves = list(book.tags.filter(category='set'))
 648
 649         book.title = book_info.title
 650         book.set_extra_info_value(book_info.to_dict())
 651         book._short_html = ''
 652         book.save()
 653
 654         meta_tags = []
 655         categories = (('kinds', 'kind'), ('genres', 'genre'), ('authors', 'author'), ('epochs', 'epoch'))
 656         for field_name, category in categories:
 657             try:
 658                 tag_names = getattr(book_info, field_name)
 659             except:
 660                 tag_names = [getattr(book_info, category)]
 661             for tag_name in tag_names:
 662                 tag_sort_key = tag_name
 663                 if category == 'author':
 664                     tag_sort_key = tag_name.last_name
 665                     tag_name = ' '.join(tag_name.first_names) + ' ' + tag_name.last_name
 666                 tag, created = Tag.objects.get_or_create(slug=slughifi(tag_name), category=category)
 667                 if created:
 668                     tag.name = tag_name
 669                     tag.sort_key = sortify(tag_sort_key.lower())
 670                     tag.save()
 671                 meta_tags.append(tag)
 672
 673         book.tags = set(meta_tags + book_shelves)
 674
 675         book_tag = book.book_tag()
 676
 677         for n, child_book in enumerate(children):
 678             child_book.parent = book
 679             child_book.parent_number = n
 680             child_book.save()
 681
 682         # Save XML and HTML files
 683         book.xml_file.save('%s.xml' % book.slug, raw_file, save=False)
 684
 685         # delete old fragments when overwriting
 686         book.fragments.all().delete()
 687
 688         if book.build_html():
 689             if not settings.NO_BUILD_TXT and build_txt:
 690                 book.build_txt()
 691
 692         if not settings.NO_BUILD_EPUB and build_epub:
 693             book.root_ancestor.build_epub()
 694
 695         book_descendants = list(book.children.all())
 696         # add l-tag to descendants and their fragments
 697         # delete unnecessary EPUB files
 698         while len(book_descendants) > 0:
 699             child_book = book_descendants.pop(0)
 700             child_book.tags = list(child_book.tags) + [book_tag]
 701             child_book.save()
 702             for fragment in child_book.fragments.all():
 703                 fragment.tags = set(list(fragment.tags) + [book_tag])
 704             book_descendants += list(child_book.children.all())
 705
 706         # refresh cache
 707         book.reset_tag_counter()
 708         book.reset_theme_counter()
 709
 710         book.save()
 711         return book
 712
 713
 714     def refresh_tag_counter(self):
 715         tags = {}
 716         for child in self.children.all().order_by():
 717             for tag_pk, value in child.tag_counter.iteritems():
 718                 tags[tag_pk] = tags.get(tag_pk, 0) + value
 719         for tag in self.tags.exclude(category__in=('book', 'theme', 'set')).order_by():
 720             tags[tag.pk] = 1
 721         self.set__tag_counter_value(tags)
 722         self.save(reset_short_html=False)
 723         return tags
 724
 725     def reset_tag_counter(self):
 726         self._tag_counter = None
 727         self.save(reset_short_html=False)
 728         if self.parent:
 729             self.parent.reset_tag_counter()
 730
 731     @property
 732     def tag_counter(self):
 733         if self._tag_counter is None:
 734             return self.refresh_tag_counter()
 735         return dict((int(k), v) for k, v in self.get__tag_counter_value().iteritems())
 736
 737     def refresh_theme_counter(self):
 738         tags = {}
 739         for fragment in Fragment.tagged.with_any([self.book_tag()]).order_by():
 740             for tag in fragment.tags.filter(category='theme').order_by():
 741                 tags[tag.pk] = tags.get(tag.pk, 0) + 1
 742         self.set__theme_counter_value(tags)
 743         self.save(reset_short_html=False)
 744         return tags
 745
 746     def reset_theme_counter(self):
 747         self._theme_counter = None
 748         self.save(reset_short_html=False)
 749         if self.parent:
 750             self.parent.reset_theme_counter()
 751
 752     @property
 753     def theme_counter(self):
 754         if self._theme_counter is None:
 755             return self.refresh_theme_counter()
 756         return dict((int(k), v) for k, v in self.get__theme_counter_value().iteritems())
 757
 758     def pretty_title(self, html_links=False):
 759         book = self
 760         names = list(book.tags.filter(category='author'))
 761
 762         books = []
 763         while book:
 764             books.append(book)
 765             book = book.parent
 766         names.extend(reversed(books))
 767
 768         if html_links:
 769             names = ['<a href="%s">%s</a>' % (tag.get_absolute_url(), tag.name) for tag in names]
 770         else:
 771             names = [tag.name for tag in names]
 772
 773         return ', '.join(names)
 774
 775     @classmethod
 776     def tagged_top_level(cls, tags):
 777         """ Returns top-level books tagged with `tags'.
 778
 779         It only returns those books which don't have ancestors which are
 780         also tagged with those tags.
 781
 782         """
 783         # get relevant books and their tags
 784         objects = cls.tagged.with_all(tags)
 785         # eliminate descendants
 786         l_tags = Tag.objects.filter(category='book', slug__in=[book.book_tag_slug() for book in objects])
 787         descendants_keys = [book.pk for book in cls.tagged.with_any(l_tags)]
 788         if descendants_keys:
 789             objects = objects.exclude(pk__in=descendants_keys)
 790
 791         return objects
 792
 793
 794 class Fragment(models.Model):
 795     text = models.TextField()
 796     short_text = models.TextField(editable=False)
 797     _short_html = models.TextField(editable=False)
 798     anchor = models.CharField(max_length=120)
 799     book = models.ForeignKey(Book, related_name='fragments')
 800
 801     objects = models.Manager()
 802     tagged = managers.ModelTaggedItemManager(Tag)
 803     tags = managers.TagDescriptor(Tag)
 804
 805     class Meta:
 806         ordering = ('book', 'anchor',)
 807         verbose_name = _('fragment')
 808         verbose_name_plural = _('fragments')
 809
 810     def get_absolute_url(self):
 811         return '%s#m%s' % (reverse('book_text', kwargs={'slug': self.book.slug}), self.anchor)
 812
 813     def short_html(self):
 814         key = '_short_html_%s' % get_language()
 815         short_html = getattr(self, key)
 816         if short_html and len(short_html):
 817             return mark_safe(short_html)
 818         else:
 819             setattr(self, key, unicode(render_to_string('catalogue/fragment_short.html',
 820                 {'fragment': self})))
 821             self.save()
 822             return mark_safe(getattr(self, key))
 823
 824
 825 class FileRecord(models.Model):
 826     slug = models.SlugField(_('slug'), max_length=120, db_index=True)
 827     type = models.CharField(_('type'), max_length=20, db_index=True)
 828     sha1 = models.CharField(_('sha-1 hash'), max_length=40)
 829     time = models.DateTimeField(_('time'), auto_now_add=True)
 830
 831     class Meta:
 832         ordering = ('-time','-slug', '-type')
 833         verbose_name = _('file record')
 834         verbose_name_plural = _('file records')
 835
 836     def __unicode__(self):
 837         return "%s %s.%s" % (self.sha1,  self.slug, self.type)
 838
 839 ###########
 840 #
 841 # SIGNALS
 842 #
 843 ###########
 844
 845
 846 def _tags_updated_handler(sender, affected_tags, **kwargs):
 847     # reset tag global counter
 848     # we want Tag.changed_at updated for API to know the tag was touched
 849     Tag.objects.filter(pk__in=[tag.pk for tag in affected_tags]).update(book_count=None, changed_at=datetime.now())
 850
 851     # if book tags changed, reset book tag counter
 852     if isinstance(sender, Book) and \
 853                 Tag.objects.filter(pk__in=(tag.pk for tag in affected_tags)).\
 854                     exclude(category__in=('book', 'theme', 'set')).count():
 855         sender.reset_tag_counter()
 856     # if fragment theme changed, reset book theme counter
 857     elif isinstance(sender, Fragment) and \
 858                 Tag.objects.filter(pk__in=(tag.pk for tag in affected_tags)).\
 859                     filter(category='theme').count():
 860         sender.book.reset_theme_counter()
 861 tags_updated.connect(_tags_updated_handler)
 862
 863
 864 def _pre_delete_handler(sender, instance, **kwargs):
 865     """ refresh Book on BookMedia delete """
 866     if sender == BookMedia:
 867         instance.book.save()
 868 pre_delete.connect(_pre_delete_handler)
 869
 870 def _post_save_handler(sender, instance, **kwargs):
 871     """ refresh all the short_html stuff on BookMedia update """
 872     if sender == BookMedia:
 873         instance.book.save()
 874 post_save.connect(_post_save_handler)