apps/catalogue/models.py

   1 # -*- coding: utf-8 -*-
   2 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
   3 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
   4 #
   5 from datetime import datetime
   6
   7 from django.db import models
   8 from django.db.models import permalink, Q
   9 from django.core.cache import cache
  10 from django.utils.translation import ugettext_lazy as _
  11 from django.contrib.auth.models import User
  12 from django.core.files import File
  13 from django.template.loader import render_to_string
  14 from django.utils.safestring import mark_safe
  15 from django.utils.translation import get_language
  16 from django.core.urlresolvers import reverse
  17 from django.db.models.signals import post_save, m2m_changed, pre_delete
  18
  19 from django.conf import settings
  20
  21 from newtagging.models import TagBase, tags_updated
  22 from newtagging import managers
  23 from catalogue.fields import JSONField, OverwritingFileField
  24 from catalogue.utils import ExistingFile
  25
  26 from librarian import dcparser, html, epub, NoDublinCore
  27 import mutagen
  28 from mutagen import id3
  29 from slughifi import slughifi
  30 from sortify import sortify
  31
  32
  33 TAG_CATEGORIES = (
  34     ('author', _('author')),
  35     ('epoch', _('epoch')),
  36     ('kind', _('kind')),
  37     ('genre', _('genre')),
  38     ('theme', _('theme')),
  39     ('set', _('set')),
  40     ('book', _('book')),
  41 )
  42
  43 MEDIA_FORMATS = (
  44     ('odt', _('ODT file')),
  45     ('mp3', _('MP3 file')),
  46     ('ogg', _('OGG file')),
  47     ('daisy', _('DAISY file')),
  48 )
  49
  50 class TagSubcategoryManager(models.Manager):
  51     def __init__(self, subcategory):
  52         super(TagSubcategoryManager, self).__init__()
  53         self.subcategory = subcategory
  54
  55     def get_query_set(self):
  56         return super(TagSubcategoryManager, self).get_query_set().filter(category=self.subcategory)
  57
  58
  59 class Tag(TagBase):
  60     name = models.CharField(_('name'), max_length=50, db_index=True)
  61     slug = models.SlugField(_('slug'), max_length=120, db_index=True)
  62     sort_key = models.CharField(_('sort key'), max_length=120, db_index=True)
  63     category = models.CharField(_('category'), max_length=50, blank=False, null=False,
  64         db_index=True, choices=TAG_CATEGORIES)
  65     description = models.TextField(_('description'), blank=True)
  66     main_page = models.BooleanField(_('main page'), default=False, db_index=True, help_text=_('Show tag on main page'))
  67
  68     user = models.ForeignKey(User, blank=True, null=True)
  69     book_count = models.IntegerField(_('book count'), blank=True, null=True)
  70     gazeta_link = models.CharField(blank=True, max_length=240)
  71     wiki_link = models.CharField(blank=True, max_length=240)
  72
  73     created_at    = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True)
  74     changed_at    = models.DateTimeField(_('creation date'), auto_now=True, db_index=True)
  75
  76     class UrlDeprecationWarning(DeprecationWarning):
  77         pass
  78
  79     categories_rev = {
  80         'autor': 'author',
  81         'epoka': 'epoch',
  82         'rodzaj': 'kind',
  83         'gatunek': 'genre',
  84         'motyw': 'theme',
  85         'polka': 'set',
  86     }
  87     categories_dict = dict((item[::-1] for item in categories_rev.iteritems()))
  88
  89     class Meta:
  90         ordering = ('sort_key',)
  91         verbose_name = _('tag')
  92         verbose_name_plural = _('tags')
  93         unique_together = (("slug", "category"),)
  94
  95     def __unicode__(self):
  96         return self.name
  97
  98     def __repr__(self):
  99         return "Tag(slug=%r)" % self.slug
 100
 101     @permalink
 102     def get_absolute_url(self):
 103         return ('catalogue.views.tagged_object_list', [self.url_chunk])
 104
 105     def has_description(self):
 106         return len(self.description) > 0
 107     has_description.short_description = _('description')
 108     has_description.boolean = True
 109
 110     def get_count(self):
 111         """ returns global book count for book tags, fragment count for themes """
 112
 113         if self.book_count is None:
 114             if self.category == 'book':
 115                 # never used
 116                 objects = Book.objects.none()
 117             elif self.category == 'theme':
 118                 objects = Fragment.tagged.with_all((self,))
 119             else:
 120                 objects = Book.tagged.with_all((self,)).order_by()
 121                 if self.category != 'set':
 122                     # eliminate descendants
 123                     l_tags = Tag.objects.filter(slug__in=[book.book_tag_slug() for book in objects])
 124                     descendants_keys = [book.pk for book in Book.tagged.with_any(l_tags)]
 125                     if descendants_keys:
 126                         objects = objects.exclude(pk__in=descendants_keys)
 127             self.book_count = objects.count()
 128             self.save()
 129         return self.book_count
 130
 131     @staticmethod
 132     def get_tag_list(tags):
 133         if isinstance(tags, basestring):
 134             real_tags = []
 135             ambiguous_slugs = []
 136             category = None
 137             deprecated = False
 138             tags_splitted = tags.split('/')
 139             for name in tags_splitted:
 140                 if category:
 141                     real_tags.append(Tag.objects.get(slug=name, category=category))
 142                     category = None
 143                 elif name in Tag.categories_rev:
 144                     category = Tag.categories_rev[name]
 145                 else:
 146                     try:
 147                         real_tags.append(Tag.objects.exclude(category='book').get(slug=name))
 148                         deprecated = True
 149                     except Tag.MultipleObjectsReturned, e:
 150                         ambiguous_slugs.append(name)
 151
 152             if category:
 153                 # something strange left off
 154                 raise Tag.DoesNotExist()
 155             if ambiguous_slugs:
 156                 # some tags should be qualified
 157                 e = Tag.MultipleObjectsReturned()
 158                 e.tags = real_tags
 159                 e.ambiguous_slugs = ambiguous_slugs
 160                 raise e
 161             if deprecated:
 162                 e = Tag.UrlDeprecationWarning()
 163                 e.tags = real_tags
 164                 raise e
 165             return real_tags
 166         else:
 167             return TagBase.get_tag_list(tags)
 168
 169     @property
 170     def url_chunk(self):
 171         return '/'.join((Tag.categories_dict[self.category], self.slug))
 172
 173
 174 # TODO: why is this hard-coded ?
 175 def book_upload_path(ext=None, maxlen=100):
 176     def get_dynamic_path(media, filename, ext=ext):
 177         # how to put related book's slug here?
 178         if not ext:
 179             if media.type == 'daisy':
 180                 ext = 'daisy.zip'
 181             else:
 182                 ext = media.type
 183         if not media.name:
 184             name = slughifi(filename.split(".")[0])
 185         else:
 186             name = slughifi(media.name)
 187         return 'book/%s/%s.%s' % (ext, name[:maxlen-len('book/%s/.%s' % (ext, ext))-4], ext)
 188     return get_dynamic_path
 189
 190
 191 class BookMedia(models.Model):
 192     type        = models.CharField(_('type'), choices=MEDIA_FORMATS, max_length="100")
 193     name        = models.CharField(_('name'), max_length="100")
 194     file        = OverwritingFileField(_('file'), upload_to=book_upload_path())
 195     uploaded_at = models.DateTimeField(_('creation date'), auto_now_add=True, editable=False)
 196     extra_info  = JSONField(_('extra information'), default='{}', editable=False)
 197     book = models.ForeignKey('Book', related_name='media')
 198     source_sha1 = models.CharField(null=True, blank=True, max_length=40, editable=False)
 199
 200     def __unicode__(self):
 201         return "%s (%s)" % (self.name, self.file.name.split("/")[-1])
 202
 203     class Meta:
 204         ordering            = ('type', 'name')
 205         verbose_name        = _('book media')
 206         verbose_name_plural = _('book media')
 207
 208     def save(self, *args, **kwargs):
 209         try:
 210             old = BookMedia.objects.get(pk=self.pk)
 211         except BookMedia.DoesNotExist, e:
 212             pass
 213         else:
 214             # if name changed, change the file name, too
 215             if slughifi(self.name) != slughifi(old.name):
 216                 self.file.save(None, ExistingFile(self.file.path), save=False, leave=True)
 217
 218         super(BookMedia, self).save(*args, **kwargs)
 219         extra_info = self.get_extra_info_value()
 220         extra_info.update(self.read_meta())
 221         self.set_extra_info_value(extra_info)
 222         self.source_sha1 = self.read_source_sha1(self.file.path, self.type)
 223         return super(BookMedia, self).save(*args, **kwargs)
 224
 225     def read_meta(self):
 226         """
 227             Reads some metadata from the audiobook.
 228         """
 229
 230         artist_name = director_name = project = funded_by = ''
 231         if self.type == 'mp3':
 232             try:
 233                 audio = id3.ID3(self.file.path)
 234                 artist_name = ', '.join(', '.join(tag.text) for tag in audio.getall('TPE1'))
 235                 director_name = ', '.join(', '.join(tag.text) for tag in audio.getall('TPE3'))
 236                 project = ", ".join([t.data for t in audio.getall('PRIV')
 237                         if t.owner=='wolnelektury.pl?project'])
 238                 funded_by = ", ".join([t.data for t in audio.getall('PRIV')
 239                         if t.owner=='wolnelektury.pl?funded_by'])
 240             except:
 241                 pass
 242         elif self.type == 'ogg':
 243             try:
 244                 audio = mutagen.File(self.file.path)
 245                 artist_name = ', '.join(audio.get('artist', []))
 246                 director_name = ', '.join(audio.get('conductor', []))
 247                 project = ", ".join(audio.get('project', []))
 248                 funded_by = ", ".join(audio.get('funded_by', []))
 249             except:
 250                 pass
 251         else:
 252             return {}
 253         return {'artist_name': artist_name, 'director_name': director_name,
 254                 'project': project, 'funded_by': funded_by}
 255
 256     @staticmethod
 257     def read_source_sha1(filepath, filetype):
 258         """
 259             Reads source file SHA1 from audiobok metadata.
 260         """
 261
 262         if filetype == 'mp3':
 263             try:
 264                 audio = id3.ID3(filepath)
 265                 return [t.data for t in audio.getall('PRIV')
 266                         if t.owner=='wolnelektury.pl?flac_sha1'][0]
 267             except:
 268                 return None
 269         elif filetype == 'ogg':
 270             try:
 271                 audio = mutagen.File(filepath)
 272                 return audio.get('flac_sha1', [None])[0]
 273             except:
 274                 return None
 275         else:
 276             return None
 277
 278
 279 class Book(models.Model):
 280     title         = models.CharField(_('title'), max_length=120)
 281     sort_key = models.CharField(_('sort_key'), max_length=120, db_index=True, editable=False)
 282     slug          = models.SlugField(_('slug'), max_length=120, unique=True, db_index=True)
 283     description   = models.TextField(_('description'), blank=True)
 284     created_at    = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True)
 285     changed_at    = models.DateTimeField(_('creation date'), auto_now=True, db_index=True)
 286     parent_number = models.IntegerField(_('parent number'), default=0)
 287     extra_info    = JSONField(_('extra information'), default='{}')
 288     gazeta_link   = models.CharField(blank=True, max_length=240)
 289     wiki_link     = models.CharField(blank=True, max_length=240)
 290     # files generated during publication
 291     xml_file      = models.FileField(_('XML file'), upload_to=book_upload_path('xml'), blank=True)
 292     html_file     = models.FileField(_('HTML file'), upload_to=book_upload_path('html'), blank=True)
 293     pdf_file      = models.FileField(_('PDF file'), upload_to=book_upload_path('pdf'), blank=True)
 294     epub_file     = models.FileField(_('EPUB file'), upload_to=book_upload_path('epub'), blank=True)
 295     txt_file      = models.FileField(_('TXT file'), upload_to=book_upload_path('txt'), blank=True)
 296
 297     parent        = models.ForeignKey('self', blank=True, null=True, related_name='children')
 298     objects  = models.Manager()
 299     tagged   = managers.ModelTaggedItemManager(Tag)
 300     tags     = managers.TagDescriptor(Tag)
 301
 302     class AlreadyExists(Exception):
 303         pass
 304
 305     class Meta:
 306         ordering = ('sort_key',)
 307         verbose_name = _('book')
 308         verbose_name_plural = _('books')
 309
 310     def __unicode__(self):
 311         return self.title
 312
 313     def save(self, force_insert=False, force_update=False, reset_short_html=True, **kwargs):
 314         self.sort_key = sortify(self.title)
 315
 316         if reset_short_html:
 317             self.reset_short_html()
 318
 319         return super(Book, self).save(force_insert, force_update)
 320
 321     @permalink
 322     def get_absolute_url(self):
 323         return ('catalogue.views.book_detail', [self.slug])
 324
 325     @property
 326     def name(self):
 327         return self.title
 328
 329     def book_tag_slug(self):
 330         return ('l-' + self.slug)[:120]
 331
 332     def book_tag(self):
 333         slug = self.book_tag_slug()
 334         book_tag, created = Tag.objects.get_or_create(slug=slug, category='book')
 335         if created:
 336             book_tag.name = self.title[:50]
 337             book_tag.sort_key = self.title.lower()
 338             book_tag.save()
 339         return book_tag
 340
 341     def has_media(self, type):
 342         if   type == 'xml':
 343             if self.xml_file:
 344                 return True
 345             else:
 346                 return False
 347         elif type == 'html':
 348             if self.html_file:
 349                 return True
 350             else:
 351                 return False
 352         elif type == 'txt':
 353             if self.txt_file:
 354                 return True
 355             else:
 356                 return False
 357         elif type == 'pdf':
 358             if self.pdf_file:
 359                 return True
 360             else:
 361                 return False
 362         elif type == 'epub':
 363             if self.epub_file:
 364                 return True
 365             else:
 366                 return False
 367         else:
 368             if self.media.filter(type=type).exists():
 369                 return True
 370             else:
 371                 return False
 372
 373     def get_media(self, type):
 374         if self.has_media(type):
 375             if   type == "xml":
 376                 return self.xml_file
 377             elif type == "html":
 378                 return self.html_file
 379             elif type == "epub":
 380                 return self.epub_file
 381             elif type == "txt":
 382                 return self.txt_file
 383             elif type == "pdf":
 384                 return self.pdf_file
 385             else:
 386                 return self.media.filter(type=type)
 387         else:
 388             return None
 389
 390     def get_mp3(self):
 391         return self.get_media("mp3")
 392     def get_odt(self):
 393         return self.get_media("odt")
 394     def get_ogg(self):
 395         return self.get_media("ogg")
 396     def get_daisy(self):
 397         return self.get_media("daisy")
 398
 399     def reset_short_html(self):
 400         cache_key = "Book.short_html/%d/%s"
 401         for lang, langname in settings.LANGUAGES:
 402             cache.delete(cache_key % (self.id, lang))
 403         # Fragment.short_html relies on book's tags, so reset it here too
 404         for fragm in self.fragments.all():
 405             fragm.reset_short_html()
 406
 407     def short_html(self):
 408         cache_key = "Book.short_html/%d/%s" % (self.id, get_language())
 409         short_html = cache.get(cache_key)
 410
 411         if short_html is not None:
 412             print 'b.s from cache'
 413             return mark_safe(short_html)
 414         else:
 415             print 'b.s manual'
 416             tags = self.tags.filter(~Q(category__in=('set', 'theme', 'book')))
 417             tags = [mark_safe(u'<a href="%s">%s</a>' % (tag.get_absolute_url(), tag.name)) for tag in tags]
 418
 419             formats = []
 420             # files generated during publication
 421             if self.has_media("html"):
 422                 formats.append(u'<a href="%s">%s</a>' % (reverse('book_text', kwargs={'slug': self.slug}), _('Read online')))
 423             if self.has_media("pdf"):
 424                 formats.append(u'<a href="%s">PDF</a>' % self.get_media('pdf').url)
 425             if self.root_ancestor.has_media("epub"):
 426                 formats.append(u'<a href="%s">EPUB</a>' % self.root_ancestor.get_media('epub').url)
 427             if self.has_media("txt"):
 428                 formats.append(u'<a href="%s">TXT</a>' % self.get_media('txt').url)
 429             # other files
 430             for m in self.media.order_by('type'):
 431                 formats.append(u'<a href="%s">%s</a>' % (m.file.url, m.type.upper()))
 432
 433             formats = [mark_safe(format) for format in formats]
 434
 435             short_html = unicode(render_to_string('catalogue/book_short.html',
 436                 {'book': self, 'tags': tags, 'formats': formats}))
 437             cache.set(cache_key, short_html)
 438             return mark_safe(short_html)
 439
 440     @property
 441     def root_ancestor(self):
 442         """ returns the oldest ancestor """
 443
 444         if not hasattr(self, '_root_ancestor'):
 445             book = self
 446             while book.parent:
 447                 book = book.parent
 448             self._root_ancestor = book
 449         return self._root_ancestor
 450
 451
 452     def has_description(self):
 453         return len(self.description) > 0
 454     has_description.short_description = _('description')
 455     has_description.boolean = True
 456
 457     # ugly ugly ugly
 458     def has_pdf_file(self):
 459         return bool(self.pdf_file)
 460     has_pdf_file.short_description = 'PDF'
 461     has_pdf_file.boolean = True
 462
 463     def has_epub_file(self):
 464         return bool(self.epub_file)
 465     has_epub_file.short_description = 'EPUB'
 466     has_epub_file.boolean = True
 467
 468     def has_txt_file(self):
 469         return bool(self.txt_file)
 470     has_txt_file.short_description = 'HTML'
 471     has_txt_file.boolean = True
 472
 473     def has_html_file(self):
 474         return bool(self.html_file)
 475     has_html_file.short_description = 'HTML'
 476     has_html_file.boolean = True
 477
 478     def has_odt_file(self):
 479         return bool(self.has_media("odt"))
 480     has_odt_file.short_description = 'ODT'
 481     has_odt_file.boolean = True
 482
 483     def has_mp3_file(self):
 484         return bool(self.has_media("mp3"))
 485     has_mp3_file.short_description = 'MP3'
 486     has_mp3_file.boolean = True
 487
 488     def has_ogg_file(self):
 489         return bool(self.has_media("ogg"))
 490     has_ogg_file.short_description = 'OGG'
 491     has_ogg_file.boolean = True
 492
 493     def has_daisy_file(self):
 494         return bool(self.has_media("daisy"))
 495     has_daisy_file.short_description = 'DAISY'
 496     has_daisy_file.boolean = True
 497
 498     def build_epub(self, remove_descendants=True):
 499         """ (Re)builds the epub file.
 500             If book has a parent, does nothing.
 501             Unless remove_descendants is False, descendants' epubs are removed.
 502         """
 503
 504         from StringIO import StringIO
 505         from hashlib import sha1
 506         from django.core.files.base import ContentFile
 507         from librarian import DocProvider
 508
 509         class BookImportDocProvider(DocProvider):
 510             """ used for joined EPUBs """
 511
 512             def __init__(self, book):
 513                 self.book = book
 514
 515             def by_slug(self, slug):
 516                 if slug == self.book.slug:
 517                     return self.book.xml_file
 518                 else:
 519                     return Book.objects.get(slug=slug).xml_file
 520
 521         if self.parent:
 522             # don't need an epub
 523             return
 524
 525         epub_file = StringIO()
 526         try:
 527             epub.transform(BookImportDocProvider(self), self.slug, output_file=epub_file)
 528             self.epub_file.save('%s.epub' % self.slug, ContentFile(epub_file.getvalue()))
 529             FileRecord(slug=self.slug, type='epub', sha1=sha1(epub_file.getvalue()).hexdigest()).save()
 530         except NoDublinCore:
 531             pass
 532
 533         book_descendants = list(self.children.all())
 534         while len(book_descendants) > 0:
 535             child_book = book_descendants.pop(0)
 536             if remove_descendants and child_book.has_epub_file():
 537                 child_book.epub_file.delete()
 538             # save anyway, to refresh short_html
 539             child_book.save()
 540             book_descendants += list(child_book.children.all())
 541
 542     def build_txt(self):
 543         from StringIO import StringIO
 544         from django.core.files.base import ContentFile
 545         from librarian import text
 546
 547         out = StringIO()
 548         text.transform(open(self.xml_file.path), out)
 549         self.txt_file.save('%s.txt' % self.slug, ContentFile(out.getvalue()))
 550
 551
 552     def build_html(self):
 553         from tempfile import NamedTemporaryFile
 554         from markupstring import MarkupString
 555
 556         meta_tags = list(self.tags.filter(
 557             category__in=('author', 'epoch', 'genre', 'kind')))
 558         book_tag = self.book_tag()
 559
 560         html_file = NamedTemporaryFile()
 561         if html.transform(self.xml_file.path, html_file, parse_dublincore=False):
 562             self.html_file.save('%s.html' % self.slug, File(html_file))
 563
 564             # get ancestor l-tags for adding to new fragments
 565             ancestor_tags = []
 566             p = self.parent
 567             while p:
 568                 ancestor_tags.append(p.book_tag())
 569                 p = p.parent
 570
 571             # Delete old fragments and create them from scratch
 572             self.fragments.all().delete()
 573             # Extract fragments
 574             closed_fragments, open_fragments = html.extract_fragments(self.html_file.path)
 575             for fragment in closed_fragments.values():
 576                 try:
 577                     theme_names = [s.strip() for s in fragment.themes.split(',')]
 578                 except AttributeError:
 579                     continue
 580                 themes = []
 581                 for theme_name in theme_names:
 582                     if not theme_name:
 583                         continue
 584                     tag, created = Tag.objects.get_or_create(slug=slughifi(theme_name), category='theme')
 585                     if created:
 586                         tag.name = theme_name
 587                         tag.sort_key = theme_name.lower()
 588                         tag.save()
 589                     themes.append(tag)
 590                 if not themes:
 591                     continue
 592
 593                 text = fragment.to_string()
 594                 short_text = ''
 595                 if (len(MarkupString(text)) > 240):
 596                     short_text = unicode(MarkupString(text)[:160])
 597                 new_fragment = Fragment.objects.create(anchor=fragment.id, book=self,
 598                     text=text, short_text=short_text)
 599
 600                 new_fragment.save()
 601                 new_fragment.tags = set(meta_tags + themes + [book_tag] + ancestor_tags)
 602             self.save()
 603             return True
 604         return False
 605
 606
 607     @classmethod
 608     def from_xml_file(cls, xml_file, **kwargs):
 609         # use librarian to parse meta-data
 610         book_info = dcparser.parse(xml_file)
 611
 612         if not isinstance(xml_file, File):
 613             xml_file = File(open(xml_file))
 614
 615         try:
 616             return cls.from_text_and_meta(xml_file, book_info, **kwargs)
 617         finally:
 618             xml_file.close()
 619
 620     @classmethod
 621     def from_text_and_meta(cls, raw_file, book_info, overwrite=False, build_epub=True, build_txt=True):
 622         import re
 623
 624         # check for parts before we do anything
 625         children = []
 626         if hasattr(book_info, 'parts'):
 627             for part_url in book_info.parts:
 628                 base, slug = part_url.rsplit('/', 1)
 629                 try:
 630                     children.append(Book.objects.get(slug=slug))
 631                 except Book.DoesNotExist, e:
 632                     raise Book.DoesNotExist(_('Book with slug = "%s" does not exist.') % slug)
 633
 634
 635         # Read book metadata
 636         book_base, book_slug = book_info.url.rsplit('/', 1)
 637         if re.search(r'[^a-zA-Z0-9-]', book_slug):
 638             raise ValueError('Invalid characters in slug')
 639         book, created = Book.objects.get_or_create(slug=book_slug)
 640
 641         if created:
 642             book_shelves = []
 643         else:
 644             if not overwrite:
 645                 raise Book.AlreadyExists(_('Book %s already exists') % book_slug)
 646             # Save shelves for this book
 647             book_shelves = list(book.tags.filter(category='set'))
 648
 649         book.title = book_info.title
 650         book.set_extra_info_value(book_info.to_dict())
 651         book.save()
 652
 653         meta_tags = []
 654         categories = (('kinds', 'kind'), ('genres', 'genre'), ('authors', 'author'), ('epochs', 'epoch'))
 655         for field_name, category in categories:
 656             try:
 657                 tag_names = getattr(book_info, field_name)
 658             except:
 659                 tag_names = [getattr(book_info, category)]
 660             for tag_name in tag_names:
 661                 tag_sort_key = tag_name
 662                 if category == 'author':
 663                     tag_sort_key = tag_name.last_name
 664                     tag_name = ' '.join(tag_name.first_names) + ' ' + tag_name.last_name
 665                 tag, created = Tag.objects.get_or_create(slug=slughifi(tag_name), category=category)
 666                 if created:
 667                     tag.name = tag_name
 668                     tag.sort_key = sortify(tag_sort_key.lower())
 669                     tag.save()
 670                 meta_tags.append(tag)
 671
 672         book.tags = set(meta_tags + book_shelves)
 673
 674         book_tag = book.book_tag()
 675
 676         for n, child_book in enumerate(children):
 677             child_book.parent = book
 678             child_book.parent_number = n
 679             child_book.save()
 680
 681         # Save XML and HTML files
 682         book.xml_file.save('%s.xml' % book.slug, raw_file, save=False)
 683
 684         # delete old fragments when overwriting
 685         book.fragments.all().delete()
 686
 687         if book.build_html():
 688             if not settings.NO_BUILD_TXT and build_txt:
 689                 book.build_txt()
 690
 691         if not settings.NO_BUILD_EPUB and build_epub:
 692             book.root_ancestor.build_epub()
 693
 694         book_descendants = list(book.children.all())
 695         # add l-tag to descendants and their fragments
 696         # delete unnecessary EPUB files
 697         while len(book_descendants) > 0:
 698             child_book = book_descendants.pop(0)
 699             child_book.tags = list(child_book.tags) + [book_tag]
 700             child_book.save()
 701             for fragment in child_book.fragments.all():
 702                 fragment.tags = set(list(fragment.tags) + [book_tag])
 703             book_descendants += list(child_book.children.all())
 704
 705         # refresh cache
 706         book.reset_tag_counter()
 707         book.reset_theme_counter()
 708
 709         book.save()
 710         return book
 711
 712     def reset_tag_counter(self):
 713         cache_key = "Book.tag_counter/%d" % self.id
 714         cache.delete(cache_key)
 715         if self.parent:
 716             self.parent.reset_tag_counter()
 717
 718     @property
 719     def tag_counter(self):
 720         cache_key = "Book.tag_counter/%d" % self.id
 721         tags = cache.get(cache_key)
 722         print 'tag'
 723         if tags is None:
 724             print 'tag manual'
 725             tags = {}
 726             for child in self.children.all().order_by():
 727                 for tag_pk, value in child.tag_counter.iteritems():
 728                     tags[tag_pk] = tags.get(tag_pk, 0) + value
 729             for tag in self.tags.exclude(category__in=('book', 'theme', 'set')).order_by():
 730                 tags[tag.pk] = 1
 731
 732             cache.set(cache_key, tags)
 733         return tags
 734
 735     def reset_theme_counter(self):
 736         cache_key = "Book.theme_counter/%d" % self.id
 737         cache.delete(cache_key)
 738         if self.parent:
 739             self.parent.reset_theme_counter()
 740
 741     @property
 742     def theme_counter(self):
 743         cache_key = "Book.theme_counter/%d" % self.id
 744         tags = cache.get(cache_key)
 745         print 'theme'
 746         if tags is None:
 747             print 'theme manual'
 748             tags = {}
 749             for fragment in Fragment.tagged.with_any([self.book_tag()]).order_by():
 750                 for tag in fragment.tags.filter(category='theme').order_by():
 751                     tags[tag.pk] = tags.get(tag.pk, 0) + 1
 752
 753             cache.set(cache_key, tags)
 754         return tags
 755
 756     def pretty_title(self, html_links=False):
 757         book = self
 758         names = list(book.tags.filter(category='author'))
 759
 760         books = []
 761         while book:
 762             books.append(book)
 763             book = book.parent
 764         names.extend(reversed(books))
 765
 766         if html_links:
 767             names = ['<a href="%s">%s</a>' % (tag.get_absolute_url(), tag.name) for tag in names]
 768         else:
 769             names = [tag.name for tag in names]
 770
 771         return ', '.join(names)
 772
 773     @classmethod
 774     def tagged_top_level(cls, tags):
 775         """ Returns top-level books tagged with `tags'.
 776
 777         It only returns those books which don't have ancestors which are
 778         also tagged with those tags.
 779
 780         """
 781         # get relevant books and their tags
 782         objects = cls.tagged.with_all(tags)
 783         # eliminate descendants
 784         l_tags = Tag.objects.filter(category='book', slug__in=[book.book_tag_slug() for book in objects])
 785         descendants_keys = [book.pk for book in cls.tagged.with_any(l_tags)]
 786         if descendants_keys:
 787             objects = objects.exclude(pk__in=descendants_keys)
 788
 789         return objects
 790
 791
 792 class Fragment(models.Model):
 793     text = models.TextField()
 794     short_text = models.TextField(editable=False)
 795     anchor = models.CharField(max_length=120)
 796     book = models.ForeignKey(Book, related_name='fragments')
 797
 798     objects = models.Manager()
 799     tagged = managers.ModelTaggedItemManager(Tag)
 800     tags = managers.TagDescriptor(Tag)
 801
 802     class Meta:
 803         ordering = ('book', 'anchor',)
 804         verbose_name = _('fragment')
 805         verbose_name_plural = _('fragments')
 806
 807     def get_absolute_url(self):
 808         return '%s#m%s' % (reverse('book_text', kwargs={'slug': self.book.slug}), self.anchor)
 809
 810     def reset_short_html(self):
 811         cache_key = "Fragment.short_html/%d/%s"
 812         for lang, langname in settings.LANGUAGES:
 813             cache.delete(cache_key % (self.id, lang))
 814
 815     def short_html(self):
 816         cache_key = "Fragment.short_html/%d/%s" % (self.id, get_language())
 817         short_html = cache.get(cache_key)
 818
 819         if short_html is not None:
 820             print 'f.s from cache'
 821             return mark_safe(short_html)
 822         else:
 823             print 'f.s manual'
 824             short_html = unicode(render_to_string('catalogue/fragment_short.html',
 825                 {'fragment': self}))
 826             cache.set(cache_key, short_html)
 827             return mark_safe(short_html)
 828
 829
 830 class FileRecord(models.Model):
 831     slug = models.SlugField(_('slug'), max_length=120, db_index=True)
 832     type = models.CharField(_('type'), max_length=20, db_index=True)
 833     sha1 = models.CharField(_('sha-1 hash'), max_length=40)
 834     time = models.DateTimeField(_('time'), auto_now_add=True)
 835
 836     class Meta:
 837         ordering = ('-time','-slug', '-type')
 838         verbose_name = _('file record')
 839         verbose_name_plural = _('file records')
 840
 841     def __unicode__(self):
 842         return "%s %s.%s" % (self.sha1,  self.slug, self.type)
 843
 844 ###########
 845 #
 846 # SIGNALS
 847 #
 848 ###########
 849
 850
 851 def _tags_updated_handler(sender, affected_tags, **kwargs):
 852     # reset tag global counter
 853     # we want Tag.changed_at updated for API to know the tag was touched
 854     Tag.objects.filter(pk__in=[tag.pk for tag in affected_tags]).update(book_count=None, changed_at=datetime.now())
 855
 856     # if book tags changed, reset book tag counter
 857     if isinstance(sender, Book) and \
 858                 Tag.objects.filter(pk__in=(tag.pk for tag in affected_tags)).\
 859                     exclude(category__in=('book', 'theme', 'set')).count():
 860         sender.reset_tag_counter()
 861     # if fragment theme changed, reset book theme counter
 862     elif isinstance(sender, Fragment) and \
 863                 Tag.objects.filter(pk__in=(tag.pk for tag in affected_tags)).\
 864                     filter(category='theme').count():
 865         sender.book.reset_theme_counter()
 866 tags_updated.connect(_tags_updated_handler)
 867
 868
 869 def _pre_delete_handler(sender, instance, **kwargs):
 870     """ refresh Book on BookMedia delete """
 871     if sender == BookMedia:
 872         instance.book.save()
 873 pre_delete.connect(_pre_delete_handler)
 874
 875 def _post_save_handler(sender, instance, **kwargs):
 876     """ refresh all the short_html stuff on BookMedia update """
 877     if sender == BookMedia:
 878         instance.book.save()
 879 post_save.connect(_post_save_handler)