apps/catalogue/models/book.py

   1 # -*- coding: utf-8 -*-
   2 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
   3 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
   4 #
   5 import re
   6 from django.conf import settings
   7 from django.core.cache import get_cache
   8 from django.db import models
   9 from django.db.models import permalink
  10 import django.dispatch
  11 from django.utils.datastructures import SortedDict
  12 from django.utils.translation import ugettext_lazy as _
  13 import jsonfield
  14 from catalogue.models import Tag, Fragment, BookMedia
  15 from catalogue.utils import create_zip, split_tags, truncate_html_words, book_upload_path
  16 from catalogue import tasks
  17 from newtagging import managers
  18
  19
  20 permanent_cache = get_cache('permanent')
  21
  22
  23 class Book(models.Model):
  24     """Represents a book imported from WL-XML."""
  25     title         = models.CharField(_('title'), max_length=120)
  26     sort_key = models.CharField(_('sort key'), max_length=120, db_index=True, editable=False)
  27     slug = models.SlugField(_('slug'), max_length=120, db_index=True,
  28             unique=True)
  29     common_slug = models.SlugField(_('slug'), max_length=120, db_index=True)
  30     language = models.CharField(_('language code'), max_length=3, db_index=True,
  31                     default=settings.CATALOGUE_DEFAULT_LANGUAGE)
  32     description   = models.TextField(_('description'), blank=True)
  33     created_at    = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True)
  34     changed_at    = models.DateTimeField(_('creation date'), auto_now=True, db_index=True)
  35     parent_number = models.IntegerField(_('parent number'), default=0)
  36     extra_info    = jsonfield.JSONField(_('extra information'), default='{}')
  37     gazeta_link   = models.CharField(blank=True, max_length=240)
  38     wiki_link     = models.CharField(blank=True, max_length=240)
  39     # files generated during publication
  40
  41     cover = models.FileField(_('cover'), upload_to=book_upload_path('png'),
  42                 null=True, blank=True)
  43     ebook_formats = ['pdf', 'epub', 'mobi', 'fb2', 'txt']
  44     formats = ebook_formats + ['html', 'xml']
  45
  46     parent        = models.ForeignKey('self', blank=True, null=True, related_name='children')
  47
  48     _related_info = jsonfield.JSONField(blank=True, null=True, editable=False)
  49
  50     objects  = models.Manager()
  51     tagged   = managers.ModelTaggedItemManager(Tag)
  52     tags     = managers.TagDescriptor(Tag)
  53
  54     html_built = django.dispatch.Signal()
  55     published = django.dispatch.Signal()
  56
  57     class AlreadyExists(Exception):
  58         pass
  59
  60     class Meta:
  61         ordering = ('sort_key',)
  62         verbose_name = _('book')
  63         verbose_name_plural = _('books')
  64         app_label = 'catalogue'
  65
  66     def __unicode__(self):
  67         return self.title
  68
  69     def save(self, force_insert=False, force_update=False, reset_short_html=True, **kwargs):
  70         from sortify import sortify
  71
  72         self.sort_key = sortify(self.title)
  73
  74         ret = super(Book, self).save(force_insert, force_update)
  75
  76         if reset_short_html:
  77             self.reset_short_html()
  78
  79         return ret
  80
  81     @permalink
  82     def get_absolute_url(self):
  83         return ('catalogue.views.book_detail', [self.slug])
  84
  85     @property
  86     def name(self):
  87         return self.title
  88
  89     def book_tag_slug(self):
  90         return ('l-' + self.slug)[:120]
  91
  92     def book_tag(self):
  93         slug = self.book_tag_slug()
  94         book_tag, created = Tag.objects.get_or_create(slug=slug, category='book')
  95         if created:
  96             book_tag.name = self.title[:50]
  97             book_tag.sort_key = self.title.lower()
  98             book_tag.save()
  99         return book_tag
 100
 101     def has_media(self, type_):
 102         if type_ in Book.formats:
 103             return bool(getattr(self, "%s_file" % type_))
 104         else:
 105             return self.media.filter(type=type_).exists()
 106
 107     def get_media(self, type_):
 108         if self.has_media(type_):
 109             if type_ in Book.formats:
 110                 return getattr(self, "%s_file" % type_)
 111             else:
 112                 return self.media.filter(type=type_)
 113         else:
 114             return None
 115
 116     def get_mp3(self):
 117         return self.get_media("mp3")
 118     def get_odt(self):
 119         return self.get_media("odt")
 120     def get_ogg(self):
 121         return self.get_media("ogg")
 122     def get_daisy(self):
 123         return self.get_media("daisy")
 124
 125     def reset_short_html(self):
 126         if self.id is None:
 127             return
 128
 129         type(self).objects.filter(pk=self.pk).update(_related_info=None)
 130         # Fragment.short_html relies on book's tags, so reset it here too
 131         for fragm in self.fragments.all().iterator():
 132             fragm.reset_short_html()
 133
 134     def has_description(self):
 135         return len(self.description) > 0
 136     has_description.short_description = _('description')
 137     has_description.boolean = True
 138
 139     # ugly ugly ugly
 140     def has_mp3_file(self):
 141         return bool(self.has_media("mp3"))
 142     has_mp3_file.short_description = 'MP3'
 143     has_mp3_file.boolean = True
 144
 145     def has_ogg_file(self):
 146         return bool(self.has_media("ogg"))
 147     has_ogg_file.short_description = 'OGG'
 148     has_ogg_file.boolean = True
 149
 150     def has_daisy_file(self):
 151         return bool(self.has_media("daisy"))
 152     has_daisy_file.short_description = 'DAISY'
 153     has_daisy_file.boolean = True
 154
 155     def wldocument(self, parse_dublincore=True):
 156         from catalogue.import_utils import ORMDocProvider
 157         from librarian.parser import WLDocument
 158
 159         return WLDocument.from_file(self.xml_file.path,
 160                 provider=ORMDocProvider(self),
 161                 parse_dublincore=parse_dublincore)
 162
 163     def build_cover(self, book_info=None):
 164         """(Re)builds the cover image."""
 165         from StringIO import StringIO
 166         from django.core.files.base import ContentFile
 167         from librarian.cover import WLCover
 168
 169         if book_info is None:
 170             book_info = self.wldocument().book_info
 171
 172         cover = WLCover(book_info).image()
 173         imgstr = StringIO()
 174         cover.save(imgstr, 'png')
 175         self.cover.save(None, ContentFile(imgstr.getvalue()))
 176
 177     def build_html(self):
 178         from django.core.files.base import ContentFile
 179         from slughifi import slughifi
 180         from sortify import sortify
 181         from librarian import html
 182
 183         meta_tags = list(self.tags.filter(
 184             category__in=('author', 'epoch', 'genre', 'kind')))
 185         book_tag = self.book_tag()
 186
 187         html_output = self.wldocument(parse_dublincore=False).as_html()
 188         if html_output:
 189             self.html_file.save('%s.html' % self.slug,
 190                     ContentFile(html_output.get_string()))
 191
 192             # get ancestor l-tags for adding to new fragments
 193             ancestor_tags = []
 194             p = self.parent
 195             while p:
 196                 ancestor_tags.append(p.book_tag())
 197                 p = p.parent
 198
 199             # Delete old fragments and create them from scratch
 200             self.fragments.all().delete()
 201             # Extract fragments
 202             closed_fragments, open_fragments = html.extract_fragments(self.html_file.path)
 203             for fragment in closed_fragments.values():
 204                 try:
 205                     theme_names = [s.strip() for s in fragment.themes.split(',')]
 206                 except AttributeError:
 207                     continue
 208                 themes = []
 209                 for theme_name in theme_names:
 210                     if not theme_name:
 211                         continue
 212                     tag, created = Tag.objects.get_or_create(slug=slughifi(theme_name), category='theme')
 213                     if created:
 214                         tag.name = theme_name
 215                         tag.sort_key = sortify(theme_name.lower())
 216                         tag.save()
 217                     themes.append(tag)
 218                 if not themes:
 219                     continue
 220
 221                 text = fragment.to_string()
 222                 short_text = truncate_html_words(text, 15)
 223                 if text == short_text:
 224                     short_text = ''
 225                 new_fragment = Fragment.objects.create(anchor=fragment.id, book=self,
 226                     text=text, short_text=short_text)
 227
 228                 new_fragment.save()
 229                 new_fragment.tags = set(meta_tags + themes + [book_tag] + ancestor_tags)
 230             self.save()
 231             self.html_built.send(sender=self)
 232             return True
 233         return False
 234
 235     # Thin wrappers for builder tasks
 236     def build_pdf(self, *args, **kwargs):
 237         """(Re)builds PDF."""
 238         return tasks.build_pdf.delay(self.pk, *args, **kwargs)
 239     def build_epub(self, *args, **kwargs):
 240         """(Re)builds EPUB."""
 241         return tasks.build_epub.delay(self.pk, *args, **kwargs)
 242     def build_mobi(self, *args, **kwargs):
 243         """(Re)builds MOBI."""
 244         return tasks.build_mobi.delay(self.pk, *args, **kwargs)
 245     def build_fb2(self, *args, **kwargs):
 246         """(Re)build FB2"""
 247         return tasks.build_fb2.delay(self.pk, *args, **kwargs)
 248     def build_txt(self, *args, **kwargs):
 249         """(Re)builds TXT."""
 250         return tasks.build_txt.delay(self.pk, *args, **kwargs)
 251
 252     @staticmethod
 253     def zip_format(format_):
 254         def pretty_file_name(book):
 255             return "%s/%s.%s" % (
 256                 b.extra_info['author'],
 257                 b.slug,
 258                 format_)
 259
 260         field_name = "%s_file" % format_
 261         books = Book.objects.filter(parent=None).exclude(**{field_name: ""})
 262         paths = [(pretty_file_name(b), getattr(b, field_name).path)
 263                     for b in books.iterator()]
 264         return create_zip(paths,
 265                     getattr(settings, "ALL_%s_ZIP" % format_.upper()))
 266
 267     def zip_audiobooks(self, format_):
 268         bm = BookMedia.objects.filter(book=self, type=format_)
 269         paths = map(lambda bm: (None, bm.file.path), bm)
 270         return create_zip(paths, "%s_%s" % (self.slug, format_))
 271
 272     def search_index(self, book_info=None, index=None, index_tags=True, commit=True):
 273         import search
 274         if index is None:
 275             index = search.Index()
 276         try:
 277             index.index_book(self, book_info)
 278             if index_tags:
 279                 idx.index_tags()
 280             if commit:
 281                 index.index.commit()
 282         except Exception, e:
 283             index.index.rollback()
 284             raise e
 285
 286
 287     @classmethod
 288     def from_xml_file(cls, xml_file, **kwargs):
 289         from django.core.files import File
 290         from librarian import dcparser
 291
 292         # use librarian to parse meta-data
 293         book_info = dcparser.parse(xml_file)
 294
 295         if not isinstance(xml_file, File):
 296             xml_file = File(open(xml_file))
 297
 298         try:
 299             return cls.from_text_and_meta(xml_file, book_info, **kwargs)
 300         finally:
 301             xml_file.close()
 302
 303     @classmethod
 304     def from_text_and_meta(cls, raw_file, book_info, overwrite=False,
 305             build_epub=True, build_txt=True, build_pdf=True, build_mobi=True, build_fb2=True,
 306             search_index=True, search_index_tags=True):
 307
 308         # check for parts before we do anything
 309         children = []
 310         if hasattr(book_info, 'parts'):
 311             for part_url in book_info.parts:
 312                 try:
 313                     children.append(Book.objects.get(slug=part_url.slug))
 314                 except Book.DoesNotExist:
 315                     raise Book.DoesNotExist(_('Book "%s" does not exist.') %
 316                             part_url.slug)
 317
 318
 319         # Read book metadata
 320         book_slug = book_info.url.slug
 321         if re.search(r'[^a-z0-9-]', book_slug):
 322             raise ValueError('Invalid characters in slug')
 323         book, created = Book.objects.get_or_create(slug=book_slug)
 324
 325         if created:
 326             book_shelves = []
 327         else:
 328             if not overwrite:
 329                 raise Book.AlreadyExists(_('Book %s already exists') % (
 330                         book_slug))
 331             # Save shelves for this book
 332             book_shelves = list(book.tags.filter(category='set'))
 333
 334         book.language = book_info.language
 335         book.title = book_info.title
 336         if book_info.variant_of:
 337             book.common_slug = book_info.variant_of.slug
 338         else:
 339             book.common_slug = book.slug
 340         book.extra_info = book_info.to_dict()
 341         book.save()
 342
 343         meta_tags = Tag.tags_from_info(book_info)
 344
 345         book.tags = set(meta_tags + book_shelves)
 346
 347         obsolete_children = set(b for b in book.children.all() if b not in children)
 348         for n, child_book in enumerate(children):
 349             child_book.parent = book
 350             child_book.parent_number = n
 351             child_book.save()
 352         # Disown unfaithful children and let them cope on their own.
 353         for child in obsolete_children:
 354             child.parent = None
 355             child.parent_number = 0
 356             child.save()
 357             tasks.fix_tree_tags.delay(child)
 358
 359         # Save XML and HTML files
 360         book.xml_file.save('%s.xml' % book.slug, raw_file, save=False)
 361         book.build_cover(book_info)
 362
 363         # delete old fragments when overwriting
 364         book.fragments.all().delete()
 365
 366         if book.build_html():
 367             # No direct saves behind this point.
 368             if not settings.NO_BUILD_TXT and build_txt:
 369                 book.build_txt()
 370
 371         if not settings.NO_BUILD_EPUB and build_epub:
 372             book.build_epub()
 373
 374         if not settings.NO_BUILD_PDF and build_pdf:
 375             book.build_pdf()
 376
 377         if not settings.NO_BUILD_MOBI and build_mobi:
 378             book.build_mobi()
 379
 380         if not settings.NO_BUILD_FB2 and build_fb2:
 381             book.build_fb2()
 382
 383         if not settings.NO_SEARCH_INDEX and search_index:
 384             tasks.index_book.delay(book.id, book_info=book_info, index_tags=search_index_tags)
 385
 386         tasks.fix_tree_tags.delay(book)
 387         cls.published.send(sender=book)
 388         return book
 389
 390     def fix_tree_tags(self):
 391         """Fixes the l-tags on the book's subtree.
 392
 393         Makes sure that:
 394         * the book has its parents book-tags,
 395         * its fragments have the book's and its parents book-tags,
 396         * runs those for every child book too,
 397         * touches all relevant tags,
 398         * resets tag and theme counter on the book and its ancestry.
 399         """
 400         def fix_subtree(book, parent_tags):
 401             affected_tags = set(book.tags)
 402             book.tags = list(book.tags.exclude(category='book')) + parent_tags
 403             sub_parent_tags = parent_tags + [book.book_tag()]
 404             for frag in book.fragments.all():
 405                 affected_tags.update(frag.tags)
 406                 frag.tags = list(frag.tags.exclude(category='book')) + sub_parent_tags
 407             for child in book.children.all():
 408                 affected_tags.update(fix_subtree(child, sub_parent_tags))
 409             return affected_tags
 410
 411         parent_tags = []
 412         parent = self.parent
 413         while parent is not None:
 414             parent_tags.append(parent.book_tag())
 415             parent = parent.parent
 416
 417         affected_tags = fix_subtree(self, parent_tags)
 418         for tag in affected_tags:
 419             tasks.touch_tag(tag)
 420
 421         book = self
 422         while book is not None:
 423             book.reset_tag_counter()
 424             book.reset_theme_counter()
 425             book = book.parent
 426
 427     def related_info(self):
 428         """Keeps info about related objects (tags, media) in cache field."""
 429         if self._related_info is not None:
 430             return self._related_info
 431         else:
 432             rel = {'tags': {}, 'media': {}}
 433
 434             tags = self.tags.filter(category__in=(
 435                     'author', 'kind', 'genre', 'epoch'))
 436             tags = split_tags(tags)
 437             for category in tags:
 438                 rel['tags'][category] = [
 439                         (t.name, t.slug) for t in tags[category]]
 440
 441             for media_format in BookMedia.formats:
 442                 rel['media'][media_format] = self.has_media(media_format)
 443
 444             book = self
 445             parents = []
 446             while book.parent:
 447                 parents.append((book.parent.title, book.parent.slug))
 448                 book = book.parent
 449             parents = parents[::-1]
 450             if parents:
 451                 rel['parents'] = parents
 452
 453             if self.pk:
 454                 type(self).objects.filter(pk=self.pk).update(_related_info=rel)
 455             return rel
 456
 457     def related_themes(self):
 458         theme_counter = self.theme_counter
 459         book_themes = list(Tag.objects.filter(pk__in=theme_counter.keys()))
 460         for tag in book_themes:
 461             tag.count = theme_counter[tag.pk]
 462         return book_themes
 463
 464     def reset_tag_counter(self):
 465         if self.id is None:
 466             return
 467
 468         cache_key = "Book.tag_counter/%d" % self.id
 469         permanent_cache.delete(cache_key)
 470         if self.parent:
 471             self.parent.reset_tag_counter()
 472
 473     @property
 474     def tag_counter(self):
 475         if self.id:
 476             cache_key = "Book.tag_counter/%d" % self.id
 477             tags = permanent_cache.get(cache_key)
 478         else:
 479             tags = None
 480
 481         if tags is None:
 482             tags = {}
 483             for child in self.children.all().order_by().iterator():
 484                 for tag_pk, value in child.tag_counter.iteritems():
 485                     tags[tag_pk] = tags.get(tag_pk, 0) + value
 486             for tag in self.tags.exclude(category__in=('book', 'theme', 'set')).order_by().iterator():
 487                 tags[tag.pk] = 1
 488
 489             if self.id:
 490                 permanent_cache.set(cache_key, tags)
 491         return tags
 492
 493     def reset_theme_counter(self):
 494         if self.id is None:
 495             return
 496
 497         cache_key = "Book.theme_counter/%d" % self.id
 498         permanent_cache.delete(cache_key)
 499         if self.parent:
 500             self.parent.reset_theme_counter()
 501
 502     @property
 503     def theme_counter(self):
 504         if self.id:
 505             cache_key = "Book.theme_counter/%d" % self.id
 506             tags = permanent_cache.get(cache_key)
 507         else:
 508             tags = None
 509
 510         if tags is None:
 511             tags = {}
 512             for fragment in Fragment.tagged.with_any([self.book_tag()]).order_by().iterator():
 513                 for tag in fragment.tags.filter(category='theme').order_by().iterator():
 514                     tags[tag.pk] = tags.get(tag.pk, 0) + 1
 515
 516             if self.id:
 517                 permanent_cache.set(cache_key, tags)
 518         return tags
 519
 520     def pretty_title(self, html_links=False):
 521         book = self
 522         names = list(book.tags.filter(category='author'))
 523
 524         books = []
 525         while book:
 526             books.append(book)
 527             book = book.parent
 528         names.extend(reversed(books))
 529
 530         if html_links:
 531             names = ['<a href="%s">%s</a>' % (tag.get_absolute_url(), tag.name) for tag in names]
 532         else:
 533             names = [tag.name for tag in names]
 534
 535         return ', '.join(names)
 536
 537     @classmethod
 538     def tagged_top_level(cls, tags):
 539         """ Returns top-level books tagged with `tags`.
 540
 541         It only returns those books which don't have ancestors which are
 542         also tagged with those tags.
 543
 544         """
 545         # get relevant books and their tags
 546         objects = cls.tagged.with_all(tags)
 547         # eliminate descendants
 548         l_tags = Tag.objects.filter(category='book',
 549             slug__in=[book.book_tag_slug() for book in objects.iterator()])
 550         descendants_keys = [book.pk for book in cls.tagged.with_any(l_tags).iterator()]
 551         if descendants_keys:
 552             objects = objects.exclude(pk__in=descendants_keys)
 553
 554         return objects
 555
 556     @classmethod
 557     def book_list(cls, filter=None):
 558         """Generates a hierarchical listing of all books.
 559
 560         Books are optionally filtered with a test function.
 561
 562         """
 563
 564         books_by_parent = {}
 565         books = cls.objects.all().order_by('parent_number', 'sort_key').only(
 566                 'title', 'parent', 'slug')
 567         if filter:
 568             books = books.filter(filter).distinct()
 569
 570             book_ids = set(b['pk'] for b in books.values("pk").iterator())
 571             for book in books.iterator():
 572                 parent = book.parent_id
 573                 if parent not in book_ids:
 574                     parent = None
 575                 books_by_parent.setdefault(parent, []).append(book)
 576         else:
 577             for book in books.iterator():
 578                 books_by_parent.setdefault(book.parent_id, []).append(book)
 579
 580         orphans = []
 581         books_by_author = SortedDict()
 582         for tag in Tag.objects.filter(category='author').iterator():
 583             books_by_author[tag] = []
 584
 585         for book in books_by_parent.get(None,()):
 586             authors = list(book.tags.filter(category='author'))
 587             if authors:
 588                 for author in authors:
 589                     books_by_author[author].append(book)
 590             else:
 591                 orphans.append(book)
 592
 593         return books_by_author, orphans, books_by_parent
 594
 595     _audiences_pl = {
 596         "SP1": (1, u"szkoła podstawowa"),
 597         "SP2": (1, u"szkoła podstawowa"),
 598         "P": (1, u"szkoła podstawowa"),
 599         "G": (2, u"gimnazjum"),
 600         "L": (3, u"liceum"),
 601         "LP": (3, u"liceum"),
 602     }
 603     def audiences_pl(self):
 604         audiences = self.extra_info.get('audiences', [])
 605         audiences = sorted(set([self._audiences_pl[a] for a in audiences]))
 606         return [a[1] for a in audiences]
 607
 608     def choose_fragment(self):
 609         tag = self.book_tag()
 610         fragments = Fragment.tagged.with_any([tag])
 611         if fragments.exists():
 612             return fragments.order_by('?')[0]
 613         elif self.parent:
 614             return self.parent.choose_fragment()
 615         else:
 616             return None
 617
 618
 619 def _has_factory(ftype):
 620     has = lambda self: bool(getattr(self, "%s_file" % ftype))
 621     has.short_description = ftype.upper()
 622     has.__doc__ = None
 623     has.boolean = True
 624     has.__name__ = "has_%s_file" % ftype
 625     return has
 626
 627
 628 # add the file fields
 629 for t in Book.formats:
 630     field_name = "%s_file" % t
 631     models.FileField(_("%s file" % t.upper()),
 632             upload_to=book_upload_path(t),
 633             blank=True).contribute_to_class(Book, field_name)
 634
 635     setattr(Book, "has_%s_file" % t, _has_factory(t))