apps/catalogue/models/book.py

   1 # -*- coding: utf-8 -*-
   2 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
   3 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
   4 #
   5 import re
   6 from django.conf import settings
   7 from django.core.cache import get_cache
   8 from django.db import models
   9 from django.db.models import permalink
  10 import django.dispatch
  11 from django.utils.datastructures import SortedDict
  12 from django.utils.translation import ugettext_lazy as _
  13 import jsonfield
  14 from catalogue.models import Tag, Fragment, BookMedia
  15 from catalogue.utils import create_zip, split_tags, truncate_html_words, book_upload_path
  16 from catalogue import tasks
  17 from newtagging import managers
  18
  19
  20 permanent_cache = get_cache('permanent')
  21
  22
  23 class Book(models.Model):
  24     """Represents a book imported from WL-XML."""
  25     title         = models.CharField(_('title'), max_length=120)
  26     sort_key = models.CharField(_('sort key'), max_length=120, db_index=True, editable=False)
  27     slug = models.SlugField(_('slug'), max_length=120, db_index=True,
  28             unique=True)
  29     common_slug = models.SlugField(_('slug'), max_length=120, db_index=True)
  30     language = models.CharField(_('language code'), max_length=3, db_index=True,
  31                     default=settings.CATALOGUE_DEFAULT_LANGUAGE)
  32     description   = models.TextField(_('description'), blank=True)
  33     created_at    = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True)
  34     changed_at    = models.DateTimeField(_('creation date'), auto_now=True, db_index=True)
  35     parent_number = models.IntegerField(_('parent number'), default=0)
  36     extra_info    = jsonfield.JSONField(_('extra information'), default='{}')
  37     gazeta_link   = models.CharField(blank=True, max_length=240)
  38     wiki_link     = models.CharField(blank=True, max_length=240)
  39     # files generated during publication
  40
  41     cover = models.FileField(_('cover'), upload_to=book_upload_path('png'),
  42                 null=True, blank=True)
  43     ebook_formats = ['pdf', 'epub', 'mobi', 'fb2', 'txt']
  44     formats = ebook_formats + ['html', 'xml']
  45
  46     parent        = models.ForeignKey('self', blank=True, null=True, related_name='children')
  47
  48     _related_info = jsonfield.JSONField(blank=True, null=True, editable=False)
  49
  50     objects  = models.Manager()
  51     tagged   = managers.ModelTaggedItemManager(Tag)
  52     tags     = managers.TagDescriptor(Tag)
  53
  54     html_built = django.dispatch.Signal()
  55     published = django.dispatch.Signal()
  56
  57     class AlreadyExists(Exception):
  58         pass
  59
  60     class Meta:
  61         ordering = ('sort_key',)
  62         verbose_name = _('book')
  63         verbose_name_plural = _('books')
  64         app_label = 'catalogue'
  65
  66     def __unicode__(self):
  67         return self.title
  68
  69     def save(self, force_insert=False, force_update=False, reset_short_html=True, **kwargs):
  70         from sortify import sortify
  71
  72         self.sort_key = sortify(self.title)
  73
  74         ret = super(Book, self).save(force_insert, force_update)
  75
  76         if reset_short_html:
  77             self.reset_short_html()
  78
  79         return ret
  80
  81     @permalink
  82     def get_absolute_url(self):
  83         return ('catalogue.views.book_detail', [self.slug])
  84
  85     @property
  86     def name(self):
  87         return self.title
  88
  89     def book_tag_slug(self):
  90         return ('l-' + self.slug)[:120]
  91
  92     def book_tag(self):
  93         slug = self.book_tag_slug()
  94         book_tag, created = Tag.objects.get_or_create(slug=slug, category='book')
  95         if created:
  96             book_tag.name = self.title[:50]
  97             book_tag.sort_key = self.title.lower()
  98             book_tag.save()
  99         return book_tag
 100
 101     def has_media(self, type_):
 102         if type_ in Book.formats:
 103             return bool(getattr(self, "%s_file" % type_))
 104         else:
 105             return self.media.filter(type=type_).exists()
 106
 107     def get_media(self, type_):
 108         if self.has_media(type_):
 109             if type_ in Book.formats:
 110                 return getattr(self, "%s_file" % type_)
 111             else:
 112                 return self.media.filter(type=type_)
 113         else:
 114             return None
 115
 116     def get_mp3(self):
 117         return self.get_media("mp3")
 118     def get_odt(self):
 119         return self.get_media("odt")
 120     def get_ogg(self):
 121         return self.get_media("ogg")
 122     def get_daisy(self):
 123         return self.get_media("daisy")
 124
 125     def reset_short_html(self):
 126         if self.id is None:
 127             return
 128
 129         type(self).objects.filter(pk=self.pk).update(_related_info=None)
 130         # Fragment.short_html relies on book's tags, so reset it here too
 131         for fragm in self.fragments.all().iterator():
 132             fragm.reset_short_html()
 133
 134     def has_description(self):
 135         return len(self.description) > 0
 136     has_description.short_description = _('description')
 137     has_description.boolean = True
 138
 139     # ugly ugly ugly
 140     def has_mp3_file(self):
 141         return bool(self.has_media("mp3"))
 142     has_mp3_file.short_description = 'MP3'
 143     has_mp3_file.boolean = True
 144
 145     def has_ogg_file(self):
 146         return bool(self.has_media("ogg"))
 147     has_ogg_file.short_description = 'OGG'
 148     has_ogg_file.boolean = True
 149
 150     def has_daisy_file(self):
 151         return bool(self.has_media("daisy"))
 152     has_daisy_file.short_description = 'DAISY'
 153     has_daisy_file.boolean = True
 154
 155     def wldocument(self, parse_dublincore=True):
 156         from catalogue.import_utils import ORMDocProvider
 157         from librarian.parser import WLDocument
 158
 159         return WLDocument.from_file(self.xml_file.path,
 160                 provider=ORMDocProvider(self),
 161                 parse_dublincore=parse_dublincore)
 162
 163     def build_cover(self, book_info=None):
 164         """(Re)builds the cover image."""
 165         from StringIO import StringIO
 166         from django.core.files.base import ContentFile
 167         from librarian.cover import WLCover
 168
 169         if book_info is None:
 170             book_info = self.wldocument().book_info
 171
 172         cover = WLCover(book_info).image()
 173         imgstr = StringIO()
 174         cover.save(imgstr, 'png')
 175         self.cover.save(None, ContentFile(imgstr.getvalue()))
 176
 177     def build_html(self):
 178         from django.core.files.base import ContentFile
 179         from slughifi import slughifi
 180         from sortify import sortify
 181         from librarian import html
 182
 183         meta_tags = list(self.tags.filter(
 184             category__in=('author', 'epoch', 'genre', 'kind')))
 185         book_tag = self.book_tag()
 186
 187         html_output = self.wldocument(parse_dublincore=False).as_html()
 188         if html_output:
 189             self.html_file.save('%s.html' % self.slug,
 190                     ContentFile(html_output.get_string()))
 191
 192             # get ancestor l-tags for adding to new fragments
 193             ancestor_tags = []
 194             p = self.parent
 195             while p:
 196                 ancestor_tags.append(p.book_tag())
 197                 p = p.parent
 198
 199             # Delete old fragments and create them from scratch
 200             self.fragments.all().delete()
 201             # Extract fragments
 202             closed_fragments, open_fragments = html.extract_fragments(self.html_file.path)
 203             for fragment in closed_fragments.values():
 204                 try:
 205                     theme_names = [s.strip() for s in fragment.themes.split(',')]
 206                 except AttributeError:
 207                     continue
 208                 themes = []
 209                 for theme_name in theme_names:
 210                     if not theme_name:
 211                         continue
 212                     tag, created = Tag.objects.get_or_create(slug=slughifi(theme_name), category='theme')
 213                     if created:
 214                         tag.name = theme_name
 215                         tag.sort_key = sortify(theme_name.lower())
 216                         tag.save()
 217                     themes.append(tag)
 218                 if not themes:
 219                     continue
 220
 221                 text = fragment.to_string()
 222                 short_text = truncate_html_words(text, 15)
 223                 if text == short_text:
 224                     short_text = ''
 225                 new_fragment = Fragment.objects.create(anchor=fragment.id, book=self,
 226                     text=text, short_text=short_text)
 227
 228                 new_fragment.save()
 229                 new_fragment.tags = set(meta_tags + themes + [book_tag] + ancestor_tags)
 230             self.save()
 231             self.html_built.send(sender=self)
 232             return True
 233         return False
 234
 235     # Thin wrappers for builder tasks
 236     def build_pdf(self, *args, **kwargs):
 237         """(Re)builds PDF."""
 238         return tasks.build_pdf.delay(self.pk, *args, **kwargs)
 239     def build_epub(self, *args, **kwargs):
 240         """(Re)builds EPUB."""
 241         return tasks.build_epub.delay(self.pk, *args, **kwargs)
 242     def build_mobi(self, *args, **kwargs):
 243         """(Re)builds MOBI."""
 244         return tasks.build_mobi.delay(self.pk, *args, **kwargs)
 245     def build_fb2(self, *args, **kwargs):
 246         """(Re)build FB2"""
 247         return tasks.build_fb2.delay(self.pk, *args, **kwargs)
 248     def build_txt(self, *args, **kwargs):
 249         """(Re)builds TXT."""
 250         return tasks.build_txt.delay(self.pk, *args, **kwargs)
 251
 252     @staticmethod
 253     def zip_format(format_):
 254         def pretty_file_name(book):
 255             return "%s/%s.%s" % (
 256                 b.extra_info['author'],
 257                 b.slug,
 258                 format_)
 259
 260         field_name = "%s_file" % format_
 261         books = Book.objects.filter(parent=None).exclude(**{field_name: ""})
 262         paths = [(pretty_file_name(b), getattr(b, field_name).path)
 263                     for b in books.iterator()]
 264         return create_zip(paths,
 265                     getattr(settings, "ALL_%s_ZIP" % format_.upper()))
 266
 267     def zip_audiobooks(self, format_):
 268         bm = BookMedia.objects.filter(book=self, type=format_)
 269         paths = map(lambda bm: (None, bm.file.path), bm)
 270         return create_zip(paths, "%s_%s" % (self.slug, format_))
 271
 272     def search_index(self, book_info=None, reuse_index=False, index_tags=True):
 273         import search
 274         if reuse_index:
 275             idx = search.ReusableIndex()
 276         else:
 277             idx = search.Index()
 278
 279         idx.open()
 280         try:
 281             idx.index_book(self, book_info)
 282             if index_tags:
 283                 idx.index_tags()
 284         finally:
 285             idx.close()
 286
 287     @classmethod
 288     def from_xml_file(cls, xml_file, **kwargs):
 289         from django.core.files import File
 290         from librarian import dcparser
 291
 292         # use librarian to parse meta-data
 293         book_info = dcparser.parse(xml_file)
 294
 295         if not isinstance(xml_file, File):
 296             xml_file = File(open(xml_file))
 297
 298         try:
 299             return cls.from_text_and_meta(xml_file, book_info, **kwargs)
 300         finally:
 301             xml_file.close()
 302
 303     @classmethod
 304     def from_text_and_meta(cls, raw_file, book_info, overwrite=False,
 305             build_epub=True, build_txt=True, build_pdf=True, build_mobi=True, build_fb2=True,
 306             search_index=True, search_index_tags=True, search_index_reuse=False):
 307
 308         # check for parts before we do anything
 309         children = []
 310         if hasattr(book_info, 'parts'):
 311             for part_url in book_info.parts:
 312                 try:
 313                     children.append(Book.objects.get(slug=part_url.slug))
 314                 except Book.DoesNotExist:
 315                     raise Book.DoesNotExist(_('Book "%s" does not exist.') %
 316                             part_url.slug)
 317
 318
 319         # Read book metadata
 320         book_slug = book_info.url.slug
 321         if re.search(r'[^a-z0-9-]', book_slug):
 322             raise ValueError('Invalid characters in slug')
 323         book, created = Book.objects.get_or_create(slug=book_slug)
 324
 325         if created:
 326             book_shelves = []
 327         else:
 328             if not overwrite:
 329                 raise Book.AlreadyExists(_('Book %s already exists') % (
 330                         book_slug))
 331             # Save shelves for this book
 332             book_shelves = list(book.tags.filter(category='set'))
 333
 334         book.language = book_info.language
 335         book.title = book_info.title
 336         if book_info.variant_of:
 337             book.common_slug = book_info.variant_of.slug
 338         else:
 339             book.common_slug = book.slug
 340         book.extra_info = book_info.to_dict()
 341         book.save()
 342
 343         meta_tags = Tag.tags_from_info(book_info)
 344
 345         book.tags = set(meta_tags + book_shelves)
 346
 347         obsolete_children = set(b for b in book.children.all() if b not in children)
 348         for n, child_book in enumerate(children):
 349             child_book.parent = book
 350             child_book.parent_number = n
 351             child_book.save()
 352         # Disown unfaithful children and let them cope on their own.
 353         for child in obsolete_children:
 354             child.parent = None
 355             child.parent_number = 0
 356             child.save()
 357             tasks.fix_tree_tags.delay(child)
 358
 359         # Save XML and HTML files
 360         book.xml_file.save('%s.xml' % book.slug, raw_file, save=False)
 361         book.build_cover(book_info)
 362
 363         # delete old fragments when overwriting
 364         book.fragments.all().delete()
 365
 366         if book.build_html():
 367             # No direct saves behind this point.
 368             if not settings.NO_BUILD_TXT and build_txt:
 369                 book.build_txt()
 370
 371         if not settings.NO_BUILD_EPUB and build_epub:
 372             book.build_epub()
 373
 374         if not settings.NO_BUILD_PDF and build_pdf:
 375             book.build_pdf()
 376
 377         if not settings.NO_BUILD_MOBI and build_mobi:
 378             book.build_mobi()
 379
 380         if not settings.NO_BUILD_FB2 and build_fb2:
 381             book.build_fb2()
 382
 383         if not settings.NO_SEARCH_INDEX and search_index:
 384             book.search_index(index_tags=search_index_tags, reuse_index=search_index_reuse)
 385             #index_book.delay(book.id, book_info)
 386
 387         tasks.fix_tree_tags.delay(book)
 388         cls.published.send(sender=book)
 389         return book
 390
 391     def fix_tree_tags(self):
 392         """Fixes the l-tags on the book's subtree.
 393
 394         Makes sure that:
 395         * the book has its parents book-tags,
 396         * its fragments have the book's and its parents book-tags,
 397         * runs those for every child book too,
 398         * touches all relevant tags,
 399         * resets tag and theme counter on the book and its ancestry.
 400         """
 401         def fix_subtree(book, parent_tags):
 402             affected_tags = set(book.tags)
 403             book.tags = list(book.tags.exclude(category='book')) + parent_tags
 404             sub_parent_tags = parent_tags + [book.book_tag()]
 405             for frag in book.fragments.all():
 406                 affected_tags.update(frag.tags)
 407                 frag.tags = list(frag.tags.exclude(category='book')) + sub_parent_tags
 408             for child in book.children.all():
 409                 affected_tags.update(fix_subtree(child, sub_parent_tags))
 410             return affected_tags
 411
 412         parent_tags = []
 413         parent = self.parent
 414         while parent is not None:
 415             parent_tags.append(parent.book_tag())
 416             parent = parent.parent
 417
 418         affected_tags = fix_subtree(self, parent_tags)
 419         for tag in affected_tags:
 420             tasks.touch_tag(tag)
 421
 422         book = self
 423         while book is not None:
 424             book.reset_tag_counter()
 425             book.reset_theme_counter()
 426             book = book.parent
 427
 428     def related_info(self):
 429         """Keeps info about related objects (tags, media) in cache field."""
 430         if self._related_info is not None:
 431             return self._related_info
 432         else:
 433             rel = {'tags': {}, 'media': {}}
 434
 435             tags = self.tags.filter(category__in=(
 436                     'author', 'kind', 'genre', 'epoch'))
 437             tags = split_tags(tags)
 438             for category in tags:
 439                 rel['tags'][category] = [
 440                         (t.name, t.slug) for t in tags[category]]
 441
 442             for media_format in BookMedia.formats:
 443                 rel['media'][media_format] = self.has_media(media_format)
 444
 445             book = self
 446             parents = []
 447             while book.parent:
 448                 parents.append((book.parent.title, book.parent.slug))
 449                 book = book.parent
 450             parents = parents[::-1]
 451             if parents:
 452                 rel['parents'] = parents
 453
 454             if self.pk:
 455                 type(self).objects.filter(pk=self.pk).update(_related_info=rel)
 456             return rel
 457
 458     def related_themes(self):
 459         theme_counter = self.theme_counter
 460         book_themes = list(Tag.objects.filter(pk__in=theme_counter.keys()))
 461         for tag in book_themes:
 462             tag.count = theme_counter[tag.pk]
 463         return book_themes
 464
 465     def reset_tag_counter(self):
 466         if self.id is None:
 467             return
 468
 469         cache_key = "Book.tag_counter/%d" % self.id
 470         permanent_cache.delete(cache_key)
 471         if self.parent:
 472             self.parent.reset_tag_counter()
 473
 474     @property
 475     def tag_counter(self):
 476         if self.id:
 477             cache_key = "Book.tag_counter/%d" % self.id
 478             tags = permanent_cache.get(cache_key)
 479         else:
 480             tags = None
 481
 482         if tags is None:
 483             tags = {}
 484             for child in self.children.all().order_by().iterator():
 485                 for tag_pk, value in child.tag_counter.iteritems():
 486                     tags[tag_pk] = tags.get(tag_pk, 0) + value
 487             for tag in self.tags.exclude(category__in=('book', 'theme', 'set')).order_by().iterator():
 488                 tags[tag.pk] = 1
 489
 490             if self.id:
 491                 permanent_cache.set(cache_key, tags)
 492         return tags
 493
 494     def reset_theme_counter(self):
 495         if self.id is None:
 496             return
 497
 498         cache_key = "Book.theme_counter/%d" % self.id
 499         permanent_cache.delete(cache_key)
 500         if self.parent:
 501             self.parent.reset_theme_counter()
 502
 503     @property
 504     def theme_counter(self):
 505         if self.id:
 506             cache_key = "Book.theme_counter/%d" % self.id
 507             tags = permanent_cache.get(cache_key)
 508         else:
 509             tags = None
 510
 511         if tags is None:
 512             tags = {}
 513             for fragment in Fragment.tagged.with_any([self.book_tag()]).order_by().iterator():
 514                 for tag in fragment.tags.filter(category='theme').order_by().iterator():
 515                     tags[tag.pk] = tags.get(tag.pk, 0) + 1
 516
 517             if self.id:
 518                 permanent_cache.set(cache_key, tags)
 519         return tags
 520
 521     def pretty_title(self, html_links=False):
 522         book = self
 523         names = list(book.tags.filter(category='author'))
 524
 525         books = []
 526         while book:
 527             books.append(book)
 528             book = book.parent
 529         names.extend(reversed(books))
 530
 531         if html_links:
 532             names = ['<a href="%s">%s</a>' % (tag.get_absolute_url(), tag.name) for tag in names]
 533         else:
 534             names = [tag.name for tag in names]
 535
 536         return ', '.join(names)
 537
 538     @classmethod
 539     def tagged_top_level(cls, tags):
 540         """ Returns top-level books tagged with `tags`.
 541
 542         It only returns those books which don't have ancestors which are
 543         also tagged with those tags.
 544
 545         """
 546         # get relevant books and their tags
 547         objects = cls.tagged.with_all(tags)
 548         # eliminate descendants
 549         l_tags = Tag.objects.filter(category='book',
 550             slug__in=[book.book_tag_slug() for book in objects.iterator()])
 551         descendants_keys = [book.pk for book in cls.tagged.with_any(l_tags).iterator()]
 552         if descendants_keys:
 553             objects = objects.exclude(pk__in=descendants_keys)
 554
 555         return objects
 556
 557     @classmethod
 558     def book_list(cls, filter=None):
 559         """Generates a hierarchical listing of all books.
 560
 561         Books are optionally filtered with a test function.
 562
 563         """
 564
 565         books_by_parent = {}
 566         books = cls.objects.all().order_by('parent_number', 'sort_key').only(
 567                 'title', 'parent', 'slug')
 568         if filter:
 569             books = books.filter(filter).distinct()
 570
 571             book_ids = set(b['pk'] for b in books.values("pk").iterator())
 572             for book in books.iterator():
 573                 parent = book.parent_id
 574                 if parent not in book_ids:
 575                     parent = None
 576                 books_by_parent.setdefault(parent, []).append(book)
 577         else:
 578             for book in books.iterator():
 579                 books_by_parent.setdefault(book.parent_id, []).append(book)
 580
 581         orphans = []
 582         books_by_author = SortedDict()
 583         for tag in Tag.objects.filter(category='author').iterator():
 584             books_by_author[tag] = []
 585
 586         for book in books_by_parent.get(None,()):
 587             authors = list(book.tags.filter(category='author'))
 588             if authors:
 589                 for author in authors:
 590                     books_by_author[author].append(book)
 591             else:
 592                 orphans.append(book)
 593
 594         return books_by_author, orphans, books_by_parent
 595
 596     _audiences_pl = {
 597         "SP1": (1, u"szkoła podstawowa"),
 598         "SP2": (1, u"szkoła podstawowa"),
 599         "P": (1, u"szkoła podstawowa"),
 600         "G": (2, u"gimnazjum"),
 601         "L": (3, u"liceum"),
 602         "LP": (3, u"liceum"),
 603     }
 604     def audiences_pl(self):
 605         audiences = self.extra_info.get('audiences', [])
 606         audiences = sorted(set([self._audiences_pl[a] for a in audiences]))
 607         return [a[1] for a in audiences]
 608
 609     def choose_fragment(self):
 610         tag = self.book_tag()
 611         fragments = Fragment.tagged.with_any([tag])
 612         if fragments.exists():
 613             return fragments.order_by('?')[0]
 614         elif self.parent:
 615             return self.parent.choose_fragment()
 616         else:
 617             return None
 618
 619
 620 def _has_factory(ftype):
 621     has = lambda self: bool(getattr(self, "%s_file" % ftype))
 622     has.short_description = ftype.upper()
 623     has.__doc__ = None
 624     has.boolean = True
 625     has.__name__ = "has_%s_file" % ftype
 626     return has
 627
 628
 629 # add the file fields
 630 for t in Book.formats:
 631     field_name = "%s_file" % t
 632     models.FileField(_("%s file" % t.upper()),
 633             upload_to=book_upload_path(t),
 634             blank=True).contribute_to_class(Book, field_name)
 635
 636     setattr(Book, "has_%s_file" % t, _has_factory(t))