apps/catalogue/models/book.py

   1 # -*- coding: utf-8 -*-
   2 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
   3 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
   4 #
   5 import re
   6 from django.conf import settings
   7 from django.core.cache import get_cache
   8 from django.db import models
   9 from django.db.models import permalink
  10 import django.dispatch
  11 from django.utils.datastructures import SortedDict
  12 from django.utils.translation import ugettext_lazy as _
  13 import jsonfield
  14 from catalogue.models import Tag, Fragment, BookMedia
  15 from catalogue.utils import create_zip, split_tags, truncate_html_words, book_upload_path
  16 from catalogue import tasks
  17 from newtagging import managers
  18
  19
  20 permanent_cache = get_cache('permanent')
  21
  22
  23 class Book(models.Model):
  24     """Represents a book imported from WL-XML."""
  25     title         = models.CharField(_('title'), max_length=120)
  26     sort_key = models.CharField(_('sort key'), max_length=120, db_index=True, editable=False)
  27     slug = models.SlugField(_('slug'), max_length=120, db_index=True,
  28             unique=True)
  29     common_slug = models.SlugField(_('slug'), max_length=120, db_index=True)
  30     language = models.CharField(_('language code'), max_length=3, db_index=True,
  31                     default=settings.CATALOGUE_DEFAULT_LANGUAGE)
  32     description   = models.TextField(_('description'), blank=True)
  33     created_at    = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True)
  34     changed_at    = models.DateTimeField(_('creation date'), auto_now=True, db_index=True)
  35     parent_number = models.IntegerField(_('parent number'), default=0)
  36     extra_info    = jsonfield.JSONField(_('extra information'), default='{}')
  37     gazeta_link   = models.CharField(blank=True, max_length=240)
  38     wiki_link     = models.CharField(blank=True, max_length=240)
  39     # files generated during publication
  40
  41     cover = models.FileField(_('cover'), upload_to=book_upload_path('png'),
  42                 null=True, blank=True)
  43     ebook_formats = ['pdf', 'epub', 'mobi', 'fb2', 'txt']
  44     formats = ebook_formats + ['html', 'xml']
  45
  46     parent        = models.ForeignKey('self', blank=True, null=True, related_name='children')
  47
  48     _related_info = jsonfield.JSONField(blank=True, null=True, editable=False)
  49
  50     objects  = models.Manager()
  51     tagged   = managers.ModelTaggedItemManager(Tag)
  52     tags     = managers.TagDescriptor(Tag)
  53
  54     html_built = django.dispatch.Signal()
  55     published = django.dispatch.Signal()
  56
  57     class AlreadyExists(Exception):
  58         pass
  59
  60     class Meta:
  61         ordering = ('sort_key',)
  62         verbose_name = _('book')
  63         verbose_name_plural = _('books')
  64         app_label = 'catalogue'
  65
  66     def __unicode__(self):
  67         return self.title
  68
  69     def save(self, force_insert=False, force_update=False, reset_short_html=True, **kwargs):
  70         from sortify import sortify
  71
  72         self.sort_key = sortify(self.title)
  73
  74         ret = super(Book, self).save(force_insert, force_update)
  75
  76         if reset_short_html:
  77             self.reset_short_html()
  78
  79         return ret
  80
  81     @permalink
  82     def get_absolute_url(self):
  83         return ('catalogue.views.book_detail', [self.slug])
  84
  85     @property
  86     def name(self):
  87         return self.title
  88
  89     def book_tag_slug(self):
  90         return ('l-' + self.slug)[:120]
  91
  92     def book_tag(self):
  93         slug = self.book_tag_slug()
  94         book_tag, created = Tag.objects.get_or_create(slug=slug, category='book')
  95         if created:
  96             book_tag.name = self.title[:50]
  97             book_tag.sort_key = self.title.lower()
  98             book_tag.save()
  99         return book_tag
 100
 101     def has_media(self, type_):
 102         if type_ in Book.formats:
 103             return bool(getattr(self, "%s_file" % type_))
 104         else:
 105             return self.media.filter(type=type_).exists()
 106
 107     def get_media(self, type_):
 108         if self.has_media(type_):
 109             if type_ in Book.formats:
 110                 return getattr(self, "%s_file" % type_)
 111             else:
 112                 return self.media.filter(type=type_)
 113         else:
 114             return None
 115
 116     def get_mp3(self):
 117         return self.get_media("mp3")
 118     def get_odt(self):
 119         return self.get_media("odt")
 120     def get_ogg(self):
 121         return self.get_media("ogg")
 122     def get_daisy(self):
 123         return self.get_media("daisy")
 124
 125     def reset_short_html(self):
 126         if self.id is None:
 127             return
 128
 129         type(self).objects.filter(pk=self.pk).update(_related_info=None)
 130         # Fragment.short_html relies on book's tags, so reset it here too
 131         for fragm in self.fragments.all().iterator():
 132             fragm.reset_short_html()
 133
 134     def has_description(self):
 135         return len(self.description) > 0
 136     has_description.short_description = _('description')
 137     has_description.boolean = True
 138
 139     # ugly ugly ugly
 140     def has_mp3_file(self):
 141         return bool(self.has_media("mp3"))
 142     has_mp3_file.short_description = 'MP3'
 143     has_mp3_file.boolean = True
 144
 145     def has_ogg_file(self):
 146         return bool(self.has_media("ogg"))
 147     has_ogg_file.short_description = 'OGG'
 148     has_ogg_file.boolean = True
 149
 150     def has_daisy_file(self):
 151         return bool(self.has_media("daisy"))
 152     has_daisy_file.short_description = 'DAISY'
 153     has_daisy_file.boolean = True
 154
 155     def wldocument(self, parse_dublincore=True):
 156         from catalogue.import_utils import ORMDocProvider
 157         from librarian.parser import WLDocument
 158
 159         return WLDocument.from_file(self.xml_file.path,
 160                 provider=ORMDocProvider(self),
 161                 parse_dublincore=parse_dublincore)
 162
 163     def build_cover(self, book_info=None):
 164         """(Re)builds the cover image."""
 165         from StringIO import StringIO
 166         from django.core.files.base import ContentFile
 167         from librarian.cover import WLCover
 168
 169         if book_info is None:
 170             book_info = self.wldocument().book_info
 171
 172         cover = WLCover(book_info).image()
 173         imgstr = StringIO()
 174         cover.save(imgstr, 'png')
 175         self.cover.save(None, ContentFile(imgstr.getvalue()))
 176
 177     def build_html(self):
 178         from django.core.files.base import ContentFile
 179         from slughifi import slughifi
 180         from sortify import sortify
 181         from librarian import html
 182
 183         meta_tags = list(self.tags.filter(
 184             category__in=('author', 'epoch', 'genre', 'kind')))
 185         book_tag = self.book_tag()
 186
 187         html_output = self.wldocument(parse_dublincore=False).as_html()
 188         if html_output:
 189             self.html_file.save('%s.html' % self.slug,
 190                     ContentFile(html_output.get_string()))
 191
 192             # get ancestor l-tags for adding to new fragments
 193             ancestor_tags = []
 194             p = self.parent
 195             while p:
 196                 ancestor_tags.append(p.book_tag())
 197                 p = p.parent
 198
 199             # Delete old fragments and create them from scratch
 200             self.fragments.all().delete()
 201             # Extract fragments
 202             closed_fragments, open_fragments = html.extract_fragments(self.html_file.path)
 203             for fragment in closed_fragments.values():
 204                 try:
 205                     theme_names = [s.strip() for s in fragment.themes.split(',')]
 206                 except AttributeError:
 207                     continue
 208                 themes = []
 209                 for theme_name in theme_names:
 210                     if not theme_name:
 211                         continue
 212                     tag, created = Tag.objects.get_or_create(slug=slughifi(theme_name), category='theme')
 213                     if created:
 214                         tag.name = theme_name
 215                         tag.sort_key = sortify(theme_name.lower())
 216                         tag.save()
 217                     themes.append(tag)
 218                 if not themes:
 219                     continue
 220
 221                 text = fragment.to_string()
 222                 short_text = truncate_html_words(text, 15)
 223                 if text == short_text:
 224                     short_text = ''
 225                 new_fragment = Fragment.objects.create(anchor=fragment.id, book=self,
 226                     text=text, short_text=short_text)
 227
 228                 new_fragment.save()
 229                 new_fragment.tags = set(meta_tags + themes + [book_tag] + ancestor_tags)
 230             self.save()
 231             self.html_built.send(sender=self)
 232             return True
 233         return False
 234
 235     # Thin wrappers for builder tasks
 236     def build_pdf(self, *args, **kwargs):
 237         """(Re)builds PDF."""
 238         return tasks.build_pdf.delay(self.pk, *args, **kwargs)
 239     def build_epub(self, *args, **kwargs):
 240         """(Re)builds EPUB."""
 241         return tasks.build_epub.delay(self.pk, *args, **kwargs)
 242     def build_mobi(self, *args, **kwargs):
 243         """(Re)builds MOBI."""
 244         return tasks.build_mobi.delay(self.pk, *args, **kwargs)
 245     def build_fb2(self, *args, **kwargs):
 246         """(Re)build FB2"""
 247         return tasks.build_fb2.delay(self.pk, *args, **kwargs)
 248     def build_txt(self, *args, **kwargs):
 249         """(Re)builds TXT."""
 250         return tasks.build_txt.delay(self.pk, *args, **kwargs)
 251
 252     @staticmethod
 253     def zip_format(format_):
 254         def pretty_file_name(book):
 255             return "%s/%s.%s" % (
 256                 b.extra_info['author'],
 257                 b.slug,
 258                 format_)
 259
 260         field_name = "%s_file" % format_
 261         books = Book.objects.filter(parent=None).exclude(**{field_name: ""})
 262         paths = [(pretty_file_name(b), getattr(b, field_name).path)
 263                     for b in books.iterator()]
 264         return create_zip(paths,
 265                     getattr(settings, "ALL_%s_ZIP" % format_.upper()))
 266
 267     def zip_audiobooks(self, format_):
 268         bm = BookMedia.objects.filter(book=self, type=format_)
 269         paths = map(lambda bm: (None, bm.file.path), bm)
 270         return create_zip(paths, "%s_%s" % (self.slug, format_))
 271
 272     def search_index(self, book_info=None, reuse_index=False, index_tags=True):
 273         import search
 274         if reuse_index:
 275             idx = search.ReusableIndex()
 276         else:
 277             idx = search.Index()
 278
 279         idx.open()
 280         try:
 281             idx.index_book(self, book_info)
 282             if index_tags:
 283                 idx.index_tags()
 284         finally:
 285             idx.close()
 286
 287     @classmethod
 288     def from_xml_file(cls, xml_file, **kwargs):
 289         from django.core.files import File
 290         from librarian import dcparser
 291
 292         # use librarian to parse meta-data
 293         book_info = dcparser.parse(xml_file)
 294
 295         if not isinstance(xml_file, File):
 296             xml_file = File(open(xml_file))
 297
 298         try:
 299             return cls.from_text_and_meta(xml_file, book_info, **kwargs)
 300         finally:
 301             xml_file.close()
 302
 303     @classmethod
 304     def from_text_and_meta(cls, raw_file, book_info, overwrite=False,
 305             build_epub=True, build_txt=True, build_pdf=True, build_mobi=True, build_fb2=True,
 306             search_index=True, search_index_tags=True, search_index_reuse=False):
 307
 308         # check for parts before we do anything
 309         children = []
 310         if hasattr(book_info, 'parts'):
 311             for part_url in book_info.parts:
 312                 try:
 313                     children.append(Book.objects.get(slug=part_url.slug))
 314                 except Book.DoesNotExist:
 315                     raise Book.DoesNotExist(_('Book "%s" does not exist.') %
 316                             part_url.slug)
 317
 318
 319         # Read book metadata
 320         book_slug = book_info.url.slug
 321         if re.search(r'[^a-z0-9-]', book_slug):
 322             raise ValueError('Invalid characters in slug')
 323         book, created = Book.objects.get_or_create(slug=book_slug)
 324
 325         if created:
 326             book_shelves = []
 327         else:
 328             if not overwrite:
 329                 raise Book.AlreadyExists(_('Book %s already exists') % (
 330                         book_slug))
 331             # Save shelves for this book
 332             book_shelves = list(book.tags.filter(category='set'))
 333
 334         book.language = book_info.language
 335         book.title = book_info.title
 336         if book_info.variant_of:
 337             book.common_slug = book_info.variant_of.slug
 338         else:
 339             book.common_slug = book.slug
 340         book.extra_info = book_info.to_dict()
 341         book.save()
 342
 343         meta_tags = Tag.tags_from_info(book_info)
 344
 345         book.tags = set(meta_tags + book_shelves)
 346
 347         book_tag = book.book_tag()
 348
 349         for n, child_book in enumerate(children):
 350             child_book.parent = book
 351             child_book.parent_number = n
 352             child_book.save()
 353
 354         # Save XML and HTML files
 355         book.xml_file.save('%s.xml' % book.slug, raw_file, save=False)
 356         book.build_cover(book_info)
 357
 358         # delete old fragments when overwriting
 359         book.fragments.all().delete()
 360
 361         if book.build_html():
 362             # No direct saves behind this point.
 363             if not settings.NO_BUILD_TXT and build_txt:
 364                 book.build_txt()
 365
 366         if not settings.NO_BUILD_EPUB and build_epub:
 367             book.build_epub()
 368
 369         if not settings.NO_BUILD_PDF and build_pdf:
 370             book.build_pdf()
 371
 372         if not settings.NO_BUILD_MOBI and build_mobi:
 373             book.build_mobi()
 374
 375         if not settings.NO_BUILD_FB2 and build_fb2:
 376             book.build_fb2()
 377
 378         if not settings.NO_SEARCH_INDEX and search_index:
 379             book.search_index(index_tags=search_index_tags, reuse_index=search_index_reuse)
 380             #index_book.delay(book.id, book_info)
 381
 382         book_descendants = list(book.children.all())
 383         descendants_tags = set()
 384         # add l-tag to descendants and their fragments
 385         while len(book_descendants) > 0:
 386             child_book = book_descendants.pop(0)
 387             descendants_tags.update(child_book.tags)
 388             child_book.tags = list(child_book.tags) + [book_tag]
 389             child_book.save()
 390             for fragment in child_book.fragments.all().iterator():
 391                 fragment.tags = set(list(fragment.tags) + [book_tag])
 392             book_descendants += list(child_book.children.all())
 393
 394         for tag in descendants_tags:
 395             tasks.touch_tag(tag)
 396
 397         # refresh cache
 398         book.reset_tag_counter()
 399         book.reset_theme_counter()
 400
 401         cls.published.send(sender=book)
 402         return book
 403
 404     def related_info(self):
 405         """Keeps info about related objects (tags, media) in cache field."""
 406         if self._related_info is not None:
 407             return self._related_info
 408         else:
 409             rel = {'tags': {}, 'media': {}}
 410
 411             tags = self.tags.filter(category__in=(
 412                     'author', 'kind', 'genre', 'epoch'))
 413             tags = split_tags(tags)
 414             for category in tags:
 415                 rel['tags'][category] = [
 416                         (t.name, t.slug) for t in tags[category]]
 417
 418             for media_format in BookMedia.formats:
 419                 rel['media'][media_format] = self.has_media(media_format)
 420
 421             book = self
 422             parents = []
 423             while book.parent:
 424                 parents.append((book.parent.title, book.parent.slug))
 425                 book = book.parent
 426             parents = parents[::-1]
 427             if parents:
 428                 rel['parents'] = parents
 429
 430             if self.pk:
 431                 type(self).objects.filter(pk=self.pk).update(_related_info=rel)
 432             return rel
 433
 434     def related_themes(self):
 435         theme_counter = self.theme_counter
 436         book_themes = list(Tag.objects.filter(pk__in=theme_counter.keys()))
 437         for tag in book_themes:
 438             tag.count = theme_counter[tag.pk]
 439         return book_themes
 440
 441     def reset_tag_counter(self):
 442         if self.id is None:
 443             return
 444
 445         cache_key = "Book.tag_counter/%d" % self.id
 446         permanent_cache.delete(cache_key)
 447         if self.parent:
 448             self.parent.reset_tag_counter()
 449
 450     @property
 451     def tag_counter(self):
 452         if self.id:
 453             cache_key = "Book.tag_counter/%d" % self.id
 454             tags = permanent_cache.get(cache_key)
 455         else:
 456             tags = None
 457
 458         if tags is None:
 459             tags = {}
 460             for child in self.children.all().order_by().iterator():
 461                 for tag_pk, value in child.tag_counter.iteritems():
 462                     tags[tag_pk] = tags.get(tag_pk, 0) + value
 463             for tag in self.tags.exclude(category__in=('book', 'theme', 'set')).order_by().iterator():
 464                 tags[tag.pk] = 1
 465
 466             if self.id:
 467                 permanent_cache.set(cache_key, tags)
 468         return tags
 469
 470     def reset_theme_counter(self):
 471         if self.id is None:
 472             return
 473
 474         cache_key = "Book.theme_counter/%d" % self.id
 475         permanent_cache.delete(cache_key)
 476         if self.parent:
 477             self.parent.reset_theme_counter()
 478
 479     @property
 480     def theme_counter(self):
 481         if self.id:
 482             cache_key = "Book.theme_counter/%d" % self.id
 483             tags = permanent_cache.get(cache_key)
 484         else:
 485             tags = None
 486
 487         if tags is None:
 488             tags = {}
 489             for fragment in Fragment.tagged.with_any([self.book_tag()]).order_by().iterator():
 490                 for tag in fragment.tags.filter(category='theme').order_by().iterator():
 491                     tags[tag.pk] = tags.get(tag.pk, 0) + 1
 492
 493             if self.id:
 494                 permanent_cache.set(cache_key, tags)
 495         return tags
 496
 497     def pretty_title(self, html_links=False):
 498         book = self
 499         names = list(book.tags.filter(category='author'))
 500
 501         books = []
 502         while book:
 503             books.append(book)
 504             book = book.parent
 505         names.extend(reversed(books))
 506
 507         if html_links:
 508             names = ['<a href="%s">%s</a>' % (tag.get_absolute_url(), tag.name) for tag in names]
 509         else:
 510             names = [tag.name for tag in names]
 511
 512         return ', '.join(names)
 513
 514     @classmethod
 515     def tagged_top_level(cls, tags):
 516         """ Returns top-level books tagged with `tags`.
 517
 518         It only returns those books which don't have ancestors which are
 519         also tagged with those tags.
 520
 521         """
 522         # get relevant books and their tags
 523         objects = cls.tagged.with_all(tags)
 524         # eliminate descendants
 525         l_tags = Tag.objects.filter(category='book',
 526             slug__in=[book.book_tag_slug() for book in objects.iterator()])
 527         descendants_keys = [book.pk for book in cls.tagged.with_any(l_tags).iterator()]
 528         if descendants_keys:
 529             objects = objects.exclude(pk__in=descendants_keys)
 530
 531         return objects
 532
 533     @classmethod
 534     def book_list(cls, filter=None):
 535         """Generates a hierarchical listing of all books.
 536
 537         Books are optionally filtered with a test function.
 538
 539         """
 540
 541         books_by_parent = {}
 542         books = cls.objects.all().order_by('parent_number', 'sort_key').only(
 543                 'title', 'parent', 'slug')
 544         if filter:
 545             books = books.filter(filter).distinct()
 546
 547             book_ids = set(b['pk'] for b in books.values("pk").iterator())
 548             for book in books.iterator():
 549                 parent = book.parent_id
 550                 if parent not in book_ids:
 551                     parent = None
 552                 books_by_parent.setdefault(parent, []).append(book)
 553         else:
 554             for book in books.iterator():
 555                 books_by_parent.setdefault(book.parent_id, []).append(book)
 556
 557         orphans = []
 558         books_by_author = SortedDict()
 559         for tag in Tag.objects.filter(category='author').iterator():
 560             books_by_author[tag] = []
 561
 562         for book in books_by_parent.get(None,()):
 563             authors = list(book.tags.filter(category='author'))
 564             if authors:
 565                 for author in authors:
 566                     books_by_author[author].append(book)
 567             else:
 568                 orphans.append(book)
 569
 570         return books_by_author, orphans, books_by_parent
 571
 572     _audiences_pl = {
 573         "SP1": (1, u"szkoła podstawowa"),
 574         "SP2": (1, u"szkoła podstawowa"),
 575         "P": (1, u"szkoła podstawowa"),
 576         "G": (2, u"gimnazjum"),
 577         "L": (3, u"liceum"),
 578         "LP": (3, u"liceum"),
 579     }
 580     def audiences_pl(self):
 581         audiences = self.extra_info.get('audiences', [])
 582         audiences = sorted(set([self._audiences_pl[a] for a in audiences]))
 583         return [a[1] for a in audiences]
 584
 585     def choose_fragment(self):
 586         tag = self.book_tag()
 587         fragments = Fragment.tagged.with_any([tag])
 588         if fragments.exists():
 589             return fragments.order_by('?')[0]
 590         elif self.parent:
 591             return self.parent.choose_fragment()
 592         else:
 593             return None
 594
 595
 596 def _has_factory(ftype):
 597     has = lambda self: bool(getattr(self, "%s_file" % ftype))
 598     has.short_description = ftype.upper()
 599     has.__doc__ = None
 600     has.boolean = True
 601     has.__name__ = "has_%s_file" % ftype
 602     return has
 603
 604
 605 # add the file fields
 606 for t in Book.formats:
 607     field_name = "%s_file" % t
 608     models.FileField(_("%s file" % t.upper()),
 609             upload_to=book_upload_path(t),
 610             blank=True).contribute_to_class(Book, field_name)
 611
 612     setattr(Book, "has_%s_file" % t, _has_factory(t))