apps/catalogue/models/book.py

   1 # -*- coding: utf-8 -*-
   2 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
   3 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
   4 #
   5 import re
   6 from django.conf import settings
   7 from django.core.cache import get_cache
   8 from django.db import models
   9 from django.db.models import permalink
  10 import django.dispatch
  11 from django.utils.datastructures import SortedDict
  12 from django.utils.translation import ugettext_lazy as _
  13 import jsonfield
  14 from catalogue.models import Tag, Fragment, BookMedia
  15 from catalogue.utils import create_zip, split_tags, truncate_html_words, book_upload_path
  16 from catalogue import tasks
  17 from newtagging import managers
  18
  19
  20 permanent_cache = get_cache('permanent')
  21
  22
  23 class Book(models.Model):
  24     """Represents a book imported from WL-XML."""
  25     title         = models.CharField(_('title'), max_length=120)
  26     sort_key = models.CharField(_('sort key'), max_length=120, db_index=True, editable=False)
  27     slug = models.SlugField(_('slug'), max_length=120, db_index=True,
  28             unique=True)
  29     common_slug = models.SlugField(_('slug'), max_length=120, db_index=True)
  30     language = models.CharField(_('language code'), max_length=3, db_index=True,
  31                     default=settings.CATALOGUE_DEFAULT_LANGUAGE)
  32     description   = models.TextField(_('description'), blank=True)
  33     created_at    = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True)
  34     changed_at    = models.DateTimeField(_('creation date'), auto_now=True, db_index=True)
  35     parent_number = models.IntegerField(_('parent number'), default=0)
  36     extra_info    = jsonfield.JSONField(_('extra information'), default='{}')
  37     gazeta_link   = models.CharField(blank=True, max_length=240)
  38     wiki_link     = models.CharField(blank=True, max_length=240)
  39     # files generated during publication
  40
  41     cover = models.FileField(_('cover'), upload_to=book_upload_path('png'),
  42                 null=True, blank=True)
  43     ebook_formats = ['pdf', 'epub', 'mobi', 'fb2', 'txt']
  44     formats = ebook_formats + ['html', 'xml']
  45
  46     parent        = models.ForeignKey('self', blank=True, null=True, related_name='children')
  47
  48     _related_info = jsonfield.JSONField(blank=True, null=True, editable=False)
  49
  50     objects  = models.Manager()
  51     tagged   = managers.ModelTaggedItemManager(Tag)
  52     tags     = managers.TagDescriptor(Tag)
  53
  54     html_built = django.dispatch.Signal()
  55     published = django.dispatch.Signal()
  56
  57     class AlreadyExists(Exception):
  58         pass
  59
  60     class Meta:
  61         ordering = ('sort_key',)
  62         verbose_name = _('book')
  63         verbose_name_plural = _('books')
  64         app_label = 'catalogue'
  65
  66     def __unicode__(self):
  67         return self.title
  68
  69     def save(self, force_insert=False, force_update=False, reset_short_html=True, **kwargs):
  70         from sortify import sortify
  71
  72         self.sort_key = sortify(self.title)
  73
  74         ret = super(Book, self).save(force_insert, force_update)
  75
  76         if reset_short_html:
  77             self.reset_short_html()
  78
  79         return ret
  80
  81     @permalink
  82     def get_absolute_url(self):
  83         return ('catalogue.views.book_detail', [self.slug])
  84
  85     @property
  86     def name(self):
  87         return self.title
  88
  89     def book_tag_slug(self):
  90         return ('l-' + self.slug)[:120]
  91
  92     def book_tag(self):
  93         slug = self.book_tag_slug()
  94         book_tag, created = Tag.objects.get_or_create(slug=slug, category='book')
  95         if created:
  96             book_tag.name = self.title[:50]
  97             book_tag.sort_key = self.title.lower()
  98             book_tag.save()
  99         return book_tag
 100
 101     def has_media(self, type_):
 102         if type_ in Book.formats:
 103             return bool(getattr(self, "%s_file" % type_))
 104         else:
 105             return self.media.filter(type=type_).exists()
 106
 107     def get_media(self, type_):
 108         if self.has_media(type_):
 109             if type_ in Book.formats:
 110                 return getattr(self, "%s_file" % type_)
 111             else:
 112                 return self.media.filter(type=type_)
 113         else:
 114             return None
 115
 116     def get_mp3(self):
 117         return self.get_media("mp3")
 118     def get_odt(self):
 119         return self.get_media("odt")
 120     def get_ogg(self):
 121         return self.get_media("ogg")
 122     def get_daisy(self):
 123         return self.get_media("daisy")
 124
 125     def reset_short_html(self):
 126         if self.id is None:
 127             return
 128
 129         type(self).objects.filter(pk=self.pk).update(_related_info=None)
 130         # Fragment.short_html relies on book's tags, so reset it here too
 131         for fragm in self.fragments.all().iterator():
 132             fragm.reset_short_html()
 133
 134     def has_description(self):
 135         return len(self.description) > 0
 136     has_description.short_description = _('description')
 137     has_description.boolean = True
 138
 139     # ugly ugly ugly
 140     def has_mp3_file(self):
 141         return bool(self.has_media("mp3"))
 142     has_mp3_file.short_description = 'MP3'
 143     has_mp3_file.boolean = True
 144
 145     def has_ogg_file(self):
 146         return bool(self.has_media("ogg"))
 147     has_ogg_file.short_description = 'OGG'
 148     has_ogg_file.boolean = True
 149
 150     def has_daisy_file(self):
 151         return bool(self.has_media("daisy"))
 152     has_daisy_file.short_description = 'DAISY'
 153     has_daisy_file.boolean = True
 154
 155     def wldocument(self, parse_dublincore=True):
 156         from catalogue.import_utils import ORMDocProvider
 157         from librarian.parser import WLDocument
 158
 159         return WLDocument.from_file(self.xml_file.path,
 160                 provider=ORMDocProvider(self),
 161                 parse_dublincore=parse_dublincore)
 162
 163     def build_cover(self, book_info=None):
 164         """(Re)builds the cover image."""
 165         from StringIO import StringIO
 166         from django.core.files.base import ContentFile
 167         from librarian.cover import WLCover
 168
 169         if book_info is None:
 170             book_info = self.wldocument().book_info
 171
 172         cover = WLCover(book_info).image()
 173         imgstr = StringIO()
 174         cover.save(imgstr, 'png')
 175         self.cover.save(None, ContentFile(imgstr.getvalue()))
 176
 177     def build_html(self):
 178         from django.core.files.base import ContentFile
 179         from slughifi import slughifi
 180         from sortify import sortify
 181         from librarian import html
 182
 183         meta_tags = list(self.tags.filter(
 184             category__in=('author', 'epoch', 'genre', 'kind')))
 185         book_tag = self.book_tag()
 186
 187         html_output = self.wldocument(parse_dublincore=False).as_html()
 188         if html_output:
 189             self.html_file.save('%s.html' % self.slug,
 190                     ContentFile(html_output.get_string()))
 191
 192             # get ancestor l-tags for adding to new fragments
 193             ancestor_tags = []
 194             p = self.parent
 195             while p:
 196                 ancestor_tags.append(p.book_tag())
 197                 p = p.parent
 198
 199             # Delete old fragments and create them from scratch
 200             self.fragments.all().delete()
 201             # Extract fragments
 202             closed_fragments, open_fragments = html.extract_fragments(self.html_file.path)
 203             for fragment in closed_fragments.values():
 204                 try:
 205                     theme_names = [s.strip() for s in fragment.themes.split(',')]
 206                 except AttributeError:
 207                     continue
 208                 themes = []
 209                 for theme_name in theme_names:
 210                     if not theme_name:
 211                         continue
 212                     tag, created = Tag.objects.get_or_create(slug=slughifi(theme_name), category='theme')
 213                     if created:
 214                         tag.name = theme_name
 215                         tag.sort_key = sortify(theme_name.lower())
 216                         tag.save()
 217                     themes.append(tag)
 218                 if not themes:
 219                     continue
 220
 221                 text = fragment.to_string()
 222                 short_text = truncate_html_words(text, 15)
 223                 if text == short_text:
 224                     short_text = ''
 225                 new_fragment = Fragment.objects.create(anchor=fragment.id, book=self,
 226                     text=text, short_text=short_text)
 227
 228                 new_fragment.save()
 229                 new_fragment.tags = set(meta_tags + themes + [book_tag] + ancestor_tags)
 230             self.save()
 231             self.html_built.send(sender=self)
 232             return True
 233         return False
 234
 235     # Thin wrappers for builder tasks
 236     def build_pdf(self, *args, **kwargs):
 237         """(Re)builds PDF."""
 238         return tasks.build_pdf.delay(self.pk, *args, **kwargs)
 239     def build_epub(self, *args, **kwargs):
 240         """(Re)builds EPUB."""
 241         return tasks.build_epub.delay(self.pk, *args, **kwargs)
 242     def build_mobi(self, *args, **kwargs):
 243         """(Re)builds MOBI."""
 244         return tasks.build_mobi.delay(self.pk, *args, **kwargs)
 245     def build_fb2(self, *args, **kwargs):
 246         """(Re)build FB2"""
 247         return tasks.build_fb2.delay(self.pk, *args, **kwargs)
 248     def build_txt(self, *args, **kwargs):
 249         """(Re)builds TXT."""
 250         return tasks.build_txt.delay(self.pk, *args, **kwargs)
 251
 252     @staticmethod
 253     def zip_format(format_):
 254         def pretty_file_name(book):
 255             return "%s/%s.%s" % (
 256                 b.extra_info['author'],
 257                 b.slug,
 258                 format_)
 259
 260         field_name = "%s_file" % format_
 261         books = Book.objects.filter(parent=None).exclude(**{field_name: ""})
 262         paths = [(pretty_file_name(b), getattr(b, field_name).path)
 263                     for b in books.iterator()]
 264         return create_zip(paths,
 265                     getattr(settings, "ALL_%s_ZIP" % format_.upper()))
 266
 267     def zip_audiobooks(self, format_):
 268         bm = BookMedia.objects.filter(book=self, type=format_)
 269         paths = map(lambda bm: (None, bm.file.path), bm)
 270         return create_zip(paths, "%s_%s" % (self.slug, format_))
 271
 272     def search_index(self, book_info=None, reuse_index=False, index_tags=True):
 273         import search
 274         if reuse_index:
 275             idx = search.ReusableIndex()
 276         else:
 277             idx = search.Index()
 278
 279         idx.open()
 280         try:
 281             idx.index_book(self, book_info)
 282             if index_tags:
 283                 idx.index_tags()
 284         finally:
 285             idx.close()
 286
 287     @classmethod
 288     def from_xml_file(cls, xml_file, **kwargs):
 289         from django.core.files import File
 290         from librarian import dcparser
 291
 292         # use librarian to parse meta-data
 293         book_info = dcparser.parse(xml_file)
 294
 295         if not isinstance(xml_file, File):
 296             xml_file = File(open(xml_file))
 297
 298         try:
 299             return cls.from_text_and_meta(xml_file, book_info, **kwargs)
 300         finally:
 301             xml_file.close()
 302
 303     @classmethod
 304     def from_text_and_meta(cls, raw_file, book_info, overwrite=False,
 305             build_epub=True, build_txt=True, build_pdf=True, build_mobi=True, build_fb2=True,
 306             search_index=True, search_index_tags=True, search_index_reuse=False):
 307
 308         # check for parts before we do anything
 309         children = []
 310         if hasattr(book_info, 'parts'):
 311             for part_url in book_info.parts:
 312                 try:
 313                     children.append(Book.objects.get(slug=part_url.slug))
 314                 except Book.DoesNotExist:
 315                     raise Book.DoesNotExist(_('Book "%s" does not exist.') %
 316                             part_url.slug)
 317
 318
 319         # Read book metadata
 320         book_slug = book_info.url.slug
 321         if re.search(r'[^a-z0-9-]', book_slug):
 322             raise ValueError('Invalid characters in slug')
 323         book, created = Book.objects.get_or_create(slug=book_slug)
 324
 325         if created:
 326             book_shelves = []
 327         else:
 328             if not overwrite:
 329                 raise Book.AlreadyExists(_('Book %s already exists') % (
 330                         book_slug))
 331             # Save shelves for this book
 332             book_shelves = list(book.tags.filter(category='set'))
 333
 334         book.language = book_info.language
 335         book.title = book_info.title
 336         if book_info.variant_of:
 337             book.common_slug = book_info.variant_of.slug
 338         else:
 339             book.common_slug = book.slug
 340         book.extra_info = book_info.to_dict()
 341         book.save()
 342
 343         meta_tags = Tag.tags_from_info(book_info)
 344
 345         book.tags = set(meta_tags + book_shelves)
 346
 347         book_tag = book.book_tag()
 348
 349         for n, child_book in enumerate(children):
 350             child_book.parent = book
 351             child_book.parent_number = n
 352             child_book.save()
 353
 354         # Save XML and HTML files
 355         book.xml_file.save('%s.xml' % book.slug, raw_file, save=False)
 356
 357         # delete old fragments when overwriting
 358         book.fragments.all().delete()
 359
 360         if book.build_html():
 361             if not settings.NO_BUILD_TXT and build_txt:
 362                 book.build_txt()
 363
 364         book.build_cover(book_info)
 365
 366         if not settings.NO_BUILD_EPUB and build_epub:
 367             book.build_epub()
 368
 369         if not settings.NO_BUILD_PDF and build_pdf:
 370             book.build_pdf()
 371
 372         if not settings.NO_BUILD_MOBI and build_mobi:
 373             book.build_mobi()
 374
 375         if not settings.NO_BUILD_FB2 and build_fb2:
 376             book.build_fb2()
 377
 378         if not settings.NO_SEARCH_INDEX and search_index:
 379             book.search_index(index_tags=search_index_tags, reuse_index=search_index_reuse)
 380             #index_book.delay(book.id, book_info)
 381
 382         book_descendants = list(book.children.all())
 383         descendants_tags = set()
 384         # add l-tag to descendants and their fragments
 385         while len(book_descendants) > 0:
 386             child_book = book_descendants.pop(0)
 387             descendants_tags.update(child_book.tags)
 388             child_book.tags = list(child_book.tags) + [book_tag]
 389             child_book.save()
 390             for fragment in child_book.fragments.all().iterator():
 391                 fragment.tags = set(list(fragment.tags) + [book_tag])
 392             book_descendants += list(child_book.children.all())
 393
 394         for tag in descendants_tags:
 395             tasks.touch_tag(tag)
 396
 397         book.save()
 398
 399         # refresh cache
 400         book.reset_tag_counter()
 401         book.reset_theme_counter()
 402
 403         cls.published.send(sender=book)
 404         return book
 405
 406     def related_info(self):
 407         """Keeps info about related objects (tags, media) in cache field."""
 408         if self._related_info is not None:
 409             return self._related_info
 410         else:
 411             rel = {'tags': {}, 'media': {}}
 412
 413             tags = self.tags.filter(category__in=(
 414                     'author', 'kind', 'genre', 'epoch'))
 415             tags = split_tags(tags)
 416             for category in tags:
 417                 rel['tags'][category] = [
 418                         (t.name, t.slug) for t in tags[category]]
 419
 420             for media_format in BookMedia.formats:
 421                 rel['media'][media_format] = self.has_media(media_format)
 422
 423             book = self
 424             parents = []
 425             while book.parent:
 426                 parents.append((book.parent.title, book.parent.slug))
 427                 book = book.parent
 428             parents = parents[::-1]
 429             if parents:
 430                 rel['parents'] = parents
 431
 432             if self.pk:
 433                 type(self).objects.filter(pk=self.pk).update(_related_info=rel)
 434             return rel
 435
 436     def related_themes(self):
 437         theme_counter = self.theme_counter
 438         book_themes = list(Tag.objects.filter(pk__in=theme_counter.keys()))
 439         for tag in book_themes:
 440             tag.count = theme_counter[tag.pk]
 441         return book_themes
 442
 443     def reset_tag_counter(self):
 444         if self.id is None:
 445             return
 446
 447         cache_key = "Book.tag_counter/%d" % self.id
 448         permanent_cache.delete(cache_key)
 449         if self.parent:
 450             self.parent.reset_tag_counter()
 451
 452     @property
 453     def tag_counter(self):
 454         if self.id:
 455             cache_key = "Book.tag_counter/%d" % self.id
 456             tags = permanent_cache.get(cache_key)
 457         else:
 458             tags = None
 459
 460         if tags is None:
 461             tags = {}
 462             for child in self.children.all().order_by().iterator():
 463                 for tag_pk, value in child.tag_counter.iteritems():
 464                     tags[tag_pk] = tags.get(tag_pk, 0) + value
 465             for tag in self.tags.exclude(category__in=('book', 'theme', 'set')).order_by().iterator():
 466                 tags[tag.pk] = 1
 467
 468             if self.id:
 469                 permanent_cache.set(cache_key, tags)
 470         return tags
 471
 472     def reset_theme_counter(self):
 473         if self.id is None:
 474             return
 475
 476         cache_key = "Book.theme_counter/%d" % self.id
 477         permanent_cache.delete(cache_key)
 478         if self.parent:
 479             self.parent.reset_theme_counter()
 480
 481     @property
 482     def theme_counter(self):
 483         if self.id:
 484             cache_key = "Book.theme_counter/%d" % self.id
 485             tags = permanent_cache.get(cache_key)
 486         else:
 487             tags = None
 488
 489         if tags is None:
 490             tags = {}
 491             for fragment in Fragment.tagged.with_any([self.book_tag()]).order_by().iterator():
 492                 for tag in fragment.tags.filter(category='theme').order_by().iterator():
 493                     tags[tag.pk] = tags.get(tag.pk, 0) + 1
 494
 495             if self.id:
 496                 permanent_cache.set(cache_key, tags)
 497         return tags
 498
 499     def pretty_title(self, html_links=False):
 500         book = self
 501         names = list(book.tags.filter(category='author'))
 502
 503         books = []
 504         while book:
 505             books.append(book)
 506             book = book.parent
 507         names.extend(reversed(books))
 508
 509         if html_links:
 510             names = ['<a href="%s">%s</a>' % (tag.get_absolute_url(), tag.name) for tag in names]
 511         else:
 512             names = [tag.name for tag in names]
 513
 514         return ', '.join(names)
 515
 516     @classmethod
 517     def tagged_top_level(cls, tags):
 518         """ Returns top-level books tagged with `tags`.
 519
 520         It only returns those books which don't have ancestors which are
 521         also tagged with those tags.
 522
 523         """
 524         # get relevant books and their tags
 525         objects = cls.tagged.with_all(tags)
 526         # eliminate descendants
 527         l_tags = Tag.objects.filter(category='book',
 528             slug__in=[book.book_tag_slug() for book in objects.iterator()])
 529         descendants_keys = [book.pk for book in cls.tagged.with_any(l_tags).iterator()]
 530         if descendants_keys:
 531             objects = objects.exclude(pk__in=descendants_keys)
 532
 533         return objects
 534
 535     @classmethod
 536     def book_list(cls, filter=None):
 537         """Generates a hierarchical listing of all books.
 538
 539         Books are optionally filtered with a test function.
 540
 541         """
 542
 543         books_by_parent = {}
 544         books = cls.objects.all().order_by('parent_number', 'sort_key').only(
 545                 'title', 'parent', 'slug')
 546         if filter:
 547             books = books.filter(filter).distinct()
 548
 549             book_ids = set(b['pk'] for b in books.values("pk").iterator())
 550             for book in books.iterator():
 551                 parent = book.parent_id
 552                 if parent not in book_ids:
 553                     parent = None
 554                 books_by_parent.setdefault(parent, []).append(book)
 555         else:
 556             for book in books.iterator():
 557                 books_by_parent.setdefault(book.parent_id, []).append(book)
 558
 559         orphans = []
 560         books_by_author = SortedDict()
 561         for tag in Tag.objects.filter(category='author').iterator():
 562             books_by_author[tag] = []
 563
 564         for book in books_by_parent.get(None,()):
 565             authors = list(book.tags.filter(category='author'))
 566             if authors:
 567                 for author in authors:
 568                     books_by_author[author].append(book)
 569             else:
 570                 orphans.append(book)
 571
 572         return books_by_author, orphans, books_by_parent
 573
 574     _audiences_pl = {
 575         "SP1": (1, u"szkoła podstawowa"),
 576         "SP2": (1, u"szkoła podstawowa"),
 577         "P": (1, u"szkoła podstawowa"),
 578         "G": (2, u"gimnazjum"),
 579         "L": (3, u"liceum"),
 580         "LP": (3, u"liceum"),
 581     }
 582     def audiences_pl(self):
 583         audiences = self.extra_info.get('audiences', [])
 584         audiences = sorted(set([self._audiences_pl[a] for a in audiences]))
 585         return [a[1] for a in audiences]
 586
 587     def choose_fragment(self):
 588         tag = self.book_tag()
 589         fragments = Fragment.tagged.with_any([tag])
 590         if fragments.exists():
 591             return fragments.order_by('?')[0]
 592         elif self.parent:
 593             return self.parent.choose_fragment()
 594         else:
 595             return None
 596
 597
 598 def _has_factory(ftype):
 599     has = lambda self: bool(getattr(self, "%s_file" % ftype))
 600     has.short_description = ftype.upper()
 601     has.__doc__ = None
 602     has.boolean = True
 603     has.__name__ = "has_%s_file" % ftype
 604     return has
 605
 606
 607 # add the file fields
 608 for t in Book.formats:
 609     field_name = "%s_file" % t
 610     models.FileField(_("%s file" % t.upper()),
 611             upload_to=book_upload_path(t),
 612             blank=True).contribute_to_class(Book, field_name)
 613
 614     setattr(Book, "has_%s_file" % t, _has_factory(t))