apps/catalogue/models/book.py

   1 # -*- coding: utf-8 -*-
   2 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
   3 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
   4 #
   5 import re
   6 from django.conf import settings
   7 from django.core.cache import get_cache
   8 from django.db import models
   9 from django.db.models import permalink
  10 import django.dispatch
  11 from django.utils.datastructures import SortedDict
  12 from django.utils.translation import ugettext_lazy as _
  13 import jsonfield
  14 from catalogue.models import Tag, Fragment, BookMedia
  15 from catalogue.utils import create_zip, split_tags, truncate_html_words, book_upload_path
  16 from catalogue import tasks
  17 from newtagging import managers
  18
  19
  20 permanent_cache = get_cache('permanent')
  21
  22
  23 class Book(models.Model):
  24     """Represents a book imported from WL-XML."""
  25     title         = models.CharField(_('title'), max_length=120)
  26     sort_key = models.CharField(_('sort key'), max_length=120, db_index=True, editable=False)
  27     slug = models.SlugField(_('slug'), max_length=120, db_index=True,
  28             unique=True)
  29     common_slug = models.SlugField(_('slug'), max_length=120, db_index=True)
  30     language = models.CharField(_('language code'), max_length=3, db_index=True,
  31                     default=settings.CATALOGUE_DEFAULT_LANGUAGE)
  32     description   = models.TextField(_('description'), blank=True)
  33     created_at    = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True)
  34     changed_at    = models.DateTimeField(_('creation date'), auto_now=True, db_index=True)
  35     parent_number = models.IntegerField(_('parent number'), default=0)
  36     extra_info    = jsonfield.JSONField(_('extra information'), default='{}')
  37     gazeta_link   = models.CharField(blank=True, max_length=240)
  38     wiki_link     = models.CharField(blank=True, max_length=240)
  39     # files generated during publication
  40
  41     cover = models.FileField(_('cover'), upload_to=book_upload_path('png'),
  42                 null=True, blank=True)
  43     ebook_formats = ['pdf', 'epub', 'mobi', 'fb2', 'txt']
  44     formats = ebook_formats + ['html', 'xml']
  45
  46     parent        = models.ForeignKey('self', blank=True, null=True, related_name='children')
  47
  48     _related_info = jsonfield.JSONField(blank=True, null=True, editable=False)
  49
  50     objects  = models.Manager()
  51     tagged   = managers.ModelTaggedItemManager(Tag)
  52     tags     = managers.TagDescriptor(Tag)
  53
  54     html_built = django.dispatch.Signal()
  55     published = django.dispatch.Signal()
  56
  57     class AlreadyExists(Exception):
  58         pass
  59
  60     class Meta:
  61         ordering = ('sort_key',)
  62         verbose_name = _('book')
  63         verbose_name_plural = _('books')
  64         app_label = 'catalogue'
  65
  66     def __unicode__(self):
  67         return self.title
  68
  69     def save(self, force_insert=False, force_update=False, reset_short_html=True, **kwargs):
  70         from sortify import sortify
  71
  72         self.sort_key = sortify(self.title)
  73
  74         ret = super(Book, self).save(force_insert, force_update)
  75
  76         if reset_short_html:
  77             self.reset_short_html()
  78
  79         return ret
  80
  81     @permalink
  82     def get_absolute_url(self):
  83         return ('catalogue.views.book_detail', [self.slug])
  84
  85     @property
  86     def name(self):
  87         return self.title
  88
  89     def book_tag_slug(self):
  90         return ('l-' + self.slug)[:120]
  91
  92     def book_tag(self):
  93         slug = self.book_tag_slug()
  94         book_tag, created = Tag.objects.get_or_create(slug=slug, category='book')
  95         if created:
  96             book_tag.name = self.title[:50]
  97             book_tag.sort_key = self.title.lower()
  98             book_tag.save()
  99         return book_tag
 100
 101     def has_media(self, type_):
 102         if type_ in Book.formats:
 103             return bool(getattr(self, "%s_file" % type_))
 104         else:
 105             return self.media.filter(type=type_).exists()
 106
 107     def get_media(self, type_):
 108         if self.has_media(type_):
 109             if type_ in Book.formats:
 110                 return getattr(self, "%s_file" % type_)
 111             else:
 112                 return self.media.filter(type=type_)
 113         else:
 114             return None
 115
 116     def get_mp3(self):
 117         return self.get_media("mp3")
 118     def get_odt(self):
 119         return self.get_media("odt")
 120     def get_ogg(self):
 121         return self.get_media("ogg")
 122     def get_daisy(self):
 123         return self.get_media("daisy")
 124
 125     def reset_short_html(self):
 126         if self.id is None:
 127             return
 128
 129         type(self).objects.filter(pk=self.pk).update(_related_info=None)
 130         # Fragment.short_html relies on book's tags, so reset it here too
 131         for fragm in self.fragments.all().iterator():
 132             fragm.reset_short_html()
 133
 134     def has_description(self):
 135         return len(self.description) > 0
 136     has_description.short_description = _('description')
 137     has_description.boolean = True
 138
 139     # ugly ugly ugly
 140     def has_mp3_file(self):
 141         return bool(self.has_media("mp3"))
 142     has_mp3_file.short_description = 'MP3'
 143     has_mp3_file.boolean = True
 144
 145     def has_ogg_file(self):
 146         return bool(self.has_media("ogg"))
 147     has_ogg_file.short_description = 'OGG'
 148     has_ogg_file.boolean = True
 149
 150     def has_daisy_file(self):
 151         return bool(self.has_media("daisy"))
 152     has_daisy_file.short_description = 'DAISY'
 153     has_daisy_file.boolean = True
 154
 155     def wldocument(self, parse_dublincore=True):
 156         from catalogue.import_utils import ORMDocProvider
 157         from librarian.parser import WLDocument
 158
 159         return WLDocument.from_file(self.xml_file.path,
 160                 provider=ORMDocProvider(self),
 161                 parse_dublincore=parse_dublincore)
 162
 163     def build_html(self):
 164         from django.core.files.base import ContentFile
 165         from slughifi import slughifi
 166         from sortify import sortify
 167         from librarian import html
 168
 169         meta_tags = list(self.tags.filter(
 170             category__in=('author', 'epoch', 'genre', 'kind')))
 171         book_tag = self.book_tag()
 172
 173         html_output = self.wldocument(parse_dublincore=False).as_html()
 174         if html_output:
 175             self.html_file.save('%s.html' % self.slug,
 176                     ContentFile(html_output.get_string()))
 177
 178             # get ancestor l-tags for adding to new fragments
 179             ancestor_tags = []
 180             p = self.parent
 181             while p:
 182                 ancestor_tags.append(p.book_tag())
 183                 p = p.parent
 184
 185             # Delete old fragments and create them from scratch
 186             self.fragments.all().delete()
 187             # Extract fragments
 188             closed_fragments, open_fragments = html.extract_fragments(self.html_file.path)
 189             for fragment in closed_fragments.values():
 190                 try:
 191                     theme_names = [s.strip() for s in fragment.themes.split(',')]
 192                 except AttributeError:
 193                     continue
 194                 themes = []
 195                 for theme_name in theme_names:
 196                     if not theme_name:
 197                         continue
 198                     tag, created = Tag.objects.get_or_create(slug=slughifi(theme_name), category='theme')
 199                     if created:
 200                         tag.name = theme_name
 201                         tag.sort_key = sortify(theme_name.lower())
 202                         tag.save()
 203                     themes.append(tag)
 204                 if not themes:
 205                     continue
 206
 207                 text = fragment.to_string()
 208                 short_text = truncate_html_words(text, 15)
 209                 if text == short_text:
 210                     short_text = ''
 211                 new_fragment = Fragment.objects.create(anchor=fragment.id, book=self,
 212                     text=text, short_text=short_text)
 213
 214                 new_fragment.save()
 215                 new_fragment.tags = set(meta_tags + themes + [book_tag] + ancestor_tags)
 216             self.save()
 217             self.html_built.send(sender=self)
 218             return True
 219         return False
 220
 221     # Thin wrappers for builder tasks
 222     def build_cover(self):
 223         """(Re)builds the cover image."""
 224         return tasks.build_cover.delay(self.pk)
 225     def build_pdf(self, *args, **kwargs):
 226         """(Re)builds PDF."""
 227         return tasks.build_pdf.delay(self.pk, *args, **kwargs)
 228     def build_epub(self, *args, **kwargs):
 229         """(Re)builds EPUB."""
 230         return tasks.build_epub.delay(self.pk, *args, **kwargs)
 231     def build_mobi(self, *args, **kwargs):
 232         """(Re)builds MOBI."""
 233         return tasks.build_mobi.delay(self.pk, *args, **kwargs)
 234     def build_fb2(self, *args, **kwargs):
 235         """(Re)build FB2"""
 236         return tasks.build_fb2.delay(self.pk, *args, **kwargs)
 237     def build_txt(self, *args, **kwargs):
 238         """(Re)builds TXT."""
 239         return tasks.build_txt.delay(self.pk, *args, **kwargs)
 240
 241     @staticmethod
 242     def zip_format(format_):
 243         def pretty_file_name(book):
 244             return "%s/%s.%s" % (
 245                 b.extra_info['author'],
 246                 b.slug,
 247                 format_)
 248
 249         field_name = "%s_file" % format_
 250         books = Book.objects.filter(parent=None).exclude(**{field_name: ""})
 251         paths = [(pretty_file_name(b), getattr(b, field_name).path)
 252                     for b in books.iterator()]
 253         return create_zip(paths,
 254                     getattr(settings, "ALL_%s_ZIP" % format_.upper()))
 255
 256     def zip_audiobooks(self, format_):
 257         bm = BookMedia.objects.filter(book=self, type=format_)
 258         paths = map(lambda bm: (None, bm.file.path), bm)
 259         return create_zip(paths, "%s_%s" % (self.slug, format_))
 260
 261     def search_index(self, book_info=None, reuse_index=False, index_tags=True):
 262         import search
 263         if reuse_index:
 264             idx = search.ReusableIndex()
 265         else:
 266             idx = search.Index()
 267
 268         idx.open()
 269         try:
 270             idx.index_book(self, book_info)
 271             if index_tags:
 272                 idx.index_tags()
 273         finally:
 274             idx.close()
 275
 276     @classmethod
 277     def from_xml_file(cls, xml_file, **kwargs):
 278         from django.core.files import File
 279         from librarian import dcparser
 280
 281         # use librarian to parse meta-data
 282         book_info = dcparser.parse(xml_file)
 283
 284         if not isinstance(xml_file, File):
 285             xml_file = File(open(xml_file))
 286
 287         try:
 288             return cls.from_text_and_meta(xml_file, book_info, **kwargs)
 289         finally:
 290             xml_file.close()
 291
 292     @classmethod
 293     def from_text_and_meta(cls, raw_file, book_info, overwrite=False,
 294             build_epub=True, build_txt=True, build_pdf=True, build_mobi=True, build_fb2=True,
 295             search_index=True, search_index_tags=True, search_index_reuse=False):
 296
 297         # check for parts before we do anything
 298         children = []
 299         if hasattr(book_info, 'parts'):
 300             for part_url in book_info.parts:
 301                 try:
 302                     children.append(Book.objects.get(slug=part_url.slug))
 303                 except Book.DoesNotExist:
 304                     raise Book.DoesNotExist(_('Book "%s" does not exist.') %
 305                             part_url.slug)
 306
 307         # Read book metadata
 308         book_slug = book_info.url.slug
 309         if re.search(r'[^a-z0-9-]', book_slug):
 310             raise ValueError('Invalid characters in slug')
 311         book, created = Book.objects.get_or_create(slug=book_slug)
 312
 313         if created:
 314             book_shelves = []
 315         else:
 316             if not overwrite:
 317                 raise Book.AlreadyExists(_('Book %s already exists') % (
 318                         book_slug))
 319             # Save shelves for this book
 320             book_shelves = list(book.tags.filter(category='set'))
 321
 322         book.language = book_info.language
 323         book.title = book_info.title
 324         if book_info.variant_of:
 325             book.common_slug = book_info.variant_of.slug
 326         else:
 327             book.common_slug = book.slug
 328         book.extra_info = book_info.to_dict()
 329         book.save()
 330
 331         meta_tags = Tag.tags_from_info(book_info)
 332
 333         book.tags = set(meta_tags + book_shelves)
 334
 335         obsolete_children = set(b for b in book.children.all() if b not in children)
 336         for n, child_book in enumerate(children):
 337             child_book.parent = book
 338             child_book.parent_number = n
 339             child_book.save()
 340         # Disown unfaithful children and let them cope on their own.
 341         for child in obsolete_children:
 342             child.parent = None
 343             child.parent_number = 0
 344             child.save()
 345             tasks.fix_tree_tags.delay(child)
 346
 347         # Save XML file
 348         book.xml_file.save('%s.xml' % book.slug, raw_file, save=False)
 349
 350         # delete old fragments when overwriting
 351         book.fragments.all().delete()
 352         # Build HTML, fix the tree tags, build cover.
 353         has_own_text = bool(book.build_html())
 354         tasks.fix_tree_tags.delay(book)
 355         book.build_cover(book_info)
 356
 357         # No saves behind this point.
 358
 359         if has_own_text:
 360             if not settings.NO_BUILD_TXT and build_txt:
 361                 book.build_txt()
 362             if not settings.NO_BUILD_FB2 and build_fb2:
 363                 book.build_fb2()
 364
 365         if not settings.NO_BUILD_EPUB and build_epub:
 366             book.build_epub()
 367
 368         if not settings.NO_BUILD_PDF and build_pdf:
 369             book.build_pdf()
 370
 371         if not settings.NO_BUILD_MOBI and build_mobi:
 372             book.build_mobi()
 373
 374         if not settings.NO_SEARCH_INDEX and search_index:
 375             book.search_index(index_tags=search_index_tags, reuse_index=search_index_reuse)
 376             #index_book.delay(book.id, book_info)
 377
 378         cls.published.send(sender=book)
 379         return book
 380
 381     def fix_tree_tags(self):
 382         """Fixes the l-tags on the book's subtree.
 383
 384         Makes sure that:
 385         * the book has its parents book-tags,
 386         * its fragments have the book's and its parents book-tags,
 387         * runs those for every child book too,
 388         * touches all relevant tags,
 389         * resets tag and theme counter on the book and its ancestry.
 390         """
 391         def fix_subtree(book, parent_tags):
 392             affected_tags = set(book.tags)
 393             book.tags = list(book.tags.exclude(category='book')) + parent_tags
 394             sub_parent_tags = parent_tags + [book.book_tag()]
 395             for frag in book.fragments.all():
 396                 affected_tags.update(frag.tags)
 397                 frag.tags = list(frag.tags.exclude(category='book')) + sub_parent_tags
 398             for child in book.children.all():
 399                 affected_tags.update(fix_subtree(child, sub_parent_tags))
 400             return affected_tags
 401
 402         parent_tags = []
 403         parent = self.parent
 404         while parent is not None:
 405             parent_tags.append(parent.book_tag())
 406             parent = parent.parent
 407
 408         affected_tags = fix_subtree(self, parent_tags)
 409         for tag in affected_tags:
 410             tasks.touch_tag(tag)
 411
 412         book = self
 413         while book is not None:
 414             book.reset_tag_counter()
 415             book.reset_theme_counter()
 416             book = book.parent
 417
 418     def related_info(self):
 419         """Keeps info about related objects (tags, media) in cache field."""
 420         if self._related_info is not None:
 421             return self._related_info
 422         else:
 423             rel = {'tags': {}, 'media': {}}
 424
 425             tags = self.tags.filter(category__in=(
 426                     'author', 'kind', 'genre', 'epoch'))
 427             tags = split_tags(tags)
 428             for category in tags:
 429                 rel['tags'][category] = [
 430                         (t.name, t.slug) for t in tags[category]]
 431
 432             for media_format in BookMedia.formats:
 433                 rel['media'][media_format] = self.has_media(media_format)
 434
 435             book = self
 436             parents = []
 437             while book.parent:
 438                 parents.append((book.parent.title, book.parent.slug))
 439                 book = book.parent
 440             parents = parents[::-1]
 441             if parents:
 442                 rel['parents'] = parents
 443
 444             if self.pk:
 445                 type(self).objects.filter(pk=self.pk).update(_related_info=rel)
 446             return rel
 447
 448     def related_themes(self):
 449         theme_counter = self.theme_counter
 450         book_themes = list(Tag.objects.filter(pk__in=theme_counter.keys()))
 451         for tag in book_themes:
 452             tag.count = theme_counter[tag.pk]
 453         return book_themes
 454
 455     def reset_tag_counter(self):
 456         if self.id is None:
 457             return
 458
 459         cache_key = "Book.tag_counter/%d" % self.id
 460         permanent_cache.delete(cache_key)
 461         if self.parent:
 462             self.parent.reset_tag_counter()
 463
 464     @property
 465     def tag_counter(self):
 466         if self.id:
 467             cache_key = "Book.tag_counter/%d" % self.id
 468             tags = permanent_cache.get(cache_key)
 469         else:
 470             tags = None
 471
 472         if tags is None:
 473             tags = {}
 474             for child in self.children.all().order_by().iterator():
 475                 for tag_pk, value in child.tag_counter.iteritems():
 476                     tags[tag_pk] = tags.get(tag_pk, 0) + value
 477             for tag in self.tags.exclude(category__in=('book', 'theme', 'set')).order_by().iterator():
 478                 tags[tag.pk] = 1
 479
 480             if self.id:
 481                 permanent_cache.set(cache_key, tags)
 482         return tags
 483
 484     def reset_theme_counter(self):
 485         if self.id is None:
 486             return
 487
 488         cache_key = "Book.theme_counter/%d" % self.id
 489         permanent_cache.delete(cache_key)
 490         if self.parent:
 491             self.parent.reset_theme_counter()
 492
 493     @property
 494     def theme_counter(self):
 495         if self.id:
 496             cache_key = "Book.theme_counter/%d" % self.id
 497             tags = permanent_cache.get(cache_key)
 498         else:
 499             tags = None
 500
 501         if tags is None:
 502             tags = {}
 503             for fragment in Fragment.tagged.with_any([self.book_tag()]).order_by().iterator():
 504                 for tag in fragment.tags.filter(category='theme').order_by().iterator():
 505                     tags[tag.pk] = tags.get(tag.pk, 0) + 1
 506
 507             if self.id:
 508                 permanent_cache.set(cache_key, tags)
 509         return tags
 510
 511     def pretty_title(self, html_links=False):
 512         book = self
 513         names = list(book.tags.filter(category='author'))
 514
 515         books = []
 516         while book:
 517             books.append(book)
 518             book = book.parent
 519         names.extend(reversed(books))
 520
 521         if html_links:
 522             names = ['<a href="%s">%s</a>' % (tag.get_absolute_url(), tag.name) for tag in names]
 523         else:
 524             names = [tag.name for tag in names]
 525
 526         return ', '.join(names)
 527
 528     @classmethod
 529     def tagged_top_level(cls, tags):
 530         """ Returns top-level books tagged with `tags`.
 531
 532         It only returns those books which don't have ancestors which are
 533         also tagged with those tags.
 534
 535         """
 536         # get relevant books and their tags
 537         objects = cls.tagged.with_all(tags)
 538         # eliminate descendants
 539         l_tags = Tag.objects.filter(category='book',
 540             slug__in=[book.book_tag_slug() for book in objects.iterator()])
 541         descendants_keys = [book.pk for book in cls.tagged.with_any(l_tags).iterator()]
 542         if descendants_keys:
 543             objects = objects.exclude(pk__in=descendants_keys)
 544
 545         return objects
 546
 547     @classmethod
 548     def book_list(cls, filter=None):
 549         """Generates a hierarchical listing of all books.
 550
 551         Books are optionally filtered with a test function.
 552
 553         """
 554
 555         books_by_parent = {}
 556         books = cls.objects.all().order_by('parent_number', 'sort_key').only(
 557                 'title', 'parent', 'slug')
 558         if filter:
 559             books = books.filter(filter).distinct()
 560
 561             book_ids = set(b['pk'] for b in books.values("pk").iterator())
 562             for book in books.iterator():
 563                 parent = book.parent_id
 564                 if parent not in book_ids:
 565                     parent = None
 566                 books_by_parent.setdefault(parent, []).append(book)
 567         else:
 568             for book in books.iterator():
 569                 books_by_parent.setdefault(book.parent_id, []).append(book)
 570
 571         orphans = []
 572         books_by_author = SortedDict()
 573         for tag in Tag.objects.filter(category='author').iterator():
 574             books_by_author[tag] = []
 575
 576         for book in books_by_parent.get(None,()):
 577             authors = list(book.tags.filter(category='author'))
 578             if authors:
 579                 for author in authors:
 580                     books_by_author[author].append(book)
 581             else:
 582                 orphans.append(book)
 583
 584         return books_by_author, orphans, books_by_parent
 585
 586     _audiences_pl = {
 587         "SP1": (1, u"szkoła podstawowa"),
 588         "SP2": (1, u"szkoła podstawowa"),
 589         "P": (1, u"szkoła podstawowa"),
 590         "G": (2, u"gimnazjum"),
 591         "L": (3, u"liceum"),
 592         "LP": (3, u"liceum"),
 593     }
 594     def audiences_pl(self):
 595         audiences = self.extra_info.get('audiences', [])
 596         audiences = sorted(set([self._audiences_pl[a] for a in audiences]))
 597         return [a[1] for a in audiences]
 598
 599     def choose_fragment(self):
 600         tag = self.book_tag()
 601         fragments = Fragment.tagged.with_any([tag])
 602         if fragments.exists():
 603             return fragments.order_by('?')[0]
 604         elif self.parent:
 605             return self.parent.choose_fragment()
 606         else:
 607             return None
 608
 609
 610 def _has_factory(ftype):
 611     has = lambda self: bool(getattr(self, "%s_file" % ftype))
 612     has.short_description = ftype.upper()
 613     has.__doc__ = None
 614     has.boolean = True
 615     has.__name__ = "has_%s_file" % ftype
 616     return has
 617
 618
 619 # add the file fields
 620 for t in Book.formats:
 621     field_name = "%s_file" % t
 622     models.FileField(_("%s file" % t.upper()),
 623             upload_to=book_upload_path(t),
 624             blank=True).contribute_to_class(Book, field_name)
 625
 626     setattr(Book, "has_%s_file" % t, _has_factory(t))