src/catalogue/models/book.py

   1 # -*- coding: utf-8 -*-
   2 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
   3 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
   4 #
   5 from collections import OrderedDict
   6 from random import randint
   7 import os.path
   8 import re
   9 import urllib
  10 from django.conf import settings
  11 from django.db import connection, models, transaction
  12 from django.db.models import permalink
  13 import django.dispatch
  14 from django.contrib.contenttypes.fields import GenericRelation
  15 from django.core.urlresolvers import reverse
  16 from django.utils.translation import ugettext_lazy as _
  17 import jsonfield
  18 from fnpdjango.storage import BofhFileSystemStorage
  19 from ssify import flush_ssi_includes
  20 from newtagging import managers
  21 from catalogue import constants
  22 from catalogue.fields import EbookField
  23 from catalogue.models import Tag, Fragment, BookMedia
  24 from catalogue.utils import create_zip, gallery_url, gallery_path
  25 from catalogue.models.tag import prefetched_relations
  26 from catalogue import app_settings
  27 from catalogue import tasks
  28 from wolnelektury.utils import makedirs
  29
  30 bofh_storage = BofhFileSystemStorage()
  31
  32
  33 def _make_upload_to(path):
  34     def _upload_to(i, n):
  35         return path % i.slug
  36     return _upload_to
  37
  38
  39 _cover_upload_to = _make_upload_to('book/cover/%s.jpg')
  40 _cover_thumb_upload_to = _make_upload_to('book/cover_thumb/%s.jpg')
  41
  42
  43 def _ebook_upload_to(upload_path):
  44     return _make_upload_to(upload_path)
  45
  46
  47 class Book(models.Model):
  48     """Represents a book imported from WL-XML."""
  49     title = models.CharField(_('title'), max_length=32767)
  50     sort_key = models.CharField(_('sort key'), max_length=120, db_index=True, editable=False)
  51     sort_key_author = models.CharField(
  52         _('sort key by author'), max_length=120, db_index=True, editable=False, default=u'')
  53     slug = models.SlugField(_('slug'), max_length=120, db_index=True, unique=True)
  54     common_slug = models.SlugField(_('slug'), max_length=120, db_index=True)
  55     language = models.CharField(_('language code'), max_length=3, db_index=True, default=app_settings.DEFAULT_LANGUAGE)
  56     description = models.TextField(_('description'), blank=True)
  57     created_at = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True)
  58     changed_at = models.DateTimeField(_('creation date'), auto_now=True, db_index=True)
  59     parent_number = models.IntegerField(_('parent number'), default=0)
  60     extra_info = jsonfield.JSONField(_('extra information'), default={})
  61     gazeta_link = models.CharField(blank=True, max_length=240)
  62     wiki_link = models.CharField(blank=True, max_length=240)
  63
  64     # files generated during publication
  65     cover = EbookField(
  66         'cover', _('cover'),
  67         null=True, blank=True,
  68         upload_to=_cover_upload_to,
  69         storage=bofh_storage, max_length=255)
  70     # Cleaner version of cover for thumbs
  71     cover_thumb = EbookField(
  72         'cover_thumb', _('cover thumbnail'),
  73         null=True, blank=True,
  74         upload_to=_cover_thumb_upload_to,
  75         max_length=255)
  76     ebook_formats = constants.EBOOK_FORMATS
  77     formats = ebook_formats + ['html', 'xml']
  78
  79     parent = models.ForeignKey('self', blank=True, null=True, related_name='children')
  80     ancestor = models.ManyToManyField('self', blank=True, editable=False, related_name='descendant', symmetrical=False)
  81
  82     objects = models.Manager()
  83     tagged = managers.ModelTaggedItemManager(Tag)
  84     tags = managers.TagDescriptor(Tag)
  85     tag_relations = GenericRelation(Tag.intermediary_table_model)
  86
  87     html_built = django.dispatch.Signal()
  88     published = django.dispatch.Signal()
  89
  90     short_html_url_name = 'catalogue_book_short'
  91
  92     class AlreadyExists(Exception):
  93         pass
  94
  95     class Meta:
  96         ordering = ('sort_key_author', 'sort_key')
  97         verbose_name = _('book')
  98         verbose_name_plural = _('books')
  99         app_label = 'catalogue'
 100
 101     def __unicode__(self):
 102         return self.title
 103
 104     def get_initial(self):
 105         try:
 106             return re.search(r'\w', self.title, re.U).group(0)
 107         except AttributeError:
 108             return ''
 109
 110     def authors(self):
 111         return self.tags.filter(category='author')
 112
 113     def tag_unicode(self, category):
 114         relations = prefetched_relations(self, category)
 115         if relations:
 116             return ', '.join(rel.tag.name for rel in relations)
 117         else:
 118             return ', '.join(self.tags.filter(category=category).values_list('name', flat=True))
 119
 120     def author_unicode(self):
 121         return self.tag_unicode('author')
 122
 123     def save(self, force_insert=False, force_update=False, **kwargs):
 124         from sortify import sortify
 125
 126         self.sort_key = sortify(self.title)[:120]
 127         self.title = unicode(self.title)  # ???
 128
 129         try:
 130             author = self.authors().first().sort_key
 131         except AttributeError:
 132             author = u''
 133         self.sort_key_author = author
 134
 135         ret = super(Book, self).save(force_insert, force_update, **kwargs)
 136
 137         return ret
 138
 139     @permalink
 140     def get_absolute_url(self):
 141         return 'catalogue.views.book_detail', [self.slug]
 142
 143     @staticmethod
 144     @permalink
 145     def create_url(slug):
 146         return 'catalogue.views.book_detail', [slug]
 147
 148     def gallery_path(self):
 149         return gallery_path(self.slug)
 150
 151     def gallery_url(self):
 152         return gallery_url(self.slug)
 153
 154     @property
 155     def name(self):
 156         return self.title
 157
 158     def language_code(self):
 159         return constants.LANGUAGES_3TO2.get(self.language, self.language)
 160
 161     def language_name(self):
 162         return dict(settings.LANGUAGES).get(self.language_code(), "")
 163
 164     def is_foreign(self):
 165         return self.language_code() != settings.LANGUAGE_CODE
 166
 167     def has_media(self, type_):
 168         if type_ in Book.formats:
 169             return bool(getattr(self, "%s_file" % type_))
 170         else:
 171             return self.media.filter(type=type_).exists()
 172
 173     def get_media(self, type_):
 174         if self.has_media(type_):
 175             if type_ in Book.formats:
 176                 return getattr(self, "%s_file" % type_)
 177             else:
 178                 return self.media.filter(type=type_)
 179         else:
 180             return None
 181
 182     def get_mp3(self):
 183         return self.get_media("mp3")
 184
 185     def get_odt(self):
 186         return self.get_media("odt")
 187
 188     def get_ogg(self):
 189         return self.get_media("ogg")
 190
 191     def get_daisy(self):
 192         return self.get_media("daisy")
 193
 194     def has_description(self):
 195         return len(self.description) > 0
 196     has_description.short_description = _('description')
 197     has_description.boolean = True
 198
 199     # ugly ugly ugly
 200     def has_mp3_file(self):
 201         return bool(self.has_media("mp3"))
 202     has_mp3_file.short_description = 'MP3'
 203     has_mp3_file.boolean = True
 204
 205     def has_ogg_file(self):
 206         return bool(self.has_media("ogg"))
 207     has_ogg_file.short_description = 'OGG'
 208     has_ogg_file.boolean = True
 209
 210     def has_daisy_file(self):
 211         return bool(self.has_media("daisy"))
 212     has_daisy_file.short_description = 'DAISY'
 213     has_daisy_file.boolean = True
 214
 215     def wldocument(self, parse_dublincore=True, inherit=True):
 216         from catalogue.import_utils import ORMDocProvider
 217         from librarian.parser import WLDocument
 218
 219         if inherit and self.parent:
 220             meta_fallbacks = self.parent.cover_info()
 221         else:
 222             meta_fallbacks = None
 223
 224         return WLDocument.from_file(
 225             self.xml_file.path,
 226             provider=ORMDocProvider(self),
 227             parse_dublincore=parse_dublincore,
 228             meta_fallbacks=meta_fallbacks)
 229
 230     @staticmethod
 231     def zip_format(format_):
 232         def pretty_file_name(book):
 233             return "%s/%s.%s" % (
 234                 book.extra_info['author'],
 235                 book.slug,
 236                 format_)
 237
 238         field_name = "%s_file" % format_
 239         books = Book.objects.filter(parent=None).exclude(**{field_name: ""})
 240         paths = [(pretty_file_name(b), getattr(b, field_name).path) for b in books.iterator()]
 241         return create_zip(paths, app_settings.FORMAT_ZIPS[format_])
 242
 243     def zip_audiobooks(self, format_):
 244         bm = BookMedia.objects.filter(book=self, type=format_)
 245         paths = map(lambda bm: (None, bm.file.path), bm)
 246         return create_zip(paths, "%s_%s" % (self.slug, format_))
 247
 248     def search_index(self, book_info=None, index=None, index_tags=True, commit=True):
 249         if index is None:
 250             from search.index import Index
 251             index = Index()
 252         try:
 253             index.index_book(self, book_info)
 254             if index_tags:
 255                 index.index_tags()
 256             if commit:
 257                 index.index.commit()
 258         except Exception, e:
 259             index.index.rollback()
 260             raise e
 261
 262     def download_pictures(self, remote_gallery_url):
 263         gallery_path = self.gallery_path()
 264         # delete previous files, so we don't include old files in ebooks
 265         if os.path.isdir(gallery_path):
 266             for filename in os.listdir(gallery_path):
 267                 file_path = os.path.join(gallery_path, filename)
 268                 os.unlink(file_path)
 269         ilustr_elements = list(self.wldocument().edoc.findall('//ilustr'))
 270         if ilustr_elements:
 271             makedirs(gallery_path)
 272             for ilustr in ilustr_elements:
 273                 ilustr_src = ilustr.get('src')
 274                 ilustr_path = os.path.join(gallery_path, ilustr_src)
 275                 urllib.urlretrieve('%s/%s' % (remote_gallery_url, ilustr_src), ilustr_path)
 276
 277     @classmethod
 278     def from_xml_file(cls, xml_file, **kwargs):
 279         from django.core.files import File
 280         from librarian import dcparser
 281
 282         # use librarian to parse meta-data
 283         book_info = dcparser.parse(xml_file)
 284
 285         if not isinstance(xml_file, File):
 286             xml_file = File(open(xml_file))
 287
 288         try:
 289             return cls.from_text_and_meta(xml_file, book_info, **kwargs)
 290         finally:
 291             xml_file.close()
 292
 293     @classmethod
 294     def from_text_and_meta(cls, raw_file, book_info, overwrite=False, dont_build=None, search_index=True,
 295                            search_index_tags=True, remote_gallery_url=None):
 296         if dont_build is None:
 297             dont_build = set()
 298         dont_build = set.union(set(dont_build), set(app_settings.DONT_BUILD))
 299
 300         # check for parts before we do anything
 301         children = []
 302         if hasattr(book_info, 'parts'):
 303             for part_url in book_info.parts:
 304                 try:
 305                     children.append(Book.objects.get(slug=part_url.slug))
 306                 except Book.DoesNotExist:
 307                     raise Book.DoesNotExist(_('Book "%s" does not exist.') % part_url.slug)
 308
 309         # Read book metadata
 310         book_slug = book_info.url.slug
 311         if re.search(r'[^a-z0-9-]', book_slug):
 312             raise ValueError('Invalid characters in slug')
 313         book, created = Book.objects.get_or_create(slug=book_slug)
 314
 315         if created:
 316             book_shelves = []
 317             old_cover = None
 318         else:
 319             if not overwrite:
 320                 raise Book.AlreadyExists(_('Book %s already exists') % book_slug)
 321             # Save shelves for this book
 322             book_shelves = list(book.tags.filter(category='set'))
 323             old_cover = book.cover_info()
 324
 325         # Save XML file
 326         book.xml_file.save('%s.xml' % book.slug, raw_file, save=False)
 327
 328         book.language = book_info.language
 329         book.title = book_info.title
 330         if book_info.variant_of:
 331             book.common_slug = book_info.variant_of.slug
 332         else:
 333             book.common_slug = book.slug
 334         book.extra_info = book_info.to_dict()
 335         book.save()
 336
 337         meta_tags = Tag.tags_from_info(book_info)
 338
 339         book.tags = set(meta_tags + book_shelves)
 340
 341         cover_changed = old_cover != book.cover_info()
 342         obsolete_children = set(b for b in book.children.all()
 343                                 if b not in children)
 344         notify_cover_changed = []
 345         for n, child_book in enumerate(children):
 346             new_child = child_book.parent != book
 347             child_book.parent = book
 348             child_book.parent_number = n
 349             child_book.save()
 350             if new_child or cover_changed:
 351                 notify_cover_changed.append(child_book)
 352         # Disown unfaithful children and let them cope on their own.
 353         for child in obsolete_children:
 354             child.parent = None
 355             child.parent_number = 0
 356             child.save()
 357             if old_cover:
 358                 notify_cover_changed.append(child)
 359
 360         cls.repopulate_ancestors()
 361         tasks.update_counters.delay()
 362
 363         if remote_gallery_url:
 364             book.download_pictures(remote_gallery_url)
 365
 366         # No saves beyond this point.
 367
 368         # Build cover.
 369         if 'cover' not in dont_build:
 370             book.cover.build_delay()
 371             book.cover_thumb.build_delay()
 372
 373         # Build HTML and ebooks.
 374         book.html_file.build_delay()
 375         if not children:
 376             for format_ in constants.EBOOK_FORMATS_WITHOUT_CHILDREN:
 377                 if format_ not in dont_build:
 378                     getattr(book, '%s_file' % format_).build_delay()
 379         for format_ in constants.EBOOK_FORMATS_WITH_CHILDREN:
 380             if format_ not in dont_build:
 381                 getattr(book, '%s_file' % format_).build_delay()
 382
 383         if not settings.NO_SEARCH_INDEX and search_index:
 384             tasks.index_book.delay(book.id, book_info=book_info, index_tags=search_index_tags)
 385
 386         for child in notify_cover_changed:
 387             child.parent_cover_changed()
 388
 389         cls.published.send(sender=cls, instance=book)
 390         return book
 391
 392     @classmethod
 393     @transaction.atomic
 394     def repopulate_ancestors(cls):
 395         """Fixes the ancestry cache."""
 396         # TODO: table names
 397         cursor = connection.cursor()
 398         if connection.vendor == 'postgres':
 399             cursor.execute("TRUNCATE catalogue_book_ancestor")
 400             cursor.execute("""
 401                 WITH RECURSIVE ancestry AS (
 402                     SELECT book.id, book.parent_id
 403                     FROM catalogue_book AS book
 404                     WHERE book.parent_id IS NOT NULL
 405                     UNION
 406                     SELECT ancestor.id, book.parent_id
 407                     FROM ancestry AS ancestor, catalogue_book AS book
 408                     WHERE ancestor.parent_id = book.id
 409                         AND book.parent_id IS NOT NULL
 410                     )
 411                 INSERT INTO catalogue_book_ancestor
 412                     (from_book_id, to_book_id)
 413                     SELECT id, parent_id
 414                     FROM ancestry
 415                     ORDER BY id;
 416                 """)
 417         else:
 418             cursor.execute("DELETE FROM catalogue_book_ancestor")
 419             for b in cls.objects.exclude(parent=None):
 420                 parent = b.parent
 421                 while parent is not None:
 422                     b.ancestor.add(parent)
 423                     parent = parent.parent
 424
 425     def flush_includes(self, languages=True):
 426         if not languages:
 427             return
 428         if languages is True:
 429             languages = [lc for (lc, _ln) in settings.LANGUAGES]
 430         flush_ssi_includes([
 431             template % (self.pk, lang)
 432             for template in [
 433                 '/katalog/b/%d/mini.%s.html',
 434                 '/katalog/b/%d/mini_nolink.%s.html',
 435                 '/katalog/b/%d/short.%s.html',
 436                 '/katalog/b/%d/wide.%s.html',
 437                 '/api/include/book/%d.%s.json',
 438                 '/api/include/book/%d.%s.xml',
 439                 ]
 440             for lang in languages
 441             ])
 442
 443     def cover_info(self, inherit=True):
 444         """Returns a dictionary to serve as fallback for BookInfo.
 445
 446         For now, the only thing inherited is the cover image.
 447         """
 448         need = False
 449         info = {}
 450         for field in ('cover_url', 'cover_by', 'cover_source'):
 451             val = self.extra_info.get(field)
 452             if val:
 453                 info[field] = val
 454             else:
 455                 need = True
 456         if inherit and need and self.parent is not None:
 457             parent_info = self.parent.cover_info()
 458             parent_info.update(info)
 459             info = parent_info
 460         return info
 461
 462     def related_themes(self):
 463         return Tag.objects.usage_for_queryset(
 464             Fragment.objects.filter(models.Q(book=self) | models.Q(book__ancestor=self)),
 465             counts=True).filter(category='theme')
 466
 467     def parent_cover_changed(self):
 468         """Called when parent book's cover image is changed."""
 469         if not self.cover_info(inherit=False):
 470             if 'cover' not in app_settings.DONT_BUILD:
 471                 self.cover.build_delay()
 472                 self.cover_thumb.build_delay()
 473             for format_ in constants.EBOOK_FORMATS_WITH_COVERS:
 474                 if format_ not in app_settings.DONT_BUILD:
 475                     getattr(self, '%s_file' % format_).build_delay()
 476             for child in self.children.all():
 477                 child.parent_cover_changed()
 478
 479     def other_versions(self):
 480         """Find other versions (i.e. in other languages) of the book."""
 481         return type(self).objects.filter(common_slug=self.common_slug).exclude(pk=self.pk)
 482
 483     def parents(self):
 484         books = []
 485         parent = self.parent
 486         while parent is not None:
 487             books.insert(0, parent)
 488             parent = parent.parent
 489         return books
 490
 491     def pretty_title(self, html_links=False):
 492         names = [(tag.name, tag.get_absolute_url()) for tag in self.authors().only('name', 'category', 'slug')]
 493         books = self.parents() + [self]
 494         names.extend([(b.title, b.get_absolute_url()) for b in books])
 495
 496         if html_links:
 497             names = ['<a href="%s">%s</a>' % (tag[1], tag[0]) for tag in names]
 498         else:
 499             names = [tag[0] for tag in names]
 500         return ', '.join(names)
 501
 502     @classmethod
 503     def tagged_top_level(cls, tags):
 504         """ Returns top-level books tagged with `tags`.
 505
 506         It only returns those books which don't have ancestors which are
 507         also tagged with those tags.
 508
 509         """
 510         objects = cls.tagged.with_all(tags)
 511         return objects.exclude(ancestor__in=objects)
 512
 513     @classmethod
 514     def book_list(cls, book_filter=None):
 515         """Generates a hierarchical listing of all books.
 516
 517         Books are optionally filtered with a test function.
 518
 519         """
 520
 521         books_by_parent = {}
 522         books = cls.objects.order_by('parent_number', 'sort_key').only('title', 'parent', 'slug')
 523         if book_filter:
 524             books = books.filter(book_filter).distinct()
 525
 526             book_ids = set(b['pk'] for b in books.values("pk").iterator())
 527             for book in books.iterator():
 528                 parent = book.parent_id
 529                 if parent not in book_ids:
 530                     parent = None
 531                 books_by_parent.setdefault(parent, []).append(book)
 532         else:
 533             for book in books.iterator():
 534                 books_by_parent.setdefault(book.parent_id, []).append(book)
 535
 536         orphans = []
 537         books_by_author = OrderedDict()
 538         for tag in Tag.objects.filter(category='author').iterator():
 539             books_by_author[tag] = []
 540
 541         for book in books_by_parent.get(None, ()):
 542             authors = list(book.authors().only('pk'))
 543             if authors:
 544                 for author in authors:
 545                     books_by_author[author].append(book)
 546             else:
 547                 orphans.append(book)
 548
 549         return books_by_author, orphans, books_by_parent
 550
 551     _audiences_pl = {
 552         "SP": (1, u"szkoła podstawowa"),
 553         "SP1": (1, u"szkoła podstawowa"),
 554         "SP2": (1, u"szkoła podstawowa"),
 555         "P": (1, u"szkoła podstawowa"),
 556         "G": (2, u"gimnazjum"),
 557         "L": (3, u"liceum"),
 558         "LP": (3, u"liceum"),
 559     }
 560
 561     def audiences_pl(self):
 562         audiences = self.extra_info.get('audiences', [])
 563         audiences = sorted(set([self._audiences_pl.get(a, (99, a)) for a in audiences]))
 564         return [a[1] for a in audiences]
 565
 566     def stage_note(self):
 567         stage = self.extra_info.get('stage')
 568         if stage and stage < '0.4':
 569             return (_('This work needs modernisation'),
 570                     reverse('infopage', args=['wymagajace-uwspolczesnienia']))
 571         else:
 572             return None, None
 573
 574     def choose_fragment(self):
 575         fragments = self.fragments.order_by()
 576         fragments_count = fragments.count()
 577         if not fragments_count and self.children.exists():
 578             fragments = Fragment.objects.filter(book__ancestor=self).order_by()
 579             fragments_count = fragments.count()
 580         if fragments_count:
 581             return fragments[randint(0, fragments_count - 1)]
 582         elif self.parent:
 583             return self.parent.choose_fragment()
 584         else:
 585             return None
 586
 587     def update_popularity(self):
 588         count = self.tags.filter(category='set').values('user').order_by('user').distinct().count()
 589         try:
 590             pop = self.popularity
 591             pop.count = count
 592             pop.save()
 593         except BookPopularity.DoesNotExist:
 594             BookPopularity.objects.create(book=self, count=count)
 595
 596
 597 def add_file_fields():
 598     for format_ in Book.formats:
 599         field_name = "%s_file" % format_
 600         # This weird globals() assignment makes Django migrations comfortable.
 601         _upload_to = _ebook_upload_to('book/%s/%%s.%s' % (format_, format_))
 602         _upload_to.__name__ = '_%s_upload_to' % format_
 603         globals()[_upload_to.__name__] = _upload_to
 604
 605         EbookField(
 606             format_, _("%s file" % format_.upper()),
 607             upload_to=_upload_to,
 608             storage=bofh_storage,
 609             max_length=255,
 610             blank=True,
 611             default=''
 612         ).contribute_to_class(Book, field_name)
 613
 614 add_file_fields()
 615
 616
 617 class BookPopularity(models.Model):
 618     book = models.OneToOneField(Book, related_name='popularity')
 619     count = models.IntegerField(default=0)