src/catalogue/models/book.py

   1 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
   2 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
   3 #
   4 from collections import OrderedDict
   5 import json
   6 from datetime import date, timedelta
   7 from random import randint
   8 import os.path
   9 import re
  10 from urllib.request import urlretrieve
  11 from django.conf import settings
  12 from django.db import connection, models, transaction
  13 import django.dispatch
  14 from django.contrib.contenttypes.fields import GenericRelation
  15 from django.urls import reverse
  16 from django.utils.translation import ugettext_lazy as _, get_language
  17 from django.utils.deconstruct import deconstructible
  18 from fnpdjango.storage import BofhFileSystemStorage
  19
  20 from librarian.cover import WLCover
  21 from librarian.html import transform_abstrakt
  22 from newtagging import managers
  23 from catalogue import constants
  24 from catalogue.fields import EbookField
  25 from catalogue.models import Tag, Fragment, BookMedia
  26 from catalogue.utils import create_zip, gallery_url, gallery_path, split_tags, get_random_hash
  27 from catalogue.models.tag import prefetched_relations
  28 from catalogue import app_settings
  29 from catalogue import tasks
  30 from wolnelektury.utils import makedirs, cached_render, clear_cached_renders
  31
  32 bofh_storage = BofhFileSystemStorage()
  33
  34
  35 @deconstructible
  36 class UploadToPath(object):
  37     def __init__(self, path):
  38         self.path = path
  39
  40     def __call__(self, instance, filename):
  41         return self.path % instance.slug
  42
  43
  44 _cover_upload_to = UploadToPath('book/cover/%s.jpg')
  45 _cover_thumb_upload_to = UploadToPath('book/cover_thumb/%s.jpg')
  46 _cover_api_thumb_upload_to = UploadToPath('book/cover_api_thumb/%s.jpg')
  47 _simple_cover_upload_to = UploadToPath('book/cover_simple/%s.jpg')
  48
  49
  50 def _ebook_upload_to(upload_path):
  51     return UploadToPath(upload_path)
  52
  53
  54 class Book(models.Model):
  55     """Represents a book imported from WL-XML."""
  56     title = models.CharField(_('title'), max_length=32767)
  57     sort_key = models.CharField(_('sort key'), max_length=120, db_index=True, editable=False)
  58     sort_key_author = models.CharField(
  59         _('sort key by author'), max_length=120, db_index=True, editable=False, default='')
  60     slug = models.SlugField(_('slug'), max_length=120, db_index=True, unique=True)
  61     common_slug = models.SlugField(_('slug'), max_length=120, db_index=True)
  62     language = models.CharField(_('language code'), max_length=3, db_index=True, default=app_settings.DEFAULT_LANGUAGE)
  63     description = models.TextField(_('description'), blank=True)
  64     abstract = models.TextField(_('abstract'), blank=True)
  65     created_at = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True)
  66     changed_at = models.DateTimeField(_('change date'), auto_now=True, db_index=True)
  67     parent_number = models.IntegerField(_('parent number'), default=0)
  68     extra_info = models.TextField(_('extra information'), default='{}')
  69     gazeta_link = models.CharField(blank=True, max_length=240)
  70     wiki_link = models.CharField(blank=True, max_length=240)
  71     print_on_demand = models.BooleanField(_('print on demand'), default=False)
  72     recommended = models.BooleanField(_('recommended'), default=False)
  73     audio_length = models.CharField(_('audio length'), blank=True, max_length=8)
  74     preview = models.BooleanField(_('preview'), default=False)
  75     preview_until = models.DateField(_('preview until'), blank=True, null=True)
  76     preview_key = models.CharField(max_length=32, blank=True, null=True)
  77     findable = models.BooleanField(_('findable'), default=True, db_index=True)
  78
  79     # files generated during publication
  80     cover = EbookField(
  81         'cover', _('cover'),
  82         null=True, blank=True,
  83         upload_to=_cover_upload_to,
  84         storage=bofh_storage, max_length=255)
  85     # Cleaner version of cover for thumbs
  86     cover_thumb = EbookField(
  87         'cover_thumb', _('cover thumbnail'),
  88         null=True, blank=True,
  89         upload_to=_cover_thumb_upload_to,
  90         max_length=255)
  91     cover_api_thumb = EbookField(
  92         'cover_api_thumb', _('cover thumbnail for mobile app'),
  93         null=True, blank=True,
  94         upload_to=_cover_api_thumb_upload_to,
  95         max_length=255)
  96     simple_cover = EbookField(
  97         'simple_cover', _('cover for mobile app'),
  98         null=True, blank=True,
  99         upload_to=_simple_cover_upload_to,
 100         max_length=255)
 101     ebook_formats = constants.EBOOK_FORMATS
 102     formats = ebook_formats + ['html', 'xml']
 103
 104     parent = models.ForeignKey('self', models.CASCADE, blank=True, null=True, related_name='children')
 105     ancestor = models.ManyToManyField('self', blank=True, editable=False, related_name='descendant', symmetrical=False)
 106
 107     cached_author = models.CharField(blank=True, max_length=240, db_index=True)
 108     has_audience = models.BooleanField(default=False)
 109
 110     objects = models.Manager()
 111     tagged = managers.ModelTaggedItemManager(Tag)
 112     tags = managers.TagDescriptor(Tag)
 113     tag_relations = GenericRelation(Tag.intermediary_table_model)
 114
 115     html_built = django.dispatch.Signal()
 116     published = django.dispatch.Signal()
 117
 118     SORT_KEY_SEP = '$'
 119
 120     class AlreadyExists(Exception):
 121         pass
 122
 123     class Meta:
 124         ordering = ('sort_key_author', 'sort_key')
 125         verbose_name = _('book')
 126         verbose_name_plural = _('books')
 127         app_label = 'catalogue'
 128
 129     def __str__(self):
 130         return self.title
 131
 132     def get_extra_info_json(self):
 133         return json.loads(self.extra_info or '{}')
 134
 135     def get_initial(self):
 136         try:
 137             return re.search(r'\w', self.title, re.U).group(0)
 138         except AttributeError:
 139             return ''
 140
 141     def authors(self):
 142         return self.tags.filter(category='author')
 143
 144     def epochs(self):
 145         return self.tags.filter(category='epoch')
 146
 147     def genres(self):
 148         return self.tags.filter(category='genre')
 149
 150     def kinds(self):
 151         return self.tags.filter(category='kind')
 152
 153     def tag_unicode(self, category):
 154         relations = prefetched_relations(self, category)
 155         if relations:
 156             return ', '.join(rel.tag.name for rel in relations)
 157         else:
 158             return ', '.join(self.tags.filter(category=category).values_list('name', flat=True))
 159
 160     def tags_by_category(self):
 161         return split_tags(self.tags.exclude(category__in=('set', 'theme')))
 162
 163     def author_unicode(self):
 164         return self.cached_author
 165
 166     def kind_unicode(self):
 167         return self.tag_unicode('kind')
 168
 169     def epoch_unicode(self):
 170         return self.tag_unicode('epoch')
 171
 172     def genre_unicode(self):
 173         return self.tag_unicode('genre')
 174
 175     def translator(self):
 176         translators = self.get_extra_info_json().get('translators')
 177         if not translators:
 178             return None
 179         if len(translators) > 3:
 180             translators = translators[:2]
 181             others = ' i inni'
 182         else:
 183             others = ''
 184         return ', '.join('\xa0'.join(reversed(translator.split(', ', 1))) for translator in translators) + others
 185
 186     def cover_source(self):
 187         return self.get_extra_info_json().get('cover_source', self.parent.cover_source() if self.parent else '')
 188
 189     def save(self, force_insert=False, force_update=False, **kwargs):
 190         from sortify import sortify
 191
 192         self.sort_key = sortify(self.title)[:120]
 193         self.title = str(self.title)  # ???
 194
 195         try:
 196             author = self.authors().first().sort_key
 197         except AttributeError:
 198             author = ''
 199         self.sort_key_author = author
 200
 201         self.cached_author = self.tag_unicode('author')
 202         self.has_audience = 'audience' in self.get_extra_info_json()
 203
 204         if self.preview and not self.preview_key:
 205             self.preview_key = get_random_hash(self.slug)[:32]
 206
 207         ret = super(Book, self).save(force_insert, force_update, **kwargs)
 208
 209         return ret
 210
 211     def get_absolute_url(self):
 212         return reverse('book_detail', args=[self.slug])
 213
 214     def gallery_path(self):
 215         return gallery_path(self.slug)
 216
 217     def gallery_url(self):
 218         return gallery_url(self.slug)
 219
 220     @property
 221     def name(self):
 222         return self.title
 223
 224     def language_code(self):
 225         return constants.LANGUAGES_3TO2.get(self.language, self.language)
 226
 227     def language_name(self):
 228         return dict(settings.LANGUAGES).get(self.language_code(), "")
 229
 230     def is_foreign(self):
 231         return self.language_code() != settings.LANGUAGE_CODE
 232
 233     def set_audio_length(self):
 234         length = self.get_audio_length()
 235         if length > 0:
 236             self.audio_length = self.format_audio_length(length)
 237             self.save()
 238
 239     @staticmethod
 240     def format_audio_length(seconds):
 241         """
 242         >>> Book.format_audio_length(1)
 243         '0:01'
 244         >>> Book.format_audio_length(3661)
 245         '1:01:01'
 246         """
 247         if seconds < 60*60:
 248             minutes = seconds // 60
 249             seconds = seconds % 60
 250             return '%d:%02d' % (minutes, seconds)
 251         else:
 252             hours = seconds // 3600
 253             minutes = seconds % 3600 // 60
 254             seconds = seconds % 60
 255             return '%d:%02d:%02d' % (hours, minutes, seconds)
 256
 257     def get_audio_length(self):
 258         total = 0
 259         for media in self.get_mp3() or ():
 260             total += app_settings.GET_MP3_LENGTH(media.file.path)
 261         return int(total)
 262
 263     def has_media(self, type_):
 264         if type_ in Book.formats:
 265             return bool(getattr(self, "%s_file" % type_))
 266         else:
 267             return self.media.filter(type=type_).exists()
 268
 269     def has_audio(self):
 270         return self.has_media('mp3')
 271
 272     def get_media(self, type_):
 273         if self.has_media(type_):
 274             if type_ in Book.formats:
 275                 return getattr(self, "%s_file" % type_)
 276             else:
 277                 return self.media.filter(type=type_)
 278         else:
 279             return None
 280
 281     def get_mp3(self):
 282         return self.get_media("mp3")
 283
 284     def get_odt(self):
 285         return self.get_media("odt")
 286
 287     def get_ogg(self):
 288         return self.get_media("ogg")
 289
 290     def get_daisy(self):
 291         return self.get_media("daisy")
 292
 293     def media_url(self, format_):
 294         media = self.get_media(format_)
 295         if media:
 296             if self.preview:
 297                 return reverse('embargo_link', kwargs={'key': self.preview_key, 'slug': self.slug, 'format_': format_})
 298             else:
 299                 return media.url
 300         else:
 301             return None
 302
 303     def html_url(self):
 304         return self.media_url('html')
 305
 306     def pdf_url(self):
 307         return self.media_url('pdf')
 308
 309     def epub_url(self):
 310         return self.media_url('epub')
 311
 312     def mobi_url(self):
 313         return self.media_url('mobi')
 314
 315     def txt_url(self):
 316         return self.media_url('txt')
 317
 318     def fb2_url(self):
 319         return self.media_url('fb2')
 320
 321     def xml_url(self):
 322         return self.media_url('xml')
 323
 324     def has_description(self):
 325         return len(self.description) > 0
 326     has_description.short_description = _('description')
 327     has_description.boolean = True
 328
 329     def has_mp3_file(self):
 330         return self.has_media("mp3")
 331     has_mp3_file.short_description = 'MP3'
 332     has_mp3_file.boolean = True
 333
 334     def has_ogg_file(self):
 335         return self.has_media("ogg")
 336     has_ogg_file.short_description = 'OGG'
 337     has_ogg_file.boolean = True
 338
 339     def has_daisy_file(self):
 340         return self.has_media("daisy")
 341     has_daisy_file.short_description = 'DAISY'
 342     has_daisy_file.boolean = True
 343
 344     def get_audiobooks(self):
 345         ogg_files = {}
 346         for m in self.media.filter(type='ogg').order_by().iterator():
 347             ogg_files[m.name] = m
 348
 349         audiobooks = []
 350         projects = set()
 351         for mp3 in self.media.filter(type='mp3').iterator():
 352             # ogg files are always from the same project
 353             meta = mp3.get_extra_info_json()
 354             project = meta.get('project')
 355             if not project:
 356                 # temporary fallback
 357                 project = 'CzytamySłuchając'
 358
 359             projects.add((project, meta.get('funded_by', '')))
 360
 361             media = {'mp3': mp3}
 362
 363             ogg = ogg_files.get(mp3.name)
 364             if ogg:
 365                 media['ogg'] = ogg
 366             audiobooks.append(media)
 367
 368         projects = sorted(projects)
 369         return audiobooks, projects
 370
 371     def wldocument(self, parse_dublincore=True, inherit=True):
 372         from catalogue.import_utils import ORMDocProvider
 373         from librarian.parser import WLDocument
 374
 375         if inherit and self.parent:
 376             meta_fallbacks = self.parent.cover_info()
 377         else:
 378             meta_fallbacks = None
 379
 380         return WLDocument.from_file(
 381             self.xml_file.path,
 382             provider=ORMDocProvider(self),
 383             parse_dublincore=parse_dublincore,
 384             meta_fallbacks=meta_fallbacks)
 385
 386     @staticmethod
 387     def zip_format(format_):
 388         def pretty_file_name(book):
 389             return "%s/%s.%s" % (
 390                 book.get_extra_info_json()['author'],
 391                 book.slug,
 392                 format_)
 393
 394         field_name = "%s_file" % format_
 395         books = Book.objects.filter(parent=None).exclude(**{field_name: ""}).exclude(preview=True).exclude(findable=False)
 396         paths = [(pretty_file_name(b), getattr(b, field_name).path) for b in books.iterator()]
 397         return create_zip(paths, app_settings.FORMAT_ZIPS[format_])
 398
 399     def zip_audiobooks(self, format_):
 400         bm = BookMedia.objects.filter(book=self, type=format_)
 401         paths = map(lambda bm: (None, bm.file.path), bm)
 402         return create_zip(paths, "%s_%s" % (self.slug, format_))
 403
 404     def search_index(self, book_info=None, index=None, index_tags=True, commit=True):
 405         if not self.findable:
 406             return
 407         if index is None:
 408             from search.index import Index
 409             index = Index()
 410         try:
 411             index.index_book(self, book_info)
 412             if index_tags:
 413                 index.index_tags()
 414             if commit:
 415                 index.index.commit()
 416         except Exception as e:
 417             index.index.rollback()
 418             raise e
 419
 420     # will make problems in conjunction with paid previews
 421     def download_pictures(self, remote_gallery_url):
 422         gallery_path = self.gallery_path()
 423         # delete previous files, so we don't include old files in ebooks
 424         if os.path.isdir(gallery_path):
 425             for filename in os.listdir(gallery_path):
 426                 file_path = os.path.join(gallery_path, filename)
 427                 os.unlink(file_path)
 428         ilustr_elements = list(self.wldocument().edoc.findall('//ilustr'))
 429         if ilustr_elements:
 430             makedirs(gallery_path)
 431             for ilustr in ilustr_elements:
 432                 ilustr_src = ilustr.get('src')
 433                 ilustr_path = os.path.join(gallery_path, ilustr_src)
 434                 urlretrieve('%s/%s' % (remote_gallery_url, ilustr_src), ilustr_path)
 435
 436     def load_abstract(self):
 437         abstract = self.wldocument(parse_dublincore=False).edoc.getroot().find('.//abstrakt')
 438         if abstract is not None:
 439             self.abstract = transform_abstrakt(abstract)
 440         else:
 441             self.abstract = ''
 442
 443     @classmethod
 444     def from_xml_file(cls, xml_file, **kwargs):
 445         from django.core.files import File
 446         from librarian import dcparser
 447
 448         # use librarian to parse meta-data
 449         book_info = dcparser.parse(xml_file)
 450
 451         if not isinstance(xml_file, File):
 452             xml_file = File(open(xml_file))
 453
 454         try:
 455             return cls.from_text_and_meta(xml_file, book_info, **kwargs)
 456         finally:
 457             xml_file.close()
 458
 459     @classmethod
 460     def from_text_and_meta(cls, raw_file, book_info, overwrite=False, dont_build=None, search_index=True,
 461                            search_index_tags=True, remote_gallery_url=None, days=0, findable=True):
 462         if dont_build is None:
 463             dont_build = set()
 464         dont_build = set.union(set(dont_build), set(app_settings.DONT_BUILD))
 465
 466         # check for parts before we do anything
 467         children = []
 468         if hasattr(book_info, 'parts'):
 469             for part_url in book_info.parts:
 470                 try:
 471                     children.append(Book.objects.get(slug=part_url.slug))
 472                 except Book.DoesNotExist:
 473                     raise Book.DoesNotExist(_('Book "%s" does not exist.') % part_url.slug)
 474
 475         # Read book metadata
 476         book_slug = book_info.url.slug
 477         if re.search(r'[^a-z0-9-]', book_slug):
 478             raise ValueError('Invalid characters in slug')
 479         book, created = Book.objects.get_or_create(slug=book_slug)
 480
 481         if created:
 482             book_shelves = []
 483             old_cover = None
 484             book.preview = bool(days)
 485             if book.preview:
 486                 book.preview_until = date.today() + timedelta(days)
 487         else:
 488             if not overwrite:
 489                 raise Book.AlreadyExists(_('Book %s already exists') % book_slug)
 490             # Save shelves for this book
 491             book_shelves = list(book.tags.filter(category='set'))
 492             old_cover = book.cover_info()
 493
 494         # Save XML file
 495         book.xml_file.save('%s.xml' % book.slug, raw_file, save=False)
 496         if book.preview:
 497             book.xml_file.set_readable(False)
 498
 499         book.findable = findable
 500         book.language = book_info.language
 501         book.title = book_info.title
 502         if book_info.variant_of:
 503             book.common_slug = book_info.variant_of.slug
 504         else:
 505             book.common_slug = book.slug
 506         book.extra_info = json.dumps(book_info.to_dict())
 507         book.load_abstract()
 508         book.save()
 509
 510         meta_tags = Tag.tags_from_info(book_info)
 511
 512         for tag in meta_tags:
 513             if not tag.for_books:
 514                 tag.for_books = True
 515                 tag.save()
 516
 517         book.tags = set(meta_tags + book_shelves)
 518         book.save()  # update sort_key_author
 519
 520         cover_changed = old_cover != book.cover_info()
 521         obsolete_children = set(b for b in book.children.all()
 522                                 if b not in children)
 523         notify_cover_changed = []
 524         for n, child_book in enumerate(children):
 525             new_child = child_book.parent != book
 526             child_book.parent = book
 527             child_book.parent_number = n
 528             child_book.save()
 529             if new_child or cover_changed:
 530                 notify_cover_changed.append(child_book)
 531         # Disown unfaithful children and let them cope on their own.
 532         for child in obsolete_children:
 533             child.parent = None
 534             child.parent_number = 0
 535             child.save()
 536             if old_cover:
 537                 notify_cover_changed.append(child)
 538
 539         cls.repopulate_ancestors()
 540         tasks.update_counters.delay()
 541
 542         if remote_gallery_url:
 543             book.download_pictures(remote_gallery_url)
 544
 545         # No saves beyond this point.
 546
 547         # Build cover.
 548         if 'cover' not in dont_build:
 549             book.cover.build_delay()
 550             book.cover_thumb.build_delay()
 551             book.cover_api_thumb.build_delay()
 552             book.simple_cover.build_delay()
 553
 554         # Build HTML and ebooks.
 555         book.html_file.build_delay()
 556         if not children:
 557             for format_ in constants.EBOOK_FORMATS_WITHOUT_CHILDREN:
 558                 if format_ not in dont_build:
 559                     getattr(book, '%s_file' % format_).build_delay()
 560         for format_ in constants.EBOOK_FORMATS_WITH_CHILDREN:
 561             if format_ not in dont_build:
 562                 getattr(book, '%s_file' % format_).build_delay()
 563
 564         if not settings.NO_SEARCH_INDEX and search_index and findable:
 565             tasks.index_book.delay(book.id, book_info=book_info, index_tags=search_index_tags)
 566
 567         for child in notify_cover_changed:
 568             child.parent_cover_changed()
 569
 570         book.update_popularity()
 571         cls.published.send(sender=cls, instance=book)
 572         return book
 573
 574     @classmethod
 575     @transaction.atomic
 576     def repopulate_ancestors(cls):
 577         """Fixes the ancestry cache."""
 578         # TODO: table names
 579         cursor = connection.cursor()
 580         if connection.vendor == 'postgres':
 581             cursor.execute("TRUNCATE catalogue_book_ancestor")
 582             cursor.execute("""
 583                 WITH RECURSIVE ancestry AS (
 584                     SELECT book.id, book.parent_id
 585                     FROM catalogue_book AS book
 586                     WHERE book.parent_id IS NOT NULL
 587                     UNION
 588                     SELECT ancestor.id, book.parent_id
 589                     FROM ancestry AS ancestor, catalogue_book AS book
 590                     WHERE ancestor.parent_id = book.id
 591                         AND book.parent_id IS NOT NULL
 592                     )
 593                 INSERT INTO catalogue_book_ancestor
 594                     (from_book_id, to_book_id)
 595                     SELECT id, parent_id
 596                     FROM ancestry
 597                     ORDER BY id;
 598                 """)
 599         else:
 600             cursor.execute("DELETE FROM catalogue_book_ancestor")
 601             for b in cls.objects.exclude(parent=None):
 602                 parent = b.parent
 603                 while parent is not None:
 604                     b.ancestor.add(parent)
 605                     parent = parent.parent
 606
 607     def clear_cache(self):
 608         clear_cached_renders(self.mini_box)
 609         clear_cached_renders(self.mini_box_nolink)
 610
 611     def cover_info(self, inherit=True):
 612         """Returns a dictionary to serve as fallback for BookInfo.
 613
 614         For now, the only thing inherited is the cover image.
 615         """
 616         need = False
 617         info = {}
 618         for field in ('cover_url', 'cover_by', 'cover_source'):
 619             val = self.get_extra_info_json().get(field)
 620             if val:
 621                 info[field] = val
 622             else:
 623                 need = True
 624         if inherit and need and self.parent is not None:
 625             parent_info = self.parent.cover_info()
 626             parent_info.update(info)
 627             info = parent_info
 628         return info
 629
 630     def related_themes(self):
 631         return Tag.objects.usage_for_queryset(
 632             Fragment.objects.filter(models.Q(book=self) | models.Q(book__ancestor=self)),
 633             counts=True).filter(category='theme')
 634
 635     def parent_cover_changed(self):
 636         """Called when parent book's cover image is changed."""
 637         if not self.cover_info(inherit=False):
 638             if 'cover' not in app_settings.DONT_BUILD:
 639                 self.cover.build_delay()
 640                 self.cover_thumb.build_delay()
 641                 self.cover_api_thumb.build_delay()
 642                 self.simple_cover.build_delay()
 643             for format_ in constants.EBOOK_FORMATS_WITH_COVERS:
 644                 if format_ not in app_settings.DONT_BUILD:
 645                     getattr(self, '%s_file' % format_).build_delay()
 646             for child in self.children.all():
 647                 child.parent_cover_changed()
 648
 649     def other_versions(self):
 650         """Find other versions (i.e. in other languages) of the book."""
 651         return type(self).objects.filter(common_slug=self.common_slug, findable=True).exclude(pk=self.pk)
 652
 653     def parents(self):
 654         books = []
 655         parent = self.parent
 656         while parent is not None:
 657             books.insert(0, parent)
 658             parent = parent.parent
 659         return books
 660
 661     def pretty_title(self, html_links=False):
 662         names = [(tag.name, tag.get_absolute_url()) for tag in self.authors().only('name', 'category', 'slug')]
 663         books = self.parents() + [self]
 664         names.extend([(b.title, b.get_absolute_url()) for b in books])
 665
 666         if html_links:
 667             names = ['<a href="%s">%s</a>' % (tag[1], tag[0]) for tag in names]
 668         else:
 669             names = [tag[0] for tag in names]
 670         return ', '.join(names)
 671
 672     def publisher(self):
 673         publisher = self.get_extra_info_json()['publisher']
 674         if isinstance(publisher, str):
 675             return publisher
 676         elif isinstance(publisher, list):
 677             return ', '.join(publisher)
 678
 679     @classmethod
 680     def tagged_top_level(cls, tags):
 681         """ Returns top-level books tagged with `tags`.
 682
 683         It only returns those books which don't have ancestors which are
 684         also tagged with those tags.
 685
 686         """
 687         objects = cls.tagged.with_all(tags)
 688         return objects.filter(findable=True).exclude(ancestor__in=objects)
 689
 690     @classmethod
 691     def book_list(cls, book_filter=None):
 692         """Generates a hierarchical listing of all books.
 693
 694         Books are optionally filtered with a test function.
 695
 696         """
 697
 698         books_by_parent = {}
 699         books = cls.objects.filter(findable=True).order_by('parent_number', 'sort_key').only('title', 'parent', 'slug', 'extra_info')
 700         if book_filter:
 701             books = books.filter(book_filter).distinct()
 702
 703             book_ids = set(b['pk'] for b in books.values("pk").iterator())
 704             for book in books.iterator():
 705                 parent = book.parent_id
 706                 if parent not in book_ids:
 707                     parent = None
 708                 books_by_parent.setdefault(parent, []).append(book)
 709         else:
 710             for book in books.iterator():
 711                 books_by_parent.setdefault(book.parent_id, []).append(book)
 712
 713         orphans = []
 714         books_by_author = OrderedDict()
 715         for tag in Tag.objects.filter(category='author').iterator():
 716             books_by_author[tag] = []
 717
 718         for book in books_by_parent.get(None, ()):
 719             authors = list(book.authors().only('pk'))
 720             if authors:
 721                 for author in authors:
 722                     books_by_author[author].append(book)
 723             else:
 724                 orphans.append(book)
 725
 726         return books_by_author, orphans, books_by_parent
 727
 728     _audiences_pl = {
 729         "SP": (1, "szkoła podstawowa"),
 730         "SP1": (1, "szkoła podstawowa"),
 731         "SP2": (1, "szkoła podstawowa"),
 732         "SP3": (1, "szkoła podstawowa"),
 733         "P": (1, "szkoła podstawowa"),
 734         "G": (2, "gimnazjum"),
 735         "L": (3, "liceum"),
 736         "LP": (3, "liceum"),
 737     }
 738
 739     def audiences_pl(self):
 740         audiences = self.get_extra_info_json().get('audiences', [])
 741         audiences = sorted(set([self._audiences_pl.get(a, (99, a)) for a in audiences]))
 742         return [a[1] for a in audiences]
 743
 744     def stage_note(self):
 745         stage = self.get_extra_info_json().get('stage')
 746         if stage and stage < '0.4':
 747             return (_('This work needs modernisation'),
 748                     reverse('infopage', args=['wymagajace-uwspolczesnienia']))
 749         else:
 750             return None, None
 751
 752     def choose_fragment(self):
 753         fragments = self.fragments.order_by()
 754         fragments_count = fragments.count()
 755         if not fragments_count and self.children.exists():
 756             fragments = Fragment.objects.filter(book__ancestor=self).order_by()
 757             fragments_count = fragments.count()
 758         if fragments_count:
 759             return fragments[randint(0, fragments_count - 1)]
 760         elif self.parent:
 761             return self.parent.choose_fragment()
 762         else:
 763             return None
 764
 765     def fragment_data(self):
 766         fragment = self.choose_fragment()
 767         if fragment:
 768             return {
 769                 'title': fragment.book.pretty_title(),
 770                 'html': re.sub('</?blockquote[^>]*>', '', fragment.get_short_text()),
 771             }
 772         else:
 773             return None
 774
 775     def update_popularity(self):
 776         count = self.tags.filter(category='set').values('user').order_by('user').distinct().count()
 777         try:
 778             pop = self.popularity
 779             pop.count = count
 780             pop.save()
 781         except BookPopularity.DoesNotExist:
 782             BookPopularity.objects.create(book=self, count=count)
 783
 784     def ridero_link(self):
 785         return 'https://ridero.eu/%s/books/wl_%s/' % (get_language(), self.slug.replace('-', '_'))
 786
 787     def like(self, user):
 788         from social.utils import likes, get_set, set_sets
 789         if not likes(user, self):
 790             tag = get_set(user, '')
 791             set_sets(user, self, [tag])
 792
 793     def unlike(self, user):
 794         from social.utils import likes, set_sets
 795         if likes(user, self):
 796             set_sets(user, self, [])
 797
 798     def full_sort_key(self):
 799         return self.SORT_KEY_SEP.join((self.sort_key_author, self.sort_key, str(self.id)))
 800
 801     def cover_color(self):
 802         return WLCover.epoch_colors.get(self.get_extra_info_json().get('epoch'), '#000000')
 803
 804     @cached_render('catalogue/book_mini_box.html')
 805     def mini_box(self):
 806         return {
 807             'book': self
 808         }
 809
 810     @cached_render('catalogue/book_mini_box.html')
 811     def mini_box_nolink(self):
 812         return {
 813             'book': self,
 814             'no_link': True,
 815         }
 816
 817 def add_file_fields():
 818     for format_ in Book.formats:
 819         field_name = "%s_file" % format_
 820         # This weird globals() assignment makes Django migrations comfortable.
 821         _upload_to = _ebook_upload_to('book/%s/%%s.%s' % (format_, format_))
 822         _upload_to.__name__ = '_%s_upload_to' % format_
 823         globals()[_upload_to.__name__] = _upload_to
 824
 825         EbookField(
 826             format_, _("%s file" % format_.upper()),
 827             upload_to=_upload_to,
 828             storage=bofh_storage,
 829             max_length=255,
 830             blank=True,
 831             default=''
 832         ).contribute_to_class(Book, field_name)
 833
 834
 835 add_file_fields()
 836
 837
 838 class BookPopularity(models.Model):
 839     book = models.OneToOneField(Book, models.CASCADE, related_name='popularity')
 840     count = models.IntegerField(default=0, db_index=True)