src/catalogue/models/book.py

   1 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
   2 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
   3 #
   4 from collections import OrderedDict
   5 import json
   6 from datetime import date, timedelta
   7 from random import randint
   8 import os.path
   9 import re
  10 from urllib.request import urlretrieve
  11 from django.conf import settings
  12 from django.db import connection, models, transaction
  13 import django.dispatch
  14 from django.contrib.contenttypes.fields import GenericRelation
  15 from django.template.loader import render_to_string
  16 from django.urls import reverse
  17 from django.utils.translation import ugettext_lazy as _, get_language
  18 from django.utils.deconstruct import deconstructible
  19 from fnpdjango.storage import BofhFileSystemStorage
  20
  21 from librarian.cover import WLCover
  22 from librarian.html import transform_abstrakt
  23 from newtagging import managers
  24 from catalogue import constants
  25 from catalogue.fields import EbookField
  26 from catalogue.models import Tag, Fragment, BookMedia
  27 from catalogue.utils import create_zip, gallery_url, gallery_path, split_tags, get_random_hash
  28 from catalogue.models.tag import prefetched_relations
  29 from catalogue import app_settings
  30 from catalogue import tasks
  31 from wolnelektury.utils import makedirs, cached_render, clear_cached_renders
  32
  33 bofh_storage = BofhFileSystemStorage()
  34
  35
  36 @deconstructible
  37 class UploadToPath(object):
  38     def __init__(self, path):
  39         self.path = path
  40
  41     def __call__(self, instance, filename):
  42         return self.path % instance.slug
  43
  44
  45 _cover_upload_to = UploadToPath('book/cover/%s.jpg')
  46 _cover_thumb_upload_to = UploadToPath('book/cover_thumb/%s.jpg')
  47 _cover_api_thumb_upload_to = UploadToPath('book/cover_api_thumb/%s.jpg')
  48 _simple_cover_upload_to = UploadToPath('book/cover_simple/%s.jpg')
  49 _cover_ebookpoint_upload_to = UploadToPath('book/cover_ebookpoint/%s.jpg')
  50
  51
  52 def _ebook_upload_to(upload_path):
  53     return UploadToPath(upload_path)
  54
  55
  56 class Book(models.Model):
  57     """Represents a book imported from WL-XML."""
  58     title = models.CharField(_('title'), max_length=32767)
  59     sort_key = models.CharField(_('sort key'), max_length=120, db_index=True, editable=False)
  60     sort_key_author = models.CharField(
  61         _('sort key by author'), max_length=120, db_index=True, editable=False, default='')
  62     slug = models.SlugField(_('slug'), max_length=120, db_index=True, unique=True)
  63     common_slug = models.SlugField(_('slug'), max_length=120, db_index=True)
  64     language = models.CharField(_('language code'), max_length=3, db_index=True, default=app_settings.DEFAULT_LANGUAGE)
  65     description = models.TextField(_('description'), blank=True)
  66     abstract = models.TextField(_('abstract'), blank=True)
  67     created_at = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True)
  68     changed_at = models.DateTimeField(_('change date'), auto_now=True, db_index=True)
  69     parent_number = models.IntegerField(_('parent number'), default=0)
  70     extra_info = models.TextField(_('extra information'), default='{}')
  71     gazeta_link = models.CharField(blank=True, max_length=240)
  72     wiki_link = models.CharField(blank=True, max_length=240)
  73     print_on_demand = models.BooleanField(_('print on demand'), default=False)
  74     recommended = models.BooleanField(_('recommended'), default=False)
  75     audio_length = models.CharField(_('audio length'), blank=True, max_length=8)
  76     preview = models.BooleanField(_('preview'), default=False)
  77     preview_until = models.DateField(_('preview until'), blank=True, null=True)
  78     preview_key = models.CharField(max_length=32, blank=True, null=True)
  79     findable = models.BooleanField(_('findable'), default=True, db_index=True)
  80
  81     # files generated during publication
  82     cover = EbookField(
  83         'cover', _('cover'),
  84         null=True, blank=True,
  85         upload_to=_cover_upload_to,
  86         storage=bofh_storage, max_length=255)
  87     cover_etag = models.CharField(max_length=255, editable=False, default='', db_index=True)
  88     # Cleaner version of cover for thumbs
  89     cover_thumb = EbookField(
  90         'cover_thumb', _('cover thumbnail'),
  91         null=True, blank=True,
  92         upload_to=_cover_thumb_upload_to,
  93         max_length=255)
  94     cover_thumb_etag = models.CharField(max_length=255, editable=False, default='', db_index=True)
  95     cover_api_thumb = EbookField(
  96         'cover_api_thumb', _('cover thumbnail for mobile app'),
  97         null=True, blank=True,
  98         upload_to=_cover_api_thumb_upload_to,
  99         max_length=255)
 100     cover_api_thumb_etag = models.CharField(max_length=255, editable=False, default='', db_index=True)
 101     simple_cover = EbookField(
 102         'simple_cover', _('cover for mobile app'),
 103         null=True, blank=True,
 104         upload_to=_simple_cover_upload_to,
 105         max_length=255)
 106     simple_cover_etag = models.CharField(max_length=255, editable=False, default='', db_index=True)
 107     cover_ebookpoint = EbookField(
 108         'cover_ebookpoint', _('cover for Ebookpoint'),
 109         null=True, blank=True,
 110         upload_to=_cover_ebookpoint_upload_to,
 111         max_length=255)
 112     cover_ebookpoint_etag = models.CharField(max_length=255, editable=False, default='', db_index=True)
 113     ebook_formats = constants.EBOOK_FORMATS
 114     formats = ebook_formats + ['html', 'xml']
 115
 116     parent = models.ForeignKey('self', models.CASCADE, blank=True, null=True, related_name='children')
 117     ancestor = models.ManyToManyField('self', blank=True, editable=False, related_name='descendant', symmetrical=False)
 118
 119     cached_author = models.CharField(blank=True, max_length=240, db_index=True)
 120     has_audience = models.BooleanField(default=False)
 121
 122     objects = models.Manager()
 123     tagged = managers.ModelTaggedItemManager(Tag)
 124     tags = managers.TagDescriptor(Tag)
 125     tag_relations = GenericRelation(Tag.intermediary_table_model)
 126
 127     html_built = django.dispatch.Signal()
 128     published = django.dispatch.Signal()
 129
 130     SORT_KEY_SEP = '$'
 131
 132     class AlreadyExists(Exception):
 133         pass
 134
 135     class Meta:
 136         ordering = ('sort_key_author', 'sort_key')
 137         verbose_name = _('book')
 138         verbose_name_plural = _('books')
 139         app_label = 'catalogue'
 140
 141     def __str__(self):
 142         return self.title
 143
 144     def get_extra_info_json(self):
 145         return json.loads(self.extra_info or '{}')
 146
 147     def get_initial(self):
 148         try:
 149             return re.search(r'\w', self.title, re.U).group(0)
 150         except AttributeError:
 151             return ''
 152
 153     def authors(self):
 154         return self.tags.filter(category='author')
 155
 156     def epochs(self):
 157         return self.tags.filter(category='epoch')
 158
 159     def genres(self):
 160         return self.tags.filter(category='genre')
 161
 162     def kinds(self):
 163         return self.tags.filter(category='kind')
 164
 165     def tag_unicode(self, category):
 166         relations = prefetched_relations(self, category)
 167         if relations:
 168             return ', '.join(rel.tag.name for rel in relations)
 169         else:
 170             return ', '.join(self.tags.filter(category=category).values_list('name', flat=True))
 171
 172     def tags_by_category(self):
 173         return split_tags(self.tags.exclude(category__in=('set', 'theme')))
 174
 175     def author_unicode(self):
 176         return self.cached_author
 177
 178     def kind_unicode(self):
 179         return self.tag_unicode('kind')
 180
 181     def epoch_unicode(self):
 182         return self.tag_unicode('epoch')
 183
 184     def genre_unicode(self):
 185         return self.tag_unicode('genre')
 186
 187     def translators(self):
 188         translators = self.get_extra_info_json().get('translators') or []
 189         return [
 190             '\xa0'.join(reversed(translator.split(', ', 1))) for translator in translators
 191         ]
 192
 193     def translator(self):
 194         translators = self.get_extra_info_json().get('translators')
 195         if not translators:
 196             return None
 197         if len(translators) > 3:
 198             translators = translators[:2]
 199             others = ' i inni'
 200         else:
 201             others = ''
 202         return ', '.join('\xa0'.join(reversed(translator.split(', ', 1))) for translator in translators) + others
 203
 204     def cover_source(self):
 205         return self.get_extra_info_json().get('cover_source', self.parent.cover_source() if self.parent else '')
 206
 207     @property
 208     def isbn_pdf(self):
 209         return self.get_extra_info_json().get('isbn_pdf')
 210
 211     @property
 212     def isbn_epub(self):
 213         return self.get_extra_info_json().get('isbn_epub')
 214
 215     @property
 216     def isbn_mobi(self):
 217         return self.get_extra_info_json().get('isbn_mobi')
 218
 219
 220     def save(self, force_insert=False, force_update=False, **kwargs):
 221         from sortify import sortify
 222
 223         self.sort_key = sortify(self.title)[:120]
 224         self.title = str(self.title)  # ???
 225
 226         try:
 227             author = self.authors().first().sort_key
 228         except AttributeError:
 229             author = ''
 230         self.sort_key_author = author
 231
 232         self.cached_author = self.tag_unicode('author')
 233         self.has_audience = 'audience' in self.get_extra_info_json()
 234
 235         if self.preview and not self.preview_key:
 236             self.preview_key = get_random_hash(self.slug)[:32]
 237
 238         ret = super(Book, self).save(force_insert, force_update, **kwargs)
 239
 240         return ret
 241
 242     def get_absolute_url(self):
 243         return reverse('book_detail', args=[self.slug])
 244
 245     def gallery_path(self):
 246         return gallery_path(self.slug)
 247
 248     def gallery_url(self):
 249         return gallery_url(self.slug)
 250
 251     def get_first_text(self):
 252         if self.html_file:
 253             return self
 254         child = self.children.all().order_by('parent_number').first()
 255         if child is not None:
 256             return child.get_first_text()
 257
 258     def get_last_text(self):
 259         if self.html_file:
 260             return self
 261         child = self.children.all().order_by('parent_number').last()
 262         if child is not None:
 263             return child.get_last_text()
 264
 265     def get_prev_text(self):
 266         if not self.parent:
 267             return None
 268         sibling = self.parent.children.filter(parent_number__lt=self.parent_number).order_by('-parent_number').first()
 269         if sibling is not None:
 270             return sibling.get_last_text()
 271         return self.parent.get_prev_text()
 272
 273     def get_next_text(self):
 274         if not self.parent:
 275             return None
 276         sibling = self.parent.children.filter(parent_number__gt=self.parent_number).order_by('parent_number').first()
 277         if sibling is not None:
 278             return sibling.get_first_text()
 279         return self.parent.get_next_text()
 280
 281     def get_siblings(self):
 282         if not self.parent:
 283             return []
 284         return self.parent.children.all().order_by('parent_number')
 285
 286     @property
 287     def name(self):
 288         return self.title
 289
 290     def language_code(self):
 291         return constants.LANGUAGES_3TO2.get(self.language, self.language)
 292
 293     def language_name(self):
 294         return dict(settings.LANGUAGES).get(self.language_code(), "")
 295
 296     def is_foreign(self):
 297         return self.language_code() != settings.LANGUAGE_CODE
 298
 299     def set_audio_length(self):
 300         length = self.get_audio_length()
 301         if length > 0:
 302             self.audio_length = self.format_audio_length(length)
 303             self.save()
 304
 305     @staticmethod
 306     def format_audio_length(seconds):
 307         """
 308         >>> Book.format_audio_length(1)
 309         '0:01'
 310         >>> Book.format_audio_length(3661)
 311         '1:01:01'
 312         """
 313         if seconds < 60*60:
 314             minutes = seconds // 60
 315             seconds = seconds % 60
 316             return '%d:%02d' % (minutes, seconds)
 317         else:
 318             hours = seconds // 3600
 319             minutes = seconds % 3600 // 60
 320             seconds = seconds % 60
 321             return '%d:%02d:%02d' % (hours, minutes, seconds)
 322
 323     def get_audio_length(self):
 324         total = 0
 325         for media in self.get_mp3() or ():
 326             total += app_settings.GET_MP3_LENGTH(media.file.path)
 327         return int(total)
 328
 329     def has_media(self, type_):
 330         if type_ in Book.formats:
 331             return bool(getattr(self, "%s_file" % type_))
 332         else:
 333             return self.media.filter(type=type_).exists()
 334
 335     def has_audio(self):
 336         return self.has_media('mp3')
 337
 338     def get_media(self, type_):
 339         if self.has_media(type_):
 340             if type_ in Book.formats:
 341                 return getattr(self, "%s_file" % type_)
 342             else:
 343                 return self.media.filter(type=type_)
 344         else:
 345             return None
 346
 347     def get_mp3(self):
 348         return self.get_media("mp3")
 349
 350     def get_odt(self):
 351         return self.get_media("odt")
 352
 353     def get_ogg(self):
 354         return self.get_media("ogg")
 355
 356     def get_daisy(self):
 357         return self.get_media("daisy")
 358
 359     def media_url(self, format_):
 360         media = self.get_media(format_)
 361         if media:
 362             if self.preview:
 363                 return reverse('embargo_link', kwargs={'key': self.preview_key, 'slug': self.slug, 'format_': format_})
 364             else:
 365                 return media.url
 366         else:
 367             return None
 368
 369     def html_url(self):
 370         return self.media_url('html')
 371
 372     def pdf_url(self):
 373         return self.media_url('pdf')
 374
 375     def epub_url(self):
 376         return self.media_url('epub')
 377
 378     def mobi_url(self):
 379         return self.media_url('mobi')
 380
 381     def txt_url(self):
 382         return self.media_url('txt')
 383
 384     def fb2_url(self):
 385         return self.media_url('fb2')
 386
 387     def xml_url(self):
 388         return self.media_url('xml')
 389
 390     def has_description(self):
 391         return len(self.description) > 0
 392     has_description.short_description = _('description')
 393     has_description.boolean = True
 394
 395     def has_mp3_file(self):
 396         return self.has_media("mp3")
 397     has_mp3_file.short_description = 'MP3'
 398     has_mp3_file.boolean = True
 399
 400     def has_ogg_file(self):
 401         return self.has_media("ogg")
 402     has_ogg_file.short_description = 'OGG'
 403     has_ogg_file.boolean = True
 404
 405     def has_daisy_file(self):
 406         return self.has_media("daisy")
 407     has_daisy_file.short_description = 'DAISY'
 408     has_daisy_file.boolean = True
 409
 410     def get_audiobooks(self):
 411         ogg_files = {}
 412         for m in self.media.filter(type='ogg').order_by().iterator():
 413             ogg_files[m.name] = m
 414
 415         audiobooks = []
 416         projects = set()
 417         for mp3 in self.media.filter(type='mp3').iterator():
 418             # ogg files are always from the same project
 419             meta = mp3.get_extra_info_json()
 420             project = meta.get('project')
 421             if not project:
 422                 # temporary fallback
 423                 project = 'CzytamySłuchając'
 424
 425             projects.add((project, meta.get('funded_by', '')))
 426
 427             media = {'mp3': mp3}
 428
 429             ogg = ogg_files.get(mp3.name)
 430             if ogg:
 431                 media['ogg'] = ogg
 432             audiobooks.append(media)
 433
 434         projects = sorted(projects)
 435         return audiobooks, projects
 436
 437     def wldocument(self, parse_dublincore=True, inherit=True):
 438         from catalogue.import_utils import ORMDocProvider
 439         from librarian.parser import WLDocument
 440
 441         if inherit and self.parent:
 442             meta_fallbacks = self.parent.cover_info()
 443         else:
 444             meta_fallbacks = None
 445
 446         return WLDocument.from_file(
 447             self.xml_file.path,
 448             provider=ORMDocProvider(self),
 449             parse_dublincore=parse_dublincore,
 450             meta_fallbacks=meta_fallbacks)
 451
 452     @staticmethod
 453     def zip_format(format_):
 454         def pretty_file_name(book):
 455             return "%s/%s.%s" % (
 456                 book.get_extra_info_json()['author'],
 457                 book.slug,
 458                 format_)
 459
 460         field_name = "%s_file" % format_
 461         books = Book.objects.filter(parent=None).exclude(**{field_name: ""}).exclude(preview=True).exclude(findable=False)
 462         paths = [(pretty_file_name(b), getattr(b, field_name).path) for b in books.iterator()]
 463         return create_zip(paths, app_settings.FORMAT_ZIPS[format_])
 464
 465     def zip_audiobooks(self, format_):
 466         bm = BookMedia.objects.filter(book=self, type=format_)
 467         paths = map(lambda bm: (None, bm.file.path), bm)
 468         licenses = set()
 469         for m in bm:
 470             license = constants.LICENSES.get(
 471                 m.get_extra_info_json().get('license'), {}
 472             ).get('locative')
 473             if license:
 474                 licenses.add(license)
 475         readme = render_to_string('catalogue/audiobook_zip_readme.txt', {
 476             'licenses': licenses,
 477         })
 478         return create_zip(paths, "%s_%s" % (self.slug, format_), {'informacje.txt': readme})
 479
 480     def search_index(self, book_info=None, index=None, index_tags=True, commit=True):
 481         if not self.findable:
 482             return
 483         if index is None:
 484             from search.index import Index
 485             index = Index()
 486         try:
 487             index.index_book(self, book_info)
 488             if index_tags:
 489                 index.index_tags()
 490             if commit:
 491                 index.index.commit()
 492         except Exception as e:
 493             index.index.rollback()
 494             raise e
 495
 496     # will make problems in conjunction with paid previews
 497     def download_pictures(self, remote_gallery_url):
 498         gallery_path = self.gallery_path()
 499         # delete previous files, so we don't include old files in ebooks
 500         if os.path.isdir(gallery_path):
 501             for filename in os.listdir(gallery_path):
 502                 file_path = os.path.join(gallery_path, filename)
 503                 os.unlink(file_path)
 504         ilustr_elements = list(self.wldocument().edoc.findall('//ilustr'))
 505         if ilustr_elements:
 506             makedirs(gallery_path)
 507             for ilustr in ilustr_elements:
 508                 ilustr_src = ilustr.get('src')
 509                 ilustr_path = os.path.join(gallery_path, ilustr_src)
 510                 urlretrieve('%s/%s' % (remote_gallery_url, ilustr_src), ilustr_path)
 511
 512     def load_abstract(self):
 513         abstract = self.wldocument(parse_dublincore=False).edoc.getroot().find('.//abstrakt')
 514         if abstract is not None:
 515             self.abstract = transform_abstrakt(abstract)
 516         else:
 517             self.abstract = ''
 518
 519     @classmethod
 520     def from_xml_file(cls, xml_file, **kwargs):
 521         from django.core.files import File
 522         from librarian import dcparser
 523
 524         # use librarian to parse meta-data
 525         book_info = dcparser.parse(xml_file)
 526
 527         if not isinstance(xml_file, File):
 528             xml_file = File(open(xml_file))
 529
 530         try:
 531             return cls.from_text_and_meta(xml_file, book_info, **kwargs)
 532         finally:
 533             xml_file.close()
 534
 535     @classmethod
 536     def from_text_and_meta(cls, raw_file, book_info, overwrite=False, dont_build=None, search_index=True,
 537                            search_index_tags=True, remote_gallery_url=None, days=0, findable=True):
 538         if dont_build is None:
 539             dont_build = set()
 540         dont_build = set.union(set(dont_build), set(app_settings.DONT_BUILD))
 541
 542         # check for parts before we do anything
 543         children = []
 544         if hasattr(book_info, 'parts'):
 545             for part_url in book_info.parts:
 546                 try:
 547                     children.append(Book.objects.get(slug=part_url.slug))
 548                 except Book.DoesNotExist:
 549                     raise Book.DoesNotExist(_('Book "%s" does not exist.') % part_url.slug)
 550
 551         # Read book metadata
 552         book_slug = book_info.url.slug
 553         if re.search(r'[^a-z0-9-]', book_slug):
 554             raise ValueError('Invalid characters in slug')
 555         book, created = Book.objects.get_or_create(slug=book_slug)
 556
 557         if created:
 558             book_shelves = []
 559             old_cover = None
 560             book.preview = bool(days)
 561             if book.preview:
 562                 book.preview_until = date.today() + timedelta(days)
 563         else:
 564             if not overwrite:
 565                 raise Book.AlreadyExists(_('Book %s already exists') % book_slug)
 566             # Save shelves for this book
 567             book_shelves = list(book.tags.filter(category='set'))
 568             old_cover = book.cover_info()
 569
 570         # Save XML file
 571         book.xml_file.save('%s.xml' % book.slug, raw_file, save=False)
 572         if book.preview:
 573             book.xml_file.set_readable(False)
 574
 575         book.findable = findable
 576         book.language = book_info.language
 577         book.title = book_info.title
 578         if book_info.variant_of:
 579             book.common_slug = book_info.variant_of.slug
 580         else:
 581             book.common_slug = book.slug
 582         book.extra_info = json.dumps(book_info.to_dict())
 583         book.load_abstract()
 584         book.save()
 585
 586         meta_tags = Tag.tags_from_info(book_info)
 587
 588         for tag in meta_tags:
 589             if not tag.for_books:
 590                 tag.for_books = True
 591                 tag.save()
 592
 593         book.tags = set(meta_tags + book_shelves)
 594         book.save()  # update sort_key_author
 595
 596         cover_changed = old_cover != book.cover_info()
 597         obsolete_children = set(b for b in book.children.all()
 598                                 if b not in children)
 599         notify_cover_changed = []
 600         for n, child_book in enumerate(children):
 601             new_child = child_book.parent != book
 602             child_book.parent = book
 603             child_book.parent_number = n
 604             child_book.save()
 605             if new_child or cover_changed:
 606                 notify_cover_changed.append(child_book)
 607         # Disown unfaithful children and let them cope on their own.
 608         for child in obsolete_children:
 609             child.parent = None
 610             child.parent_number = 0
 611             child.save()
 612             if old_cover:
 613                 notify_cover_changed.append(child)
 614
 615         cls.repopulate_ancestors()
 616         tasks.update_counters.delay()
 617
 618         if remote_gallery_url:
 619             book.download_pictures(remote_gallery_url)
 620
 621         # No saves beyond this point.
 622
 623         # Build cover.
 624         if 'cover' not in dont_build:
 625             book.cover.build_delay()
 626             book.cover_thumb.build_delay()
 627             book.cover_api_thumb.build_delay()
 628             book.simple_cover.build_delay()
 629             book.cover_ebookpoint.build_delay()
 630
 631         # Build HTML and ebooks.
 632         book.html_file.build_delay()
 633         if not children:
 634             for format_ in constants.EBOOK_FORMATS_WITHOUT_CHILDREN:
 635                 if format_ not in dont_build:
 636                     getattr(book, '%s_file' % format_).build_delay()
 637         for format_ in constants.EBOOK_FORMATS_WITH_CHILDREN:
 638             if format_ not in dont_build:
 639                 getattr(book, '%s_file' % format_).build_delay()
 640
 641         if not settings.NO_SEARCH_INDEX and search_index and findable:
 642             tasks.index_book.delay(book.id, book_info=book_info, index_tags=search_index_tags)
 643
 644         for child in notify_cover_changed:
 645             child.parent_cover_changed()
 646
 647         book.update_popularity()
 648         tasks.update_references.delay(book.id)
 649
 650         cls.published.send(sender=cls, instance=book)
 651         return book
 652
 653     def get_master(self):
 654         master_tags = [
 655             'opowiadanie',
 656             'powiesc',
 657             'dramat_wierszowany_l',
 658             'dramat_wierszowany_lp',
 659             'dramat_wspolczesny', 'liryka_l', 'liryka_lp',
 660             'wywiad',
 661         ]
 662         from librarian.parser import WLDocument
 663         wld = WLDocument.from_file(self.xml_file.path, parse_dublincore=False)
 664         root = wld.edoc.getroot()
 665         for master in root.iter():
 666             if master.tag in master_tags:
 667                 return master
 668
 669     def update_references(self):
 670         from references.models import Entity, Reference
 671         master = self.get_master()
 672         found = set()
 673         for i, sec in enumerate(master):
 674             for ref in sec.findall('.//ref'):
 675                 href = ref.attrib.get('href', '')
 676                 if not href or href in found:
 677                     continue
 678                 found.add(href)
 679                 entity, created = Entity.objects.get_or_create(
 680                     uri=href
 681                 )
 682                 ref, created = Reference.objects.get_or_create(
 683                     book=self,
 684                     entity=entity
 685                 )
 686                 ref.first_section = 'sec%d' % (i + 1)
 687                 entity.populate()
 688                 entity.save()
 689         Reference.objects.filter(book=self).exclude(entity__uri__in=found).delete()
 690
 691     @property
 692     def references(self):
 693         return self.reference_set.all().select_related('entity')
 694
 695     @classmethod
 696     @transaction.atomic
 697     def repopulate_ancestors(cls):
 698         """Fixes the ancestry cache."""
 699         # TODO: table names
 700         cursor = connection.cursor()
 701         if connection.vendor == 'postgres':
 702             cursor.execute("TRUNCATE catalogue_book_ancestor")
 703             cursor.execute("""
 704                 WITH RECURSIVE ancestry AS (
 705                     SELECT book.id, book.parent_id
 706                     FROM catalogue_book AS book
 707                     WHERE book.parent_id IS NOT NULL
 708                     UNION
 709                     SELECT ancestor.id, book.parent_id
 710                     FROM ancestry AS ancestor, catalogue_book AS book
 711                     WHERE ancestor.parent_id = book.id
 712                         AND book.parent_id IS NOT NULL
 713                     )
 714                 INSERT INTO catalogue_book_ancestor
 715                     (from_book_id, to_book_id)
 716                     SELECT id, parent_id
 717                     FROM ancestry
 718                     ORDER BY id;
 719                 """)
 720         else:
 721             cursor.execute("DELETE FROM catalogue_book_ancestor")
 722             for b in cls.objects.exclude(parent=None):
 723                 parent = b.parent
 724                 while parent is not None:
 725                     b.ancestor.add(parent)
 726                     parent = parent.parent
 727
 728     def clear_cache(self):
 729         clear_cached_renders(self.mini_box)
 730         clear_cached_renders(self.mini_box_nolink)
 731
 732     def cover_info(self, inherit=True):
 733         """Returns a dictionary to serve as fallback for BookInfo.
 734
 735         For now, the only thing inherited is the cover image.
 736         """
 737         need = False
 738         info = {}
 739         for field in ('cover_url', 'cover_by', 'cover_source'):
 740             val = self.get_extra_info_json().get(field)
 741             if val:
 742                 info[field] = val
 743             else:
 744                 need = True
 745         if inherit and need and self.parent is not None:
 746             parent_info = self.parent.cover_info()
 747             parent_info.update(info)
 748             info = parent_info
 749         return info
 750
 751     def related_themes(self):
 752         return Tag.objects.usage_for_queryset(
 753             Fragment.objects.filter(models.Q(book=self) | models.Q(book__ancestor=self)),
 754             counts=True).filter(category='theme')
 755
 756     def parent_cover_changed(self):
 757         """Called when parent book's cover image is changed."""
 758         if not self.cover_info(inherit=False):
 759             if 'cover' not in app_settings.DONT_BUILD:
 760                 self.cover.build_delay()
 761                 self.cover_thumb.build_delay()
 762                 self.cover_api_thumb.build_delay()
 763                 self.simple_cover.build_delay()
 764             for format_ in constants.EBOOK_FORMATS_WITH_COVERS:
 765                 if format_ not in app_settings.DONT_BUILD:
 766                     getattr(self, '%s_file' % format_).build_delay()
 767             for child in self.children.all():
 768                 child.parent_cover_changed()
 769
 770     def other_versions(self):
 771         """Find other versions (i.e. in other languages) of the book."""
 772         return type(self).objects.filter(common_slug=self.common_slug, findable=True).exclude(pk=self.pk)
 773
 774     def parents(self):
 775         books = []
 776         parent = self.parent
 777         while parent is not None:
 778             books.insert(0, parent)
 779             parent = parent.parent
 780         return books
 781
 782     def pretty_title(self, html_links=False):
 783         names = [(tag.name, tag.get_absolute_url()) for tag in self.authors().only('name', 'category', 'slug')]
 784         books = self.parents() + [self]
 785         names.extend([(b.title, b.get_absolute_url()) for b in books])
 786
 787         if html_links:
 788             names = ['<a href="%s">%s</a>' % (tag[1], tag[0]) for tag in names]
 789         else:
 790             names = [tag[0] for tag in names]
 791         return ', '.join(names)
 792
 793     def publisher(self):
 794         publisher = self.get_extra_info_json()['publisher']
 795         if isinstance(publisher, str):
 796             return publisher
 797         elif isinstance(publisher, list):
 798             return ', '.join(publisher)
 799
 800     @classmethod
 801     def tagged_top_level(cls, tags):
 802         """ Returns top-level books tagged with `tags`.
 803
 804         It only returns those books which don't have ancestors which are
 805         also tagged with those tags.
 806
 807         """
 808         objects = cls.tagged.with_all(tags)
 809         return objects.filter(findable=True).exclude(ancestor__in=objects)
 810
 811     @classmethod
 812     def book_list(cls, book_filter=None):
 813         """Generates a hierarchical listing of all books.
 814
 815         Books are optionally filtered with a test function.
 816
 817         """
 818
 819         books_by_parent = {}
 820         books = cls.objects.filter(findable=True).order_by('parent_number', 'sort_key').only('title', 'parent', 'slug', 'extra_info')
 821         if book_filter:
 822             books = books.filter(book_filter).distinct()
 823
 824             book_ids = set(b['pk'] for b in books.values("pk").iterator())
 825             for book in books.iterator():
 826                 parent = book.parent_id
 827                 if parent not in book_ids:
 828                     parent = None
 829                 books_by_parent.setdefault(parent, []).append(book)
 830         else:
 831             for book in books.iterator():
 832                 books_by_parent.setdefault(book.parent_id, []).append(book)
 833
 834         orphans = []
 835         books_by_author = OrderedDict()
 836         for tag in Tag.objects.filter(category='author').iterator():
 837             books_by_author[tag] = []
 838
 839         for book in books_by_parent.get(None, ()):
 840             authors = list(book.authors().only('pk'))
 841             if authors:
 842                 for author in authors:
 843                     books_by_author[author].append(book)
 844             else:
 845                 orphans.append(book)
 846
 847         return books_by_author, orphans, books_by_parent
 848
 849     _audiences_pl = {
 850         "SP": (1, "szkoła podstawowa"),
 851         "SP1": (1, "szkoła podstawowa"),
 852         "SP2": (1, "szkoła podstawowa"),
 853         "SP3": (1, "szkoła podstawowa"),
 854         "P": (1, "szkoła podstawowa"),
 855         "G": (2, "gimnazjum"),
 856         "L": (3, "liceum"),
 857         "LP": (3, "liceum"),
 858     }
 859
 860     def audiences_pl(self):
 861         audiences = self.get_extra_info_json().get('audiences', [])
 862         audiences = sorted(set([self._audiences_pl.get(a, (99, a)) for a in audiences]))
 863         return [a[1] for a in audiences]
 864
 865     def stage_note(self):
 866         stage = self.get_extra_info_json().get('stage')
 867         if stage and stage < '0.4':
 868             return (_('This work needs modernisation'),
 869                     reverse('infopage', args=['wymagajace-uwspolczesnienia']))
 870         else:
 871             return None, None
 872
 873     def choose_fragment(self):
 874         fragments = self.fragments.order_by()
 875         fragments_count = fragments.count()
 876         if not fragments_count and self.children.exists():
 877             fragments = Fragment.objects.filter(book__ancestor=self).order_by()
 878             fragments_count = fragments.count()
 879         if fragments_count:
 880             return fragments[randint(0, fragments_count - 1)]
 881         elif self.parent:
 882             return self.parent.choose_fragment()
 883         else:
 884             return None
 885
 886     def fragment_data(self):
 887         fragment = self.choose_fragment()
 888         if fragment:
 889             return {
 890                 'title': fragment.book.pretty_title(),
 891                 'html': re.sub('</?blockquote[^>]*>', '', fragment.get_short_text()),
 892             }
 893         else:
 894             return None
 895
 896     def update_popularity(self):
 897         count = self.tags.filter(category='set').values('user').order_by('user').distinct().count()
 898         try:
 899             pop = self.popularity
 900             pop.count = count
 901             pop.save()
 902         except BookPopularity.DoesNotExist:
 903             BookPopularity.objects.create(book=self, count=count)
 904
 905     def ridero_link(self):
 906         return 'https://ridero.eu/%s/books/wl_%s/' % (get_language(), self.slug.replace('-', '_'))
 907
 908     def like(self, user):
 909         from social.utils import likes, get_set, set_sets
 910         if not likes(user, self):
 911             tag = get_set(user, '')
 912             set_sets(user, self, [tag])
 913
 914     def unlike(self, user):
 915         from social.utils import likes, set_sets
 916         if likes(user, self):
 917             set_sets(user, self, [])
 918
 919     def full_sort_key(self):
 920         return self.SORT_KEY_SEP.join((self.sort_key_author, self.sort_key, str(self.id)))
 921
 922     def cover_color(self):
 923         return WLCover.epoch_colors.get(self.get_extra_info_json().get('epoch'), '#000000')
 924
 925     @cached_render('catalogue/book_mini_box.html')
 926     def mini_box(self):
 927         return {
 928             'book': self
 929         }
 930
 931     @cached_render('catalogue/book_mini_box.html')
 932     def mini_box_nolink(self):
 933         return {
 934             'book': self,
 935             'no_link': True,
 936         }
 937
 938 def add_file_fields():
 939     for format_ in Book.formats:
 940         field_name = "%s_file" % format_
 941         # This weird globals() assignment makes Django migrations comfortable.
 942         _upload_to = _ebook_upload_to('book/%s/%%s.%s' % (format_, format_))
 943         _upload_to.__name__ = '_%s_upload_to' % format_
 944         globals()[_upload_to.__name__] = _upload_to
 945
 946         EbookField(
 947             format_, _("%s file" % format_.upper()),
 948             upload_to=_upload_to,
 949             storage=bofh_storage,
 950             max_length=255,
 951             blank=True,
 952             default=''
 953         ).contribute_to_class(Book, field_name)
 954         if format_ != 'xml':
 955             models.CharField(max_length=255, editable=False, default='', db_index=True).contribute_to_class(Book, f'{field_name}_etag')
 956
 957
 958 add_file_fields()
 959
 960
 961 class BookPopularity(models.Model):
 962     book = models.OneToOneField(Book, models.CASCADE, related_name='popularity')
 963     count = models.IntegerField(default=0, db_index=True)