src/catalogue/models/book.py

   1 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
   2 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
   3 #
   4 from collections import OrderedDict
   5 import json
   6 from datetime import date, timedelta
   7 from random import randint
   8 import os.path
   9 import re
  10 from urllib.request import urlretrieve
  11 from django.apps import apps
  12 from django.conf import settings
  13 from django.db import connection, models, transaction
  14 import django.dispatch
  15 from django.contrib.contenttypes.fields import GenericRelation
  16 from django.template.loader import render_to_string
  17 from django.urls import reverse
  18 from django.utils.translation import ugettext_lazy as _, get_language
  19 from django.utils.deconstruct import deconstructible
  20 from fnpdjango.storage import BofhFileSystemStorage
  21 from lxml import html
  22 from librarian.cover import WLCover
  23 from librarian.html import transform_abstrakt
  24 from newtagging import managers
  25 from catalogue import constants
  26 from catalogue.fields import EbookField
  27 from catalogue.models import Tag, Fragment, BookMedia
  28 from catalogue.utils import create_zip, gallery_url, gallery_path, split_tags, get_random_hash
  29 from catalogue.models.tag import prefetched_relations
  30 from catalogue import app_settings
  31 from catalogue import tasks
  32 from wolnelektury.utils import makedirs, cached_render, clear_cached_renders
  33
  34 bofh_storage = BofhFileSystemStorage()
  35
  36
  37 @deconstructible
  38 class UploadToPath(object):
  39     def __init__(self, path):
  40         self.path = path
  41
  42     def __call__(self, instance, filename):
  43         return self.path % instance.slug
  44
  45
  46 _cover_upload_to = UploadToPath('book/cover/%s.jpg')
  47 _cover_clean_upload_to = UploadToPath('book/cover_clean/%s.jpg')
  48 _cover_thumb_upload_to = UploadToPath('book/cover_thumb/%s.jpg')
  49 _cover_api_thumb_upload_to = UploadToPath('book/cover_api_thumb/%s.jpg')
  50 _simple_cover_upload_to = UploadToPath('book/cover_simple/%s.jpg')
  51 _cover_ebookpoint_upload_to = UploadToPath('book/cover_ebookpoint/%s.jpg')
  52
  53
  54 def _ebook_upload_to(upload_path):
  55     return UploadToPath(upload_path)
  56
  57
  58 class Book(models.Model):
  59     """Represents a book imported from WL-XML."""
  60     title = models.CharField(_('title'), max_length=32767)
  61     sort_key = models.CharField(_('sort key'), max_length=120, db_index=True, editable=False)
  62     sort_key_author = models.CharField(
  63         _('sort key by author'), max_length=120, db_index=True, editable=False, default='')
  64     slug = models.SlugField(_('slug'), max_length=120, db_index=True, unique=True)
  65     common_slug = models.SlugField(_('slug'), max_length=120, db_index=True)
  66     language = models.CharField(_('language code'), max_length=3, db_index=True, default=app_settings.DEFAULT_LANGUAGE)
  67     description = models.TextField(_('description'), blank=True)
  68     abstract = models.TextField(_('abstract'), blank=True)
  69     toc = models.TextField(_('toc'), blank=True)
  70     created_at = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True)
  71     changed_at = models.DateTimeField(_('change date'), auto_now=True, db_index=True)
  72     parent_number = models.IntegerField(_('parent number'), default=0)
  73     extra_info = models.TextField(_('extra information'), default='{}')
  74     gazeta_link = models.CharField(blank=True, max_length=240)
  75     wiki_link = models.CharField(blank=True, max_length=240)
  76     print_on_demand = models.BooleanField(_('print on demand'), default=False)
  77     recommended = models.BooleanField(_('recommended'), default=False)
  78     audio_length = models.CharField(_('audio length'), blank=True, max_length=8)
  79     preview = models.BooleanField(_('preview'), default=False)
  80     preview_until = models.DateField(_('preview until'), blank=True, null=True)
  81     preview_key = models.CharField(max_length=32, blank=True, null=True)
  82     findable = models.BooleanField(_('findable'), default=True, db_index=True)
  83
  84     # files generated during publication
  85     cover = EbookField(
  86         'cover', _('cover'),
  87         null=True, blank=True,
  88         upload_to=_cover_upload_to,
  89         storage=bofh_storage, max_length=255)
  90     cover_etag = models.CharField(max_length=255, editable=False, default='', db_index=True)
  91     # Cleaner version of cover for thumbs
  92     cover_clean = EbookField(
  93         'cover_clean', _('clean cover'),
  94         null=True, blank=True,
  95         upload_to=_cover_clean_upload_to,
  96         max_length=255
  97     )
  98     cover_clean_etag = models.CharField(max_length=255, editable=False, default='', db_index=True)
  99     cover_thumb = EbookField(
 100         'cover_thumb', _('cover thumbnail'),
 101         null=True, blank=True,
 102         upload_to=_cover_thumb_upload_to,
 103         max_length=255)
 104     cover_thumb_etag = models.CharField(max_length=255, editable=False, default='', db_index=True)
 105     cover_api_thumb = EbookField(
 106         'cover_api_thumb', _('cover thumbnail for mobile app'),
 107         null=True, blank=True,
 108         upload_to=_cover_api_thumb_upload_to,
 109         max_length=255)
 110     cover_api_thumb_etag = models.CharField(max_length=255, editable=False, default='', db_index=True)
 111     simple_cover = EbookField(
 112         'simple_cover', _('cover for mobile app'),
 113         null=True, blank=True,
 114         upload_to=_simple_cover_upload_to,
 115         max_length=255)
 116     simple_cover_etag = models.CharField(max_length=255, editable=False, default='', db_index=True)
 117     cover_ebookpoint = EbookField(
 118         'cover_ebookpoint', _('cover for Ebookpoint'),
 119         null=True, blank=True,
 120         upload_to=_cover_ebookpoint_upload_to,
 121         max_length=255)
 122     cover_ebookpoint_etag = models.CharField(max_length=255, editable=False, default='', db_index=True)
 123     ebook_formats = constants.EBOOK_FORMATS
 124     formats = ebook_formats + ['html', 'xml']
 125
 126     parent = models.ForeignKey('self', models.CASCADE, blank=True, null=True, related_name='children')
 127     ancestor = models.ManyToManyField('self', blank=True, editable=False, related_name='descendant', symmetrical=False)
 128
 129     cached_author = models.CharField(blank=True, max_length=240, db_index=True)
 130     has_audience = models.BooleanField(default=False)
 131
 132     objects = models.Manager()
 133     tagged = managers.ModelTaggedItemManager(Tag)
 134     tags = managers.TagDescriptor(Tag)
 135     tag_relations = GenericRelation(Tag.intermediary_table_model)
 136
 137     html_built = django.dispatch.Signal()
 138     published = django.dispatch.Signal()
 139
 140     SORT_KEY_SEP = '$'
 141
 142     class AlreadyExists(Exception):
 143         pass
 144
 145     class Meta:
 146         ordering = ('sort_key_author', 'sort_key')
 147         verbose_name = _('book')
 148         verbose_name_plural = _('books')
 149         app_label = 'catalogue'
 150
 151     def __str__(self):
 152         return self.title
 153
 154     def get_extra_info_json(self):
 155         return json.loads(self.extra_info or '{}')
 156
 157     def get_initial(self):
 158         try:
 159             return re.search(r'\w', self.title, re.U).group(0)
 160         except AttributeError:
 161             return ''
 162
 163     def authors(self):
 164         return self.tags.filter(category='author')
 165
 166     def epochs(self):
 167         return self.tags.filter(category='epoch')
 168
 169     def genres(self):
 170         return self.tags.filter(category='genre')
 171
 172     def kinds(self):
 173         return self.tags.filter(category='kind')
 174
 175     def tag_unicode(self, category):
 176         relations = prefetched_relations(self, category)
 177         if relations:
 178             return ', '.join(rel.tag.name for rel in relations)
 179         else:
 180             return ', '.join(self.tags.filter(category=category).values_list('name', flat=True))
 181
 182     def tags_by_category(self):
 183         return split_tags(self.tags.exclude(category__in=('set', 'theme')))
 184
 185     def author_unicode(self):
 186         return self.cached_author
 187
 188     def kind_unicode(self):
 189         return self.tag_unicode('kind')
 190
 191     def epoch_unicode(self):
 192         return self.tag_unicode('epoch')
 193
 194     def genre_unicode(self):
 195         return self.tag_unicode('genre')
 196
 197     def translators(self):
 198         translators = self.get_extra_info_json().get('translators') or []
 199         return [
 200             '\xa0'.join(reversed(translator.split(', ', 1))) for translator in translators
 201         ]
 202
 203     def translator(self):
 204         translators = self.get_extra_info_json().get('translators')
 205         if not translators:
 206             return None
 207         if len(translators) > 3:
 208             translators = translators[:2]
 209             others = ' i inni'
 210         else:
 211             others = ''
 212         return ', '.join('\xa0'.join(reversed(translator.split(', ', 1))) for translator in translators) + others
 213
 214     def cover_source(self):
 215         return self.get_extra_info_json().get('cover_source', self.parent.cover_source() if self.parent else '')
 216
 217     @property
 218     def isbn_pdf(self):
 219         return self.get_extra_info_json().get('isbn_pdf')
 220
 221     @property
 222     def isbn_epub(self):
 223         return self.get_extra_info_json().get('isbn_epub')
 224
 225     @property
 226     def isbn_mobi(self):
 227         return self.get_extra_info_json().get('isbn_mobi')
 228
 229     def is_accessible_to(self, user):
 230         if not self.preview:
 231             return True
 232         if not user.is_authenticated:
 233             return False
 234         Membership = apps.get_model('club', 'Membership')
 235         if Membership.is_active_for(user):
 236             return True
 237         Funding = apps.get_model('funding', 'Funding')
 238         if Funding.objects.filter(user=user, offer__book=self):
 239             return True
 240         return False
 241
 242     def save(self, force_insert=False, force_update=False, **kwargs):
 243         from sortify import sortify
 244
 245         self.sort_key = sortify(self.title)[:120]
 246         self.title = str(self.title)  # ???
 247
 248         try:
 249             author = self.authors().first().sort_key
 250         except AttributeError:
 251             author = ''
 252         self.sort_key_author = author
 253
 254         self.cached_author = self.tag_unicode('author')
 255         self.has_audience = 'audience' in self.get_extra_info_json()
 256
 257         if self.preview and not self.preview_key:
 258             self.preview_key = get_random_hash(self.slug)[:32]
 259
 260         ret = super(Book, self).save(force_insert, force_update, **kwargs)
 261
 262         return ret
 263
 264     def get_absolute_url(self):
 265         return reverse('book_detail', args=[self.slug])
 266
 267     def gallery_path(self):
 268         return gallery_path(self.slug)
 269
 270     def gallery_url(self):
 271         return gallery_url(self.slug)
 272
 273     def get_first_text(self):
 274         if self.html_file:
 275             return self
 276         child = self.children.all().order_by('parent_number').first()
 277         if child is not None:
 278             return child.get_first_text()
 279
 280     def get_last_text(self):
 281         if self.html_file:
 282             return self
 283         child = self.children.all().order_by('parent_number').last()
 284         if child is not None:
 285             return child.get_last_text()
 286
 287     def get_prev_text(self):
 288         if not self.parent:
 289             return None
 290         sibling = self.parent.children.filter(parent_number__lt=self.parent_number).order_by('-parent_number').first()
 291         if sibling is not None:
 292             return sibling.get_last_text()
 293
 294         if self.parent.html_file:
 295             return self.parent
 296
 297         return self.parent.get_prev_text()
 298
 299     def get_next_text(self):
 300         child = self.children.order_by('parent_number').first()
 301         if child is not None:
 302             return child.get_first_text()
 303
 304         if not self.parent:
 305             return None
 306         sibling = self.parent.children.filter(parent_number__gt=self.parent_number).order_by('parent_number').first()
 307         if sibling is not None:
 308             return sibling.get_first_text()
 309         return self.parent.get_next_text()
 310
 311     def get_child_audiobook(self):
 312         BookMedia = apps.get_model('catalogue', 'BookMedia')
 313         if not BookMedia.objects.filter(book__ancestor=self).exists():
 314             return None
 315         for child in self.children.all():
 316             if child.has_mp3_file():
 317                 return child
 318             child_sub = child.get_child_audiobook()
 319             if child_sub is not None:
 320                 return child_sub
 321
 322     def get_siblings(self):
 323         if not self.parent:
 324             return []
 325         return self.parent.children.all().order_by('parent_number')
 326
 327     def get_children(self):
 328         return self.children.all().order_by('parent_number')
 329
 330     @property
 331     def name(self):
 332         return self.title
 333
 334     def language_code(self):
 335         return constants.LANGUAGES_3TO2.get(self.language, self.language)
 336
 337     def language_name(self):
 338         return dict(settings.LANGUAGES).get(self.language_code(), "")
 339
 340     def is_foreign(self):
 341         return self.language_code() != settings.LANGUAGE_CODE
 342
 343     def set_audio_length(self):
 344         length = self.get_audio_length()
 345         if length > 0:
 346             self.audio_length = self.format_audio_length(length)
 347             self.save()
 348
 349     @staticmethod
 350     def format_audio_length(seconds):
 351         """
 352         >>> Book.format_audio_length(1)
 353         '0:01'
 354         >>> Book.format_audio_length(3661)
 355         '1:01:01'
 356         """
 357         if seconds < 60*60:
 358             minutes = seconds // 60
 359             seconds = seconds % 60
 360             return '%d:%02d' % (minutes, seconds)
 361         else:
 362             hours = seconds // 3600
 363             minutes = seconds % 3600 // 60
 364             seconds = seconds % 60
 365             return '%d:%02d:%02d' % (hours, minutes, seconds)
 366
 367     def get_audio_length(self):
 368         total = 0
 369         for media in self.get_mp3() or ():
 370             total += app_settings.GET_MP3_LENGTH(media.file.path)
 371         return int(total)
 372
 373     def has_media(self, type_):
 374         if type_ in Book.formats:
 375             return bool(getattr(self, "%s_file" % type_))
 376         else:
 377             return self.media.filter(type=type_).exists()
 378
 379     def has_audio(self):
 380         return self.has_media('mp3')
 381
 382     def get_media(self, type_):
 383         if self.has_media(type_):
 384             if type_ in Book.formats:
 385                 return getattr(self, "%s_file" % type_)
 386             else:
 387                 return self.media.filter(type=type_)
 388         else:
 389             return None
 390
 391     def get_mp3(self):
 392         return self.get_media("mp3")
 393
 394     def get_odt(self):
 395         return self.get_media("odt")
 396
 397     def get_ogg(self):
 398         return self.get_media("ogg")
 399
 400     def get_daisy(self):
 401         return self.get_media("daisy")
 402
 403     def media_url(self, format_):
 404         media = self.get_media(format_)
 405         if media:
 406             if self.preview:
 407                 return reverse('embargo_link', kwargs={'key': self.preview_key, 'slug': self.slug, 'format_': format_})
 408             else:
 409                 return media.url
 410         else:
 411             return None
 412
 413     def html_url(self):
 414         return self.media_url('html')
 415
 416     def pdf_url(self):
 417         return self.media_url('pdf')
 418
 419     def epub_url(self):
 420         return self.media_url('epub')
 421
 422     def mobi_url(self):
 423         return self.media_url('mobi')
 424
 425     def txt_url(self):
 426         return self.media_url('txt')
 427
 428     def fb2_url(self):
 429         return self.media_url('fb2')
 430
 431     def xml_url(self):
 432         return self.media_url('xml')
 433
 434     def has_description(self):
 435         return len(self.description) > 0
 436     has_description.short_description = _('description')
 437     has_description.boolean = True
 438
 439     def has_mp3_file(self):
 440         return self.has_media("mp3")
 441     has_mp3_file.short_description = 'MP3'
 442     has_mp3_file.boolean = True
 443
 444     def has_ogg_file(self):
 445         return self.has_media("ogg")
 446     has_ogg_file.short_description = 'OGG'
 447     has_ogg_file.boolean = True
 448
 449     def has_daisy_file(self):
 450         return self.has_media("daisy")
 451     has_daisy_file.short_description = 'DAISY'
 452     has_daisy_file.boolean = True
 453
 454     @property
 455     def media_daisy(self):
 456         return self.get_media('daisy')
 457
 458     def get_audiobooks(self):
 459         ogg_files = {}
 460         for m in self.media.filter(type='ogg').order_by().iterator():
 461             ogg_files[m.name] = m
 462
 463         audiobooks = []
 464         projects = set()
 465         total_duration = 0
 466         for mp3 in self.media.filter(type='mp3').iterator():
 467             # ogg files are always from the same project
 468             meta = mp3.get_extra_info_json()
 469             project = meta.get('project')
 470             if not project:
 471                 # temporary fallback
 472                 project = 'CzytamySłuchając'
 473
 474             projects.add((project, meta.get('funded_by', '')))
 475             total_duration += mp3.duration or 0
 476
 477             media = {'mp3': mp3}
 478
 479             ogg = ogg_files.get(mp3.name)
 480             if ogg:
 481                 media['ogg'] = ogg
 482             audiobooks.append(media)
 483
 484         projects = sorted(projects)
 485         total_duration = '%d:%02d' % (
 486             total_duration // 60,
 487             total_duration % 60
 488         )
 489         return audiobooks, projects, total_duration
 490
 491     def wldocument(self, parse_dublincore=True, inherit=True):
 492         from catalogue.import_utils import ORMDocProvider
 493         from librarian.parser import WLDocument
 494
 495         if inherit and self.parent:
 496             meta_fallbacks = self.parent.cover_info()
 497         else:
 498             meta_fallbacks = None
 499
 500         return WLDocument.from_file(
 501             self.xml_file.path,
 502             provider=ORMDocProvider(self),
 503             parse_dublincore=parse_dublincore,
 504             meta_fallbacks=meta_fallbacks)
 505
 506     def wldocument2(self):
 507         from catalogue.import_utils import ORMDocProvider
 508         from librarian.document import WLDocument
 509         doc = WLDocument(
 510             self.xml_file.path,
 511             provider=ORMDocProvider(self)
 512         )
 513         doc.meta.update(self.cover_info())
 514         return doc
 515
 516
 517     @staticmethod
 518     def zip_format(format_):
 519         def pretty_file_name(book):
 520             return "%s/%s.%s" % (
 521                 book.get_extra_info_json()['author'],
 522                 book.slug,
 523                 format_)
 524
 525         field_name = "%s_file" % format_
 526         books = Book.objects.filter(parent=None).exclude(**{field_name: ""}).exclude(preview=True).exclude(findable=False)
 527         paths = [(pretty_file_name(b), getattr(b, field_name).path) for b in books.iterator()]
 528         return create_zip(paths, app_settings.FORMAT_ZIPS[format_])
 529
 530     def zip_audiobooks(self, format_):
 531         bm = BookMedia.objects.filter(book=self, type=format_)
 532         paths = map(lambda bm: (bm.get_nice_filename(), bm.file.path), bm)
 533         licenses = set()
 534         for m in bm:
 535             license = constants.LICENSES.get(
 536                 m.get_extra_info_json().get('license'), {}
 537             ).get('locative')
 538             if license:
 539                 licenses.add(license)
 540         readme = render_to_string('catalogue/audiobook_zip_readme.txt', {
 541             'licenses': licenses,
 542         })
 543         return create_zip(paths, "%s_%s" % (self.slug, format_), {'informacje.txt': readme})
 544
 545     def search_index(self, book_info=None, index=None, index_tags=True, commit=True):
 546         if not self.findable:
 547             return
 548         if index is None:
 549             from search.index import Index
 550             index = Index()
 551         try:
 552             index.index_book(self, book_info)
 553             if index_tags:
 554                 index.index_tags()
 555             if commit:
 556                 index.index.commit()
 557         except Exception as e:
 558             index.index.rollback()
 559             raise e
 560
 561     # will make problems in conjunction with paid previews
 562     def download_pictures(self, remote_gallery_url):
 563         gallery_path = self.gallery_path()
 564         # delete previous files, so we don't include old files in ebooks
 565         if os.path.isdir(gallery_path):
 566             for filename in os.listdir(gallery_path):
 567                 file_path = os.path.join(gallery_path, filename)
 568                 os.unlink(file_path)
 569         ilustr_elements = list(self.wldocument().edoc.findall('//ilustr'))
 570         if ilustr_elements:
 571             makedirs(gallery_path)
 572             for ilustr in ilustr_elements:
 573                 ilustr_src = ilustr.get('src')
 574                 ilustr_path = os.path.join(gallery_path, ilustr_src)
 575                 urlretrieve('%s/%s' % (remote_gallery_url, ilustr_src), ilustr_path)
 576
 577     def load_abstract(self):
 578         abstract = self.wldocument(parse_dublincore=False).edoc.getroot().find('.//abstrakt')
 579         if abstract is not None:
 580             self.abstract = transform_abstrakt(abstract)
 581         else:
 582             self.abstract = ''
 583
 584     def load_toc(self):
 585         self.toc = ''
 586         if self.html_file:
 587             parser = html.HTMLParser(encoding='utf-8')
 588             tree = html.parse(self.html_file.path, parser=parser)
 589             toc = tree.find('//div[@id="toc"]/ol')
 590             if toc is None or not len(toc):
 591                 return
 592             html_link = reverse('book_text', args=[self.slug])
 593             for a in toc.findall('.//a'):
 594                 a.attrib['href'] = html_link + a.attrib['href']
 595             self.toc = html.tostring(toc, encoding='unicode')
 596             # div#toc
 597
 598     @classmethod
 599     def from_xml_file(cls, xml_file, **kwargs):
 600         from django.core.files import File
 601         from librarian import dcparser
 602
 603         # use librarian to parse meta-data
 604         book_info = dcparser.parse(xml_file)
 605
 606         if not isinstance(xml_file, File):
 607             xml_file = File(open(xml_file))
 608
 609         try:
 610             return cls.from_text_and_meta(xml_file, book_info, **kwargs)
 611         finally:
 612             xml_file.close()
 613
 614     @classmethod
 615     def from_text_and_meta(cls, raw_file, book_info, overwrite=False, dont_build=None, search_index=True,
 616                            search_index_tags=True, remote_gallery_url=None, days=0, findable=True):
 617         if dont_build is None:
 618             dont_build = set()
 619         dont_build = set.union(set(dont_build), set(app_settings.DONT_BUILD))
 620
 621         # check for parts before we do anything
 622         children = []
 623         if hasattr(book_info, 'parts'):
 624             for part_url in book_info.parts:
 625                 try:
 626                     children.append(Book.objects.get(slug=part_url.slug))
 627                 except Book.DoesNotExist:
 628                     raise Book.DoesNotExist(_('Book "%s" does not exist.') % part_url.slug)
 629
 630         # Read book metadata
 631         book_slug = book_info.url.slug
 632         if re.search(r'[^a-z0-9-]', book_slug):
 633             raise ValueError('Invalid characters in slug')
 634         book, created = Book.objects.get_or_create(slug=book_slug)
 635
 636         if created:
 637             book_shelves = []
 638             old_cover = None
 639             book.preview = bool(days)
 640             if book.preview:
 641                 book.preview_until = date.today() + timedelta(days)
 642         else:
 643             if not overwrite:
 644                 raise Book.AlreadyExists(_('Book %s already exists') % book_slug)
 645             # Save shelves for this book
 646             book_shelves = list(book.tags.filter(category='set'))
 647             old_cover = book.cover_info()
 648
 649         # Save XML file
 650         book.xml_file.save('%s.xml' % book.slug, raw_file, save=False)
 651         if book.preview:
 652             book.xml_file.set_readable(False)
 653
 654         book.findable = findable
 655         book.language = book_info.language
 656         book.title = book_info.title
 657         if book_info.variant_of:
 658             book.common_slug = book_info.variant_of.slug
 659         else:
 660             book.common_slug = book.slug
 661         book.extra_info = json.dumps(book_info.to_dict())
 662         book.load_abstract()
 663         book.load_toc()
 664         book.save()
 665
 666         meta_tags = Tag.tags_from_info(book_info)
 667
 668         for tag in meta_tags:
 669             if not tag.for_books:
 670                 tag.for_books = True
 671                 tag.save()
 672
 673         book.tags = set(meta_tags + book_shelves)
 674         book.save()  # update sort_key_author
 675
 676         cover_changed = old_cover != book.cover_info()
 677         obsolete_children = set(b for b in book.children.all()
 678                                 if b not in children)
 679         notify_cover_changed = []
 680         for n, child_book in enumerate(children):
 681             new_child = child_book.parent != book
 682             child_book.parent = book
 683             child_book.parent_number = n
 684             child_book.save()
 685             if new_child or cover_changed:
 686                 notify_cover_changed.append(child_book)
 687         # Disown unfaithful children and let them cope on their own.
 688         for child in obsolete_children:
 689             child.parent = None
 690             child.parent_number = 0
 691             child.save()
 692             if old_cover:
 693                 notify_cover_changed.append(child)
 694
 695         cls.repopulate_ancestors()
 696         tasks.update_counters.delay()
 697
 698         if remote_gallery_url:
 699             book.download_pictures(remote_gallery_url)
 700
 701         # No saves beyond this point.
 702
 703         # Build cover.
 704         if 'cover' not in dont_build:
 705             book.cover.build_delay()
 706             book.cover_clean.build_delay()
 707             book.cover_thumb.build_delay()
 708             book.cover_api_thumb.build_delay()
 709             book.simple_cover.build_delay()
 710             book.cover_ebookpoint.build_delay()
 711
 712         # Build HTML and ebooks.
 713         book.html_file.build_delay()
 714         if not children:
 715             for format_ in constants.EBOOK_FORMATS_WITHOUT_CHILDREN:
 716                 if format_ not in dont_build:
 717                     getattr(book, '%s_file' % format_).build_delay()
 718         for format_ in constants.EBOOK_FORMATS_WITH_CHILDREN:
 719             if format_ not in dont_build:
 720                 getattr(book, '%s_file' % format_).build_delay()
 721
 722         if not settings.NO_SEARCH_INDEX and search_index and findable:
 723             tasks.index_book.delay(book.id, book_info=book_info, index_tags=search_index_tags)
 724
 725         for child in notify_cover_changed:
 726             child.parent_cover_changed()
 727
 728         book.update_popularity()
 729         tasks.update_references.delay(book.id)
 730
 731         cls.published.send(sender=cls, instance=book)
 732         return book
 733
 734     def get_master(self):
 735         master_tags = [
 736             'opowiadanie',
 737             'powiesc',
 738             'dramat_wierszowany_l',
 739             'dramat_wierszowany_lp',
 740             'dramat_wspolczesny', 'liryka_l', 'liryka_lp',
 741             'wywiad',
 742         ]
 743         from librarian.parser import WLDocument
 744         wld = WLDocument.from_file(self.xml_file.path, parse_dublincore=False)
 745         root = wld.edoc.getroot()
 746         for master in root.iter():
 747             if master.tag in master_tags:
 748                 return master
 749
 750     def update_references(self):
 751         from references.models import Entity, Reference
 752         master = self.get_master()
 753         if master is None:
 754             master = []
 755         found = set()
 756         for i, sec in enumerate(master):
 757             for ref in sec.findall('.//ref'):
 758                 href = ref.attrib.get('href', '')
 759                 if not href or href in found:
 760                     continue
 761                 found.add(href)
 762                 entity, created = Entity.objects.get_or_create(
 763                     uri=href
 764                 )
 765                 ref, created = Reference.objects.get_or_create(
 766                     book=self,
 767                     entity=entity
 768                 )
 769                 ref.first_section = 'sec%d' % (i + 1)
 770                 entity.populate()
 771                 entity.save()
 772         Reference.objects.filter(book=self).exclude(entity__uri__in=found).delete()
 773
 774     @property
 775     def references(self):
 776         return self.reference_set.all().select_related('entity')
 777
 778     @classmethod
 779     @transaction.atomic
 780     def repopulate_ancestors(cls):
 781         """Fixes the ancestry cache."""
 782         # TODO: table names
 783         cursor = connection.cursor()
 784         if connection.vendor == 'postgres':
 785             cursor.execute("TRUNCATE catalogue_book_ancestor")
 786             cursor.execute("""
 787                 WITH RECURSIVE ancestry AS (
 788                     SELECT book.id, book.parent_id
 789                     FROM catalogue_book AS book
 790                     WHERE book.parent_id IS NOT NULL
 791                     UNION
 792                     SELECT ancestor.id, book.parent_id
 793                     FROM ancestry AS ancestor, catalogue_book AS book
 794                     WHERE ancestor.parent_id = book.id
 795                         AND book.parent_id IS NOT NULL
 796                     )
 797                 INSERT INTO catalogue_book_ancestor
 798                     (from_book_id, to_book_id)
 799                     SELECT id, parent_id
 800                     FROM ancestry
 801                     ORDER BY id;
 802                 """)
 803         else:
 804             cursor.execute("DELETE FROM catalogue_book_ancestor")
 805             for b in cls.objects.exclude(parent=None):
 806                 parent = b.parent
 807                 while parent is not None:
 808                     b.ancestor.add(parent)
 809                     parent = parent.parent
 810
 811     @property
 812     def ancestors(self):
 813         if self.parent:
 814             for anc in self.parent.ancestors:
 815                 yield anc
 816             yield self.parent
 817         else:
 818             return []
 819
 820     def clear_cache(self):
 821         clear_cached_renders(self.mini_box)
 822         clear_cached_renders(self.mini_box_nolink)
 823
 824     def cover_info(self, inherit=True):
 825         """Returns a dictionary to serve as fallback for BookInfo.
 826
 827         For now, the only thing inherited is the cover image.
 828         """
 829         need = False
 830         info = {}
 831         for field in ('cover_url', 'cover_by', 'cover_source'):
 832             val = self.get_extra_info_json().get(field)
 833             if val:
 834                 info[field] = val
 835             else:
 836                 need = True
 837         if inherit and need and self.parent is not None:
 838             parent_info = self.parent.cover_info()
 839             parent_info.update(info)
 840             info = parent_info
 841         return info
 842
 843     def related_themes(self):
 844         return Tag.objects.usage_for_queryset(
 845             Fragment.objects.filter(models.Q(book=self) | models.Q(book__ancestor=self)),
 846             counts=True).filter(category='theme').order_by('-count')
 847
 848     def parent_cover_changed(self):
 849         """Called when parent book's cover image is changed."""
 850         if not self.cover_info(inherit=False):
 851             if 'cover' not in app_settings.DONT_BUILD:
 852                 self.cover.build_delay()
 853                 self.cover_clean.build_delay()
 854                 self.cover_thumb.build_delay()
 855                 self.cover_api_thumb.build_delay()
 856                 self.simple_cover.build_delay()
 857                 self.cover_ebookpoint.build_delay()
 858             for format_ in constants.EBOOK_FORMATS_WITH_COVERS:
 859                 if format_ not in app_settings.DONT_BUILD:
 860                     getattr(self, '%s_file' % format_).build_delay()
 861             for child in self.children.all():
 862                 child.parent_cover_changed()
 863
 864     def other_versions(self):
 865         """Find other versions (i.e. in other languages) of the book."""
 866         return type(self).objects.filter(common_slug=self.common_slug, findable=True).exclude(pk=self.pk)
 867
 868     def parents(self):
 869         books = []
 870         parent = self.parent
 871         while parent is not None:
 872             books.insert(0, parent)
 873             parent = parent.parent
 874         return books
 875
 876     def pretty_title(self, html_links=False):
 877         names = [(tag.name, tag.get_absolute_url()) for tag in self.authors().only('name', 'category', 'slug')]
 878         books = self.parents() + [self]
 879         names.extend([(b.title, b.get_absolute_url()) for b in books])
 880
 881         if html_links:
 882             names = ['<a href="%s">%s</a>' % (tag[1], tag[0]) for tag in names]
 883         else:
 884             names = [tag[0] for tag in names]
 885         return ', '.join(names)
 886
 887     def publisher(self):
 888         publisher = self.get_extra_info_json()['publisher']
 889         if isinstance(publisher, str):
 890             return publisher
 891         elif isinstance(publisher, list):
 892             return ', '.join(publisher)
 893
 894     @classmethod
 895     def tagged_top_level(cls, tags):
 896         """ Returns top-level books tagged with `tags`.
 897
 898         It only returns those books which don't have ancestors which are
 899         also tagged with those tags.
 900
 901         """
 902         objects = cls.tagged.with_all(tags)
 903         return objects.filter(findable=True).exclude(ancestor__in=objects)
 904
 905     @classmethod
 906     def book_list(cls, book_filter=None):
 907         """Generates a hierarchical listing of all books.
 908
 909         Books are optionally filtered with a test function.
 910
 911         """
 912
 913         books_by_parent = {}
 914         books = cls.objects.filter(findable=True).order_by('parent_number', 'sort_key').only('title', 'parent', 'slug', 'extra_info')
 915         if book_filter:
 916             books = books.filter(book_filter).distinct()
 917
 918             book_ids = set(b['pk'] for b in books.values("pk").iterator())
 919             for book in books.iterator():
 920                 parent = book.parent_id
 921                 if parent not in book_ids:
 922                     parent = None
 923                 books_by_parent.setdefault(parent, []).append(book)
 924         else:
 925             for book in books.iterator():
 926                 books_by_parent.setdefault(book.parent_id, []).append(book)
 927
 928         orphans = []
 929         books_by_author = OrderedDict()
 930         for tag in Tag.objects.filter(category='author').iterator():
 931             books_by_author[tag] = []
 932
 933         for book in books_by_parent.get(None, ()):
 934             authors = list(book.authors().only('pk'))
 935             if authors:
 936                 for author in authors:
 937                     books_by_author[author].append(book)
 938             else:
 939                 orphans.append(book)
 940
 941         return books_by_author, orphans, books_by_parent
 942
 943     _audiences_pl = {
 944         "SP": (1, "szkoła podstawowa"),
 945         "SP1": (1, "szkoła podstawowa"),
 946         "SP2": (1, "szkoła podstawowa"),
 947         "SP3": (1, "szkoła podstawowa"),
 948         "P": (1, "szkoła podstawowa"),
 949         "G": (2, "gimnazjum"),
 950         "L": (3, "liceum"),
 951         "LP": (3, "liceum"),
 952     }
 953
 954     def audiences_pl(self):
 955         audiences = self.get_extra_info_json().get('audiences', [])
 956         audiences = sorted(set([self._audiences_pl.get(a, (99, a)) for a in audiences]))
 957         return [a[1] for a in audiences]
 958
 959     def stage_note(self):
 960         stage = self.get_extra_info_json().get('stage')
 961         if stage and stage < '0.4':
 962             return (_('This work needs modernisation'),
 963                     reverse('infopage', args=['wymagajace-uwspolczesnienia']))
 964         else:
 965             return None, None
 966
 967     def choose_fragments(self, number):
 968         fragments = self.fragments.order_by()
 969         fragments_count = fragments.count()
 970         if not fragments_count and self.children.exists():
 971             fragments = Fragment.objects.filter(book__ancestor=self).order_by()
 972             fragments_count = fragments.count()
 973         if fragments_count:
 974             if fragments_count > number:
 975                 offset = randint(0, fragments_count - number)
 976             else:
 977                 offset = 0
 978             return fragments[offset : offset + number]
 979         elif self.parent:
 980             return self.parent.choose_fragments(number)
 981         else:
 982             return []
 983
 984     def choose_fragment(self):
 985         fragments = self.choose_fragments(1)
 986         if fragments:
 987             return fragments[0]
 988         else:
 989             return None
 990
 991     def fragment_data(self):
 992         fragment = self.choose_fragment()
 993         if fragment:
 994             return {
 995                 'title': fragment.book.pretty_title(),
 996                 'html': re.sub('</?blockquote[^>]*>', '', fragment.get_short_text()),
 997             }
 998         else:
 999             return None
1000
1001     def update_popularity(self):
1002         count = self.tags.filter(category='set').values('user').order_by('user').distinct().count()
1003         try:
1004             pop = self.popularity
1005             pop.count = count
1006             pop.save()
1007         except BookPopularity.DoesNotExist:
1008             BookPopularity.objects.create(book=self, count=count)
1009
1010     def ridero_link(self):
1011         return 'https://ridero.eu/%s/books/wl_%s/' % (get_language(), self.slug.replace('-', '_'))
1012
1013     def like(self, user):
1014         from social.utils import likes, get_set, set_sets
1015         if not likes(user, self):
1016             tag = get_set(user, '')
1017             set_sets(user, self, [tag])
1018
1019     def unlike(self, user):
1020         from social.utils import likes, set_sets
1021         if likes(user, self):
1022             set_sets(user, self, [])
1023
1024     def full_sort_key(self):
1025         return self.SORT_KEY_SEP.join((self.sort_key_author, self.sort_key, str(self.id)))
1026
1027     def cover_color(self):
1028         return WLCover.epoch_colors.get(self.get_extra_info_json().get('epoch'), '#000000')
1029
1030     @cached_render('catalogue/book_mini_box.html')
1031     def mini_box(self):
1032         return {
1033             'book': self
1034         }
1035
1036     @cached_render('catalogue/book_mini_box.html')
1037     def mini_box_nolink(self):
1038         return {
1039             'book': self,
1040             'no_link': True,
1041         }
1042
1043 def add_file_fields():
1044     for format_ in Book.formats:
1045         field_name = "%s_file" % format_
1046         # This weird globals() assignment makes Django migrations comfortable.
1047         _upload_to = _ebook_upload_to('book/%s/%%s.%s' % (format_, format_))
1048         _upload_to.__name__ = '_%s_upload_to' % format_
1049         globals()[_upload_to.__name__] = _upload_to
1050
1051         EbookField(
1052             format_, _("%s file" % format_.upper()),
1053             upload_to=_upload_to,
1054             storage=bofh_storage,
1055             max_length=255,
1056             blank=True,
1057             default=''
1058         ).contribute_to_class(Book, field_name)
1059         if format_ != 'xml':
1060             models.CharField(max_length=255, editable=False, default='', db_index=True).contribute_to_class(Book, f'{field_name}_etag')
1061
1062
1063 add_file_fields()
1064
1065
1066 class BookPopularity(models.Model):
1067     book = models.OneToOneField(Book, models.CASCADE, related_name='popularity')
1068     count = models.IntegerField(default=0, db_index=True)