src/catalogue/models/book.py

   1 # This file is part of Wolne Lektury, licensed under GNU Affero GPLv3 or later.
   2 # Copyright © Fundacja Wolne Lektury. See NOTICE for more information.
   3 #
   4 from collections import OrderedDict
   5 import json
   6 from datetime import date, timedelta
   7 from random import randint
   8 import os.path
   9 import re
  10 from urllib.request import urlretrieve
  11 from django.apps import apps
  12 from django.conf import settings
  13 from django.db import connection, models, transaction
  14 import django.dispatch
  15 from django.contrib.contenttypes.fields import GenericRelation
  16 from django.template.loader import render_to_string
  17 from django.urls import reverse
  18 from django.utils.translation import gettext_lazy as _, get_language
  19 from fnpdjango.storage import BofhFileSystemStorage
  20 from lxml import html
  21 from librarian.cover import WLCover
  22 from librarian.builders.html import AbstraktHtmlBuilder
  23 from librarian.builders import builders
  24 from newtagging import managers
  25 from catalogue import constants
  26 from catalogue import fields
  27 from catalogue.models import Tag, Fragment, BookMedia
  28 from catalogue.utils import create_zip, gallery_url, gallery_path, split_tags, get_random_hash
  29 from catalogue.models.tag import prefetched_relations
  30 from catalogue import app_settings
  31 from wolnelektury.utils import makedirs, cached_render, clear_cached_renders
  32
  33 bofh_storage = BofhFileSystemStorage()
  34
  35
  36 class Book(models.Model):
  37     """Represents a book imported from WL-XML."""
  38     title = models.CharField('tytuł', max_length=32767)
  39     sort_key = models.CharField('klucz sortowania', max_length=120, db_index=True, editable=False)
  40     sort_key_author = models.CharField(
  41         'klucz sortowania wg autora', max_length=120, db_index=True, editable=False, default='')
  42     slug = models.SlugField('slug', max_length=120, db_index=True, unique=True)
  43     common_slug = models.SlugField('wspólny slug', max_length=120, db_index=True)
  44     language = models.CharField('kod języka', max_length=3, db_index=True, default=app_settings.DEFAULT_LANGUAGE)
  45     description = models.TextField('opis', blank=True)
  46     abstract = models.TextField('abstrakt', blank=True)
  47     toc = models.TextField('spis treści', blank=True)
  48     created_at = models.DateTimeField('data utworzenia', auto_now_add=True, db_index=True)
  49     changed_at = models.DateTimeField('data motyfikacji', auto_now=True, db_index=True)
  50     parent_number = models.IntegerField('numer w ramach rodzica', default=0)
  51     extra_info = models.TextField('dodatkowe informacje', default='{}')
  52     gazeta_link = models.CharField(blank=True, max_length=240)
  53     wiki_link = models.CharField(blank=True, max_length=240)
  54     print_on_demand = models.BooleanField('druk na żądanie', default=False)
  55     recommended = models.BooleanField('polecane', default=False)
  56     audio_length = models.CharField('długość audio', blank=True, max_length=8)
  57     preview = models.BooleanField('prapremiera', default=False)
  58     preview_until = models.DateField('prapremiera do', blank=True, null=True)
  59     preview_key = models.CharField(max_length=32, blank=True, null=True)
  60     findable = models.BooleanField('wyszukiwalna', default=True, db_index=True)
  61
  62     # files generated during publication
  63     xml_file = fields.XmlField(storage=bofh_storage, with_etag=False)
  64     html_file = fields.HtmlField(storage=bofh_storage)
  65     fb2_file = fields.Fb2Field(storage=bofh_storage)
  66     txt_file = fields.TxtField(storage=bofh_storage)
  67     epub_file = fields.EpubField(storage=bofh_storage)
  68     mobi_file = fields.MobiField(storage=bofh_storage)
  69     pdf_file = fields.PdfField(storage=bofh_storage)
  70
  71     cover = fields.CoverField('okładka', storage=bofh_storage)
  72     # Cleaner version of cover for thumbs
  73     cover_clean = fields.CoverCleanField('czysta okładka')
  74     cover_thumb = fields.CoverThumbField('miniatura okładki')
  75     cover_api_thumb = fields.CoverApiThumbField(
  76         'mniaturka okładki dla aplikacji')
  77     simple_cover = fields.SimpleCoverField('okładka dla aplikacji')
  78     cover_ebookpoint = fields.CoverEbookpointField(
  79         'okładka dla Ebookpoint')
  80
  81     ebook_formats = constants.EBOOK_FORMATS
  82     formats = ebook_formats + ['html', 'xml']
  83
  84     parent = models.ForeignKey('self', models.CASCADE, blank=True, null=True, related_name='children')
  85     ancestor = models.ManyToManyField('self', blank=True, editable=False, related_name='descendant', symmetrical=False)
  86
  87     cached_author = models.CharField(blank=True, max_length=240, db_index=True)
  88     has_audience = models.BooleanField(default=False)
  89
  90     objects = models.Manager()
  91     tagged = managers.ModelTaggedItemManager(Tag)
  92     tags = managers.TagDescriptor(Tag)
  93     tag_relations = GenericRelation(Tag.intermediary_table_model)
  94     translators = models.ManyToManyField(Tag, blank=True)
  95
  96     html_built = django.dispatch.Signal()
  97     published = django.dispatch.Signal()
  98
  99     SORT_KEY_SEP = '$'
 100
 101     is_book = True
 102
 103     class AlreadyExists(Exception):
 104         pass
 105
 106     class Meta:
 107         ordering = ('sort_key_author', 'sort_key')
 108         verbose_name = 'książka'
 109         verbose_name_plural = 'książki'
 110         app_label = 'catalogue'
 111
 112     def __str__(self):
 113         return self.title
 114
 115     def get_extra_info_json(self):
 116         return json.loads(self.extra_info or '{}')
 117
 118     def get_initial(self):
 119         try:
 120             return re.search(r'\w', self.title, re.U).group(0)
 121         except AttributeError:
 122             return ''
 123
 124     def authors(self):
 125         return self.tags.filter(category='author')
 126
 127     def epochs(self):
 128         return self.tags.filter(category='epoch')
 129
 130     def genres(self):
 131         return self.tags.filter(category='genre')
 132
 133     def kinds(self):
 134         return self.tags.filter(category='kind')
 135
 136     def tag_unicode(self, category):
 137         relations = prefetched_relations(self, category)
 138         if relations:
 139             return ', '.join(rel.tag.name for rel in relations)
 140         else:
 141             return ', '.join(self.tags.filter(category=category).values_list('name', flat=True))
 142
 143     def tags_by_category(self):
 144         return split_tags(self.tags.exclude(category__in=('set', 'theme')))
 145
 146     def author_unicode(self):
 147         return self.cached_author
 148
 149     def kind_unicode(self):
 150         return self.tag_unicode('kind')
 151
 152     def epoch_unicode(self):
 153         return self.tag_unicode('epoch')
 154
 155     def genre_unicode(self):
 156         return self.tag_unicode('genre')
 157
 158     def translator(self):
 159         translators = self.get_extra_info_json().get('translators')
 160         if not translators:
 161             return None
 162         if len(translators) > 3:
 163             translators = translators[:2]
 164             others = ' i inni'
 165         else:
 166             others = ''
 167         return ', '.join('\xa0'.join(reversed(translator.split(', ', 1))) for translator in translators) + others
 168
 169     def cover_source(self):
 170         return self.get_extra_info_json().get('cover_source', self.parent.cover_source() if self.parent else '')
 171
 172     @property
 173     def isbn_pdf(self):
 174         return self.get_extra_info_json().get('isbn_pdf')
 175
 176     @property
 177     def isbn_epub(self):
 178         return self.get_extra_info_json().get('isbn_epub')
 179
 180     @property
 181     def isbn_mobi(self):
 182         return self.get_extra_info_json().get('isbn_mobi')
 183
 184     def is_accessible_to(self, user):
 185         if not self.preview:
 186             return True
 187         if not user.is_authenticated:
 188             return False
 189         Membership = apps.get_model('club', 'Membership')
 190         if Membership.is_active_for(user):
 191             return True
 192         Funding = apps.get_model('funding', 'Funding')
 193         if Funding.objects.filter(user=user, offer__book=self):
 194             return True
 195         return False
 196
 197     def save(self, force_insert=False, force_update=False, **kwargs):
 198         from sortify import sortify
 199
 200         self.sort_key = sortify(self.title)[:120]
 201         self.title = str(self.title)  # ???
 202
 203         try:
 204             author = self.authors().first().sort_key
 205         except AttributeError:
 206             author = ''
 207         self.sort_key_author = author
 208
 209         self.cached_author = self.tag_unicode('author')
 210         self.has_audience = 'audience' in self.get_extra_info_json()
 211
 212         if self.preview and not self.preview_key:
 213             self.preview_key = get_random_hash(self.slug)[:32]
 214
 215         ret = super(Book, self).save(force_insert, force_update, **kwargs)
 216
 217         return ret
 218
 219     def get_absolute_url(self):
 220         return reverse('book_detail', args=[self.slug])
 221
 222     def gallery_path(self):
 223         return gallery_path(self.slug)
 224
 225     def gallery_url(self):
 226         return gallery_url(self.slug)
 227
 228     def get_first_text(self):
 229         if self.html_file:
 230             return self
 231         child = self.children.all().order_by('parent_number').first()
 232         if child is not None:
 233             return child.get_first_text()
 234
 235     def get_last_text(self):
 236         if self.html_file:
 237             return self
 238         child = self.children.all().order_by('parent_number').last()
 239         if child is not None:
 240             return child.get_last_text()
 241
 242     def get_prev_text(self):
 243         if not self.parent:
 244             return None
 245         sibling = self.parent.children.filter(parent_number__lt=self.parent_number).order_by('-parent_number').first()
 246         if sibling is not None:
 247             return sibling.get_last_text()
 248
 249         if self.parent.html_file:
 250             return self.parent
 251
 252         return self.parent.get_prev_text()
 253
 254     def get_next_text(self, inside=True):
 255         if inside:
 256             child = self.children.order_by('parent_number').first()
 257             if child is not None:
 258                 return child.get_first_text()
 259
 260         if not self.parent:
 261             return None
 262         sibling = self.parent.children.filter(parent_number__gt=self.parent_number).order_by('parent_number').first()
 263         if sibling is not None:
 264             return sibling.get_first_text()
 265         return self.parent.get_next_text(inside=False)
 266
 267     def get_child_audiobook(self):
 268         BookMedia = apps.get_model('catalogue', 'BookMedia')
 269         if not BookMedia.objects.filter(book__ancestor=self).exists():
 270             return None
 271         for child in self.children.order_by('parent_number').all():
 272             if child.has_mp3_file():
 273                 return child
 274             child_sub = child.get_child_audiobook()
 275             if child_sub is not None:
 276                 return child_sub
 277
 278     def get_siblings(self):
 279         if not self.parent:
 280             return []
 281         return self.parent.children.all().order_by('parent_number')
 282
 283     def get_children(self):
 284         return self.children.all().order_by('parent_number')
 285
 286     @property
 287     def name(self):
 288         return self.title
 289
 290     def language_code(self):
 291         return constants.LANGUAGES_3TO2.get(self.language, self.language)
 292
 293     def language_name(self):
 294         return dict(settings.LANGUAGES).get(self.language_code(), "")
 295
 296     def is_foreign(self):
 297         return self.language_code() != settings.LANGUAGE_CODE
 298
 299     def set_audio_length(self):
 300         length = self.get_audio_length()
 301         if length > 0:
 302             self.audio_length = self.format_audio_length(length)
 303             self.save()
 304
 305     @staticmethod
 306     def format_audio_length(seconds):
 307         """
 308         >>> Book.format_audio_length(1)
 309         '0:01'
 310         >>> Book.format_audio_length(3661)
 311         '1:01:01'
 312         """
 313         if seconds < 60*60:
 314             minutes = seconds // 60
 315             seconds = seconds % 60
 316             return '%d:%02d' % (minutes, seconds)
 317         else:
 318             hours = seconds // 3600
 319             minutes = seconds % 3600 // 60
 320             seconds = seconds % 60
 321             return '%d:%02d:%02d' % (hours, minutes, seconds)
 322
 323     def get_audio_length(self):
 324         total = 0
 325         for media in self.get_mp3() or ():
 326             total += app_settings.GET_MP3_LENGTH(media.file.path)
 327         return int(total)
 328
 329     def get_time(self):
 330         try:
 331             return round(self.xml_file.size / 1000 * 40)
 332         except ValueError:
 333             return 0
 334
 335     def has_media(self, type_):
 336         if type_ in Book.formats:
 337             return bool(getattr(self, "%s_file" % type_))
 338         else:
 339             return self.media.filter(type=type_).exists()
 340
 341     def has_audio(self):
 342         return self.has_media('mp3')
 343
 344     def get_media(self, type_):
 345         if self.has_media(type_):
 346             if type_ in Book.formats:
 347                 return getattr(self, "%s_file" % type_)
 348             else:
 349                 return self.media.filter(type=type_)
 350         else:
 351             return None
 352
 353     def get_mp3(self):
 354         return self.get_media("mp3")
 355
 356     def get_odt(self):
 357         return self.get_media("odt")
 358
 359     def get_ogg(self):
 360         return self.get_media("ogg")
 361
 362     def get_daisy(self):
 363         return self.get_media("daisy")
 364
 365     def get_audio_epub(self):
 366         return self.get_media("audio.epub")
 367
 368     def media_url(self, format_):
 369         media = self.get_media(format_)
 370         if media:
 371             if self.preview:
 372                 return reverse('embargo_link', kwargs={'key': self.preview_key, 'slug': self.slug, 'format_': format_})
 373             else:
 374                 return media.url
 375         else:
 376             return None
 377
 378     def html_url(self):
 379         return self.media_url('html')
 380
 381     def pdf_url(self):
 382         return self.media_url('pdf')
 383
 384     def epub_url(self):
 385         return self.media_url('epub')
 386
 387     def mobi_url(self):
 388         return self.media_url('mobi')
 389
 390     def txt_url(self):
 391         return self.media_url('txt')
 392
 393     def fb2_url(self):
 394         return self.media_url('fb2')
 395
 396     def xml_url(self):
 397         return self.media_url('xml')
 398
 399     def has_description(self):
 400         return len(self.description) > 0
 401     has_description.short_description = 'opis'
 402     has_description.boolean = True
 403
 404     def has_mp3_file(self):
 405         return self.has_media("mp3")
 406     has_mp3_file.short_description = 'MP3'
 407     has_mp3_file.boolean = True
 408
 409     def has_ogg_file(self):
 410         return self.has_media("ogg")
 411     has_ogg_file.short_description = 'OGG'
 412     has_ogg_file.boolean = True
 413
 414     def has_daisy_file(self):
 415         return self.has_media("daisy")
 416     has_daisy_file.short_description = 'DAISY'
 417     has_daisy_file.boolean = True
 418
 419     def has_sync_file(self):
 420         return settings.FEATURE_SYNCHRO and self.has_media("sync")
 421
 422     def get_sync(self):
 423         with self.get_media('sync').first().file.open('r') as f:
 424             sync = f.read().split('\n')
 425         offset = float(sync[0])
 426         items = []
 427         for line in sync[1:]:
 428             if not line:
 429                 continue
 430             start, end, elid = line.split()
 431             items.append([elid, float(start) + offset])
 432         return json.dumps(items)
 433
 434     def has_audio_epub_file(self):
 435         return self.has_media("audio.epub")
 436
 437     @property
 438     def media_daisy(self):
 439         return self.get_media('daisy')
 440
 441     @property
 442     def media_audio_epub(self):
 443         return self.get_media('audio.epub')
 444
 445     def get_audiobooks(self):
 446         ogg_files = {}
 447         for m in self.media.filter(type='ogg').order_by().iterator():
 448             ogg_files[m.name] = m
 449
 450         audiobooks = []
 451         projects = set()
 452         total_duration = 0
 453         for mp3 in self.media.filter(type='mp3').iterator():
 454             # ogg files are always from the same project
 455             meta = mp3.get_extra_info_json()
 456             project = meta.get('project')
 457             if not project:
 458                 # temporary fallback
 459                 project = 'CzytamySłuchając'
 460
 461             projects.add((project, meta.get('funded_by', '')))
 462             total_duration += mp3.duration or 0
 463
 464             media = {'mp3': mp3}
 465
 466             ogg = ogg_files.get(mp3.name)
 467             if ogg:
 468                 media['ogg'] = ogg
 469             audiobooks.append(media)
 470
 471         projects = sorted(projects)
 472         total_duration = '%d:%02d' % (
 473             total_duration // 60,
 474             total_duration % 60
 475         )
 476         return audiobooks, projects, total_duration
 477
 478     def wldocument(self, parse_dublincore=True, inherit=True):
 479         from catalogue.import_utils import ORMDocProvider
 480         from librarian.parser import WLDocument
 481
 482         if inherit and self.parent:
 483             meta_fallbacks = self.parent.cover_info()
 484         else:
 485             meta_fallbacks = None
 486
 487         return WLDocument.from_file(
 488             self.xml_file.path,
 489             provider=ORMDocProvider(self),
 490             parse_dublincore=parse_dublincore,
 491             meta_fallbacks=meta_fallbacks)
 492
 493     def wldocument2(self):
 494         from catalogue.import_utils import ORMDocProvider
 495         from librarian.document import WLDocument
 496         doc = WLDocument(
 497             self.xml_file.path,
 498             provider=ORMDocProvider(self)
 499         )
 500         doc.meta.update(self.cover_info())
 501         return doc
 502
 503
 504     @staticmethod
 505     def zip_format(format_):
 506         def pretty_file_name(book):
 507             return "%s/%s.%s" % (
 508                 book.get_extra_info_json()['author'],
 509                 book.slug,
 510                 format_)
 511
 512         field_name = "%s_file" % format_
 513         field = getattr(Book, field_name)
 514         books = Book.objects.filter(parent=None).exclude(**{field_name: ""}).exclude(preview=True).exclude(findable=False)
 515         paths = [(pretty_file_name(b), getattr(b, field_name).path) for b in books.iterator()]
 516         return create_zip(paths, field.ZIP)
 517
 518     def zip_audiobooks(self, format_):
 519         bm = BookMedia.objects.filter(book=self, type=format_)
 520         paths = map(lambda bm: (bm.get_nice_filename(), bm.file.path), bm)
 521         licenses = set()
 522         for m in bm:
 523             license = constants.LICENSES.get(
 524                 m.get_extra_info_json().get('license'), {}
 525             ).get('locative')
 526             if license:
 527                 licenses.add(license)
 528         readme = render_to_string('catalogue/audiobook_zip_readme.txt', {
 529             'licenses': licenses,
 530             'meta': self.wldocument2().meta,
 531         })
 532         return create_zip(paths, "%s_%s" % (self.slug, format_), {'informacje.txt': readme})
 533
 534     def search_index(self, index=None):
 535         if not self.findable:
 536             return
 537         from search.index import Index
 538         Index.index_book(self)
 539
 540     # will make problems in conjunction with paid previews
 541     def download_pictures(self, remote_gallery_url):
 542         # This is only needed for legacy relative image paths.
 543         gallery_path = self.gallery_path()
 544         # delete previous files, so we don't include old files in ebooks
 545         if os.path.isdir(gallery_path):
 546             for filename in os.listdir(gallery_path):
 547                 file_path = os.path.join(gallery_path, filename)
 548                 os.unlink(file_path)
 549         ilustr_elements = list(self.wldocument().edoc.findall('//ilustr'))
 550         if ilustr_elements:
 551             makedirs(gallery_path)
 552             for ilustr in ilustr_elements:
 553                 ilustr_src = ilustr.get('src')
 554                 if '/' in ilustr_src:
 555                     continue
 556                 ilustr_path = os.path.join(gallery_path, ilustr_src)
 557                 urlretrieve('%s/%s' % (remote_gallery_url, ilustr_src), ilustr_path)
 558
 559     def load_abstract(self):
 560         self.abstract = AbstraktHtmlBuilder().build(
 561             self.wldocument2()).get_bytes().decode('utf-8')
 562
 563     def load_toc(self):
 564         self.toc = ''
 565         if self.html_file:
 566             parser = html.HTMLParser(encoding='utf-8')
 567             tree = html.parse(self.html_file.path, parser=parser)
 568             toc = tree.find('//div[@id="toc"]/ol')
 569             if toc is None or not len(toc):
 570                 return
 571             html_link = reverse('book_text', args=[self.slug])
 572             for a in toc.findall('.//a'):
 573                 a.attrib['href'] = html_link + a.attrib['href']
 574             self.toc = html.tostring(toc, encoding='unicode')
 575             # div#toc
 576
 577     @classmethod
 578     def from_xml_file(cls, xml_file, **kwargs):
 579         from django.core.files import File
 580         from librarian import dcparser
 581
 582         # use librarian to parse meta-data
 583         book_info = dcparser.parse(xml_file)
 584
 585         if not isinstance(xml_file, File):
 586             xml_file = File(open(xml_file))
 587
 588         try:
 589             return cls.from_text_and_meta(xml_file, book_info, **kwargs)
 590         finally:
 591             xml_file.close()
 592
 593     @classmethod
 594     def from_text_and_meta(cls, raw_file, book_info, overwrite=False, dont_build=None, search_index=True,
 595                            remote_gallery_url=None, days=0, findable=True, logo=None, logo_mono=None, logo_alt=None):
 596         from catalogue import tasks
 597
 598         if dont_build is None:
 599             dont_build = set()
 600         dont_build = set.union(set(dont_build), set(app_settings.DONT_BUILD))
 601
 602         # check for parts before we do anything
 603         children = []
 604         if hasattr(book_info, 'parts'):
 605             for part_url in book_info.parts:
 606                 try:
 607                     children.append(Book.objects.get(slug=part_url.slug))
 608                 except Book.DoesNotExist:
 609                     raise Book.DoesNotExist('Książka "%s" nie istnieje.' % part_url.slug)
 610
 611         # Read book metadata
 612         book_slug = book_info.url.slug
 613         if re.search(r'[^a-z0-9-]', book_slug):
 614             raise ValueError('Invalid characters in slug')
 615         book, created = Book.objects.get_or_create(slug=book_slug)
 616
 617         if created:
 618             book_shelves = []
 619             old_cover = None
 620             book.preview = bool(days)
 621             if book.preview:
 622                 book.preview_until = date.today() + timedelta(days)
 623         else:
 624             if not overwrite:
 625                 raise Book.AlreadyExists('Książka %s już istnieje' % book_slug)
 626             # Save shelves for this book
 627             book_shelves = list(book.tags.filter(category='set'))
 628             old_cover = book.cover_info()
 629
 630         # Save XML file
 631         book.xml_file.save('%s.xml' % book.slug, raw_file, save=False)
 632         if book.preview:
 633             book.xml_file.set_readable(False)
 634
 635         book.findable = findable
 636         book.language = book_info.language
 637         book.title = book_info.title
 638         if book_info.variant_of:
 639             book.common_slug = book_info.variant_of.slug
 640         else:
 641             book.common_slug = book.slug
 642         extra = book_info.to_dict()
 643         if logo:
 644             extra['logo'] = logo
 645         if logo_mono:
 646             extra['logo_mono'] = logo_mono
 647         if logo_alt:
 648             extra['logo_alt'] = logo_alt
 649         book.extra_info = json.dumps(extra)
 650         book.load_abstract()
 651         book.load_toc()
 652         book.save()
 653
 654         meta_tags = Tag.tags_from_info(book_info)
 655
 656         just_tags = [t for (t, rel) in meta_tags if not rel]
 657         book.tags = set(just_tags + book_shelves)
 658         book.save()  # update sort_key_author
 659
 660         book.translators.set([t for (t, rel) in meta_tags if rel == 'translator'])
 661
 662         cover_changed = old_cover != book.cover_info()
 663         obsolete_children = set(b for b in book.children.all()
 664                                 if b not in children)
 665         notify_cover_changed = []
 666         for n, child_book in enumerate(children):
 667             new_child = child_book.parent != book
 668             child_book.parent = book
 669             child_book.parent_number = n
 670             child_book.save()
 671             if new_child or cover_changed:
 672                 notify_cover_changed.append(child_book)
 673         # Disown unfaithful children and let them cope on their own.
 674         for child in obsolete_children:
 675             child.parent = None
 676             child.parent_number = 0
 677             child.save()
 678             if old_cover:
 679                 notify_cover_changed.append(child)
 680
 681         cls.repopulate_ancestors()
 682         tasks.update_counters.delay()
 683
 684         if remote_gallery_url:
 685             book.download_pictures(remote_gallery_url)
 686
 687         # No saves beyond this point.
 688
 689         # Build cover.
 690         if 'cover' not in dont_build:
 691             book.cover.build_delay()
 692             book.cover_clean.build_delay()
 693             book.cover_thumb.build_delay()
 694             book.cover_api_thumb.build_delay()
 695             book.simple_cover.build_delay()
 696             book.cover_ebookpoint.build_delay()
 697
 698         # Build HTML and ebooks.
 699         book.html_file.build_delay()
 700         if not children:
 701             for format_ in constants.EBOOK_FORMATS_WITHOUT_CHILDREN:
 702                 if format_ not in dont_build:
 703                     getattr(book, '%s_file' % format_).build_delay()
 704         for format_ in constants.EBOOK_FORMATS_WITH_CHILDREN:
 705             if format_ not in dont_build:
 706                 getattr(book, '%s_file' % format_).build_delay()
 707
 708         if not settings.NO_SEARCH_INDEX and search_index and findable:
 709             tasks.index_book.delay(book.id)
 710
 711         for child in notify_cover_changed:
 712             child.parent_cover_changed()
 713
 714         book.update_popularity()
 715         tasks.update_references.delay(book.id)
 716
 717         cls.published.send(sender=cls, instance=book)
 718         return book
 719
 720     # TODO TEST
 721     def update_references(self):
 722         Entity = apps.get_model('references', 'Entity')
 723         doc = self.wldocument2()
 724         doc.assign_ids()
 725
 726         refs = {}
 727         for i, ref_elem in enumerate(doc.references()):
 728             uri = ref_elem.attrib.get('href', '')
 729             if not uri:
 730                 continue
 731             if uri in refs:
 732                 ref = refs[uri]
 733             else:
 734                 entity, entity_created = Entity.objects.get_or_create(uri=uri)
 735                 if entity_created:
 736                     try:
 737                         entity.populate()
 738                     except:
 739                         pass
 740                     else:
 741                         entity.save()
 742                 ref, ref_created = entity.reference_set.get_or_create(book=self)
 743                 refs[uri] = ref
 744                 if not ref_created:
 745                     ref.occurence_set.all().delete()
 746             anchor = ref_elem.get_link()
 747
 748             snippet = ref_elem.get_snippet()
 749             b = builders['html-snippet']()
 750             for s in snippet:
 751                 s.html_build(b)
 752             html = b.output().get_bytes().decode('utf-8')
 753
 754             ref.occurence_set.create(
 755                 section=i,
 756                 anchor=anchor,
 757                 html=html
 758             )
 759         self.reference_set.exclude(entity__uri__in=refs).delete()
 760
 761     @property
 762     def references(self):
 763         return self.reference_set.all().select_related('entity')
 764
 765     @classmethod
 766     @transaction.atomic
 767     def repopulate_ancestors(cls):
 768         """Fixes the ancestry cache."""
 769         # TODO: table names
 770         cursor = connection.cursor()
 771         if connection.vendor == 'postgres':
 772             cursor.execute("TRUNCATE catalogue_book_ancestor")
 773             cursor.execute("""
 774                 WITH RECURSIVE ancestry AS (
 775                     SELECT book.id, book.parent_id
 776                     FROM catalogue_book AS book
 777                     WHERE book.parent_id IS NOT NULL
 778                     UNION
 779                     SELECT ancestor.id, book.parent_id
 780                     FROM ancestry AS ancestor, catalogue_book AS book
 781                     WHERE ancestor.parent_id = book.id
 782                         AND book.parent_id IS NOT NULL
 783                     )
 784                 INSERT INTO catalogue_book_ancestor
 785                     (from_book_id, to_book_id)
 786                     SELECT id, parent_id
 787                     FROM ancestry
 788                     ORDER BY id;
 789                 """)
 790         else:
 791             cursor.execute("DELETE FROM catalogue_book_ancestor")
 792             for b in cls.objects.exclude(parent=None):
 793                 parent = b.parent
 794                 while parent is not None:
 795                     b.ancestor.add(parent)
 796                     parent = parent.parent
 797
 798     @property
 799     def ancestors(self):
 800         if self.parent:
 801             for anc in self.parent.ancestors:
 802                 yield anc
 803             yield self.parent
 804         else:
 805             return []
 806
 807     def clear_cache(self):
 808         clear_cached_renders(self.mini_box)
 809         clear_cached_renders(self.mini_box_nolink)
 810
 811     def cover_info(self, inherit=True):
 812         """Returns a dictionary to serve as fallback for BookInfo.
 813
 814         For now, the only thing inherited is the cover image.
 815         """
 816         need = False
 817         info = {}
 818         for field in ('cover_url', 'cover_by', 'cover_source'):
 819             val = self.get_extra_info_json().get(field)
 820             if val:
 821                 info[field] = val
 822             else:
 823                 need = True
 824         if inherit and need and self.parent is not None:
 825             parent_info = self.parent.cover_info()
 826             parent_info.update(info)
 827             info = parent_info
 828         return info
 829
 830     def related_themes(self):
 831         return Tag.objects.usage_for_queryset(
 832             Fragment.objects.filter(models.Q(book=self) | models.Q(book__ancestor=self)),
 833             counts=True).filter(category='theme').order_by('-count')
 834
 835     def parent_cover_changed(self):
 836         """Called when parent book's cover image is changed."""
 837         if not self.cover_info(inherit=False):
 838             if 'cover' not in app_settings.DONT_BUILD:
 839                 self.cover.build_delay()
 840                 self.cover_clean.build_delay()
 841                 self.cover_thumb.build_delay()
 842                 self.cover_api_thumb.build_delay()
 843                 self.simple_cover.build_delay()
 844                 self.cover_ebookpoint.build_delay()
 845             for format_ in constants.EBOOK_FORMATS_WITH_COVERS:
 846                 if format_ not in app_settings.DONT_BUILD:
 847                     getattr(self, '%s_file' % format_).build_delay()
 848             for child in self.children.all():
 849                 child.parent_cover_changed()
 850
 851     def other_versions(self):
 852         """Find other versions (i.e. in other languages) of the book."""
 853         return type(self).objects.filter(common_slug=self.common_slug, findable=True).exclude(pk=self.pk)
 854
 855     def parents(self):
 856         books = []
 857         parent = self.parent
 858         while parent is not None:
 859             books.insert(0, parent)
 860             parent = parent.parent
 861         return books
 862
 863     def pretty_title(self, html_links=False):
 864         names = [(tag.name, tag.get_absolute_url()) for tag in self.authors().only('name', 'category', 'slug')]
 865         books = self.parents() + [self]
 866         names.extend([(b.title, b.get_absolute_url()) for b in books])
 867
 868         if html_links:
 869             names = ['<a href="%s">%s</a>' % (tag[1], tag[0]) for tag in names]
 870         else:
 871             names = [tag[0] for tag in names]
 872         return ', '.join(names)
 873
 874     def publisher(self):
 875         publisher = self.get_extra_info_json()['publisher']
 876         if isinstance(publisher, str):
 877             return publisher
 878         elif isinstance(publisher, list):
 879             return ', '.join(publisher)
 880
 881     @classmethod
 882     def tagged_top_level(cls, tags):
 883         """ Returns top-level books tagged with `tags`.
 884
 885         It only returns those books which don't have ancestors which are
 886         also tagged with those tags.
 887
 888         """
 889         objects = cls.tagged.with_all(tags)
 890         return objects.filter(findable=True).exclude(ancestor__in=objects)
 891
 892     @classmethod
 893     def book_list(cls, book_filter=None):
 894         """Generates a hierarchical listing of all books.
 895
 896         Books are optionally filtered with a test function.
 897
 898         """
 899
 900         books_by_parent = {}
 901         books = cls.objects.filter(findable=True).order_by('parent_number', 'sort_key').only('title', 'parent', 'slug', 'extra_info')
 902         if book_filter:
 903             books = books.filter(book_filter).distinct()
 904
 905             book_ids = set(b['pk'] for b in books.values("pk").iterator())
 906             for book in books.iterator():
 907                 parent = book.parent_id
 908                 if parent not in book_ids:
 909                     parent = None
 910                 books_by_parent.setdefault(parent, []).append(book)
 911         else:
 912             for book in books.iterator():
 913                 books_by_parent.setdefault(book.parent_id, []).append(book)
 914
 915         orphans = []
 916         books_by_author = OrderedDict()
 917         for tag in Tag.objects.filter(category='author').iterator():
 918             books_by_author[tag] = []
 919
 920         for book in books_by_parent.get(None, ()):
 921             authors = list(book.authors().only('pk'))
 922             if authors:
 923                 for author in authors:
 924                     books_by_author[author].append(book)
 925             else:
 926                 orphans.append(book)
 927
 928         return books_by_author, orphans, books_by_parent
 929
 930     _audiences_pl = {
 931         "SP": (1, "szkoła podstawowa"),
 932         "SP1": (1, "szkoła podstawowa"),
 933         "SP2": (1, "szkoła podstawowa"),
 934         "SP3": (1, "szkoła podstawowa"),
 935         "P": (1, "szkoła podstawowa"),
 936         "G": (2, "gimnazjum"),
 937         "L": (3, "liceum"),
 938         "LP": (3, "liceum"),
 939     }
 940
 941     def audiences_pl(self):
 942         audiences = self.get_extra_info_json().get('audiences', [])
 943         audiences = sorted(set([self._audiences_pl.get(a, (99, a)) for a in audiences]))
 944         return [a[1] for a in audiences]
 945
 946     def stage_note(self):
 947         stage = self.get_extra_info_json().get('stage')
 948         if stage and stage < '0.4':
 949             return (_('Ten utwór wymaga uwspółcześnienia'),
 950                     reverse('infopage', args=['wymagajace-uwspolczesnienia']))
 951         else:
 952             return None, None
 953
 954     def choose_fragments(self, number):
 955         fragments = self.fragments.order_by()
 956         fragments_count = fragments.count()
 957         if not fragments_count and self.children.exists():
 958             fragments = Fragment.objects.filter(book__ancestor=self).order_by()
 959             fragments_count = fragments.count()
 960         if fragments_count:
 961             if fragments_count > number:
 962                 offset = randint(0, fragments_count - number)
 963             else:
 964                 offset = 0
 965             return fragments[offset : offset + number]
 966         elif self.parent:
 967             return self.parent.choose_fragments(number)
 968         else:
 969             return []
 970
 971     def choose_fragment(self):
 972         fragments = self.choose_fragments(1)
 973         if fragments:
 974             return fragments[0]
 975         else:
 976             return None
 977
 978     def fragment_data(self):
 979         fragment = self.choose_fragment()
 980         if fragment:
 981             return {
 982                 'title': fragment.book.pretty_title(),
 983                 'html': re.sub('</?blockquote[^>]*>', '', fragment.get_short_text()),
 984             }
 985         else:
 986             return None
 987
 988     def update_popularity(self):
 989         count = self.tags.filter(category='set').values('user').order_by('user').distinct().count()
 990         try:
 991             pop = self.popularity
 992             pop.count = count
 993             pop.save()
 994         except BookPopularity.DoesNotExist:
 995             BookPopularity.objects.create(book=self, count=count)
 996
 997     def ridero_link(self):
 998         return 'https://ridero.eu/%s/books/wl_%s/' % (get_language(), self.slug.replace('-', '_'))
 999
1000     def like(self, user):
1001         from social.utils import likes, get_set, set_sets
1002         if not likes(user, self):
1003             tag = get_set(user, '')
1004             set_sets(user, self, [tag])
1005
1006     def unlike(self, user):
1007         from social.utils import likes, set_sets
1008         if likes(user, self):
1009             set_sets(user, self, [])
1010
1011     def full_sort_key(self):
1012         return self.SORT_KEY_SEP.join((self.sort_key_author, self.sort_key, str(self.id)))
1013
1014     def cover_color(self):
1015         return WLCover.epoch_colors.get(self.get_extra_info_json().get('epoch'), '#000000')
1016
1017     @cached_render('catalogue/book_mini_box.html')
1018     def mini_box(self):
1019         return {
1020             'book': self
1021         }
1022
1023     @cached_render('catalogue/book_mini_box.html')
1024     def mini_box_nolink(self):
1025         return {
1026             'book': self,
1027             'no_link': True,
1028         }
1029
1030
1031 class BookPopularity(models.Model):
1032     book = models.OneToOneField(Book, models.CASCADE, related_name='popularity')
1033     count = models.IntegerField(default=0, db_index=True)