src/catalogue/models/book.py

   1 # This file is part of Wolne Lektury, licensed under GNU Affero GPLv3 or later.
   2 # Copyright © Fundacja Wolne Lektury. See NOTICE for more information.
   3 #
   4 from collections import OrderedDict
   5 import json
   6 from datetime import date, timedelta
   7 from random import randint
   8 import os.path
   9 import re
  10 import requests
  11 from slugify import slugify
  12 from sortify import sortify
  13 from urllib.request import urlretrieve
  14 from django.apps import apps
  15 from django.conf import settings
  16 from django.db import connection, models, transaction
  17 import django.dispatch
  18 from django.contrib.contenttypes.fields import GenericRelation
  19 from django.template.loader import render_to_string
  20 from django.urls import reverse
  21 from django.utils.translation import gettext_lazy as _, get_language
  22 from fnpdjango.storage import BofhFileSystemStorage
  23 from lxml import html
  24 from librarian.cover import WLCover
  25 from librarian.html import transform_abstrakt
  26 from librarian.builders import builders
  27 from newtagging import managers
  28 from catalogue import constants
  29 from catalogue import fields
  30 from catalogue.models import Tag, Fragment, BookMedia
  31 from catalogue.utils import create_zip, gallery_url, gallery_path, split_tags, get_random_hash
  32 from catalogue.models.tag import prefetched_relations
  33 from catalogue import app_settings
  34 from wolnelektury.utils import makedirs, cached_render, clear_cached_renders
  35
  36 bofh_storage = BofhFileSystemStorage()
  37
  38
  39 class Book(models.Model):
  40     """Represents a book imported from WL-XML."""
  41     title = models.CharField('tytuł', max_length=32767)
  42     sort_key = models.CharField('klucz sortowania', max_length=120, db_index=True, db_collation='C', editable=False)
  43     sort_key_author = models.CharField(
  44         'klucz sortowania wg autora', max_length=120, db_index=True, db_collation='C', editable=False, default='')
  45     slug = models.SlugField('slug', max_length=120, db_index=True, unique=True)
  46     common_slug = models.SlugField('wspólny slug', max_length=120, db_index=True)
  47     language = models.CharField('kod języka', max_length=3, db_index=True, default=app_settings.DEFAULT_LANGUAGE)
  48     description = models.TextField('opis', blank=True)
  49     license = models.CharField('licencja', max_length=255, blank=True, db_index=True)
  50     abstract = models.TextField('abstrakt', blank=True)
  51     toc = models.TextField('spis treści', blank=True)
  52     created_at = models.DateTimeField('data utworzenia', auto_now_add=True, db_index=True)
  53     changed_at = models.DateTimeField('data motyfikacji', auto_now=True, db_index=True)
  54     parent_number = models.IntegerField('numer w ramach rodzica', default=0)
  55     extra_info = models.TextField('dodatkowe informacje', default='{}')
  56     gazeta_link = models.CharField(blank=True, max_length=240)
  57     wiki_link = models.CharField(blank=True, max_length=240)
  58     print_on_demand = models.BooleanField('druk na żądanie', default=False)
  59     recommended = models.BooleanField('polecane', default=False)
  60     audio_length = models.CharField('długość audio', blank=True, max_length=8)
  61     preview = models.BooleanField('prapremiera', default=False)
  62     preview_until = models.DateField('prapremiera do', blank=True, null=True)
  63     preview_key = models.CharField(max_length=32, blank=True, null=True)
  64     findable = models.BooleanField('wyszukiwalna', default=True, db_index=True)
  65     can_sell = models.BooleanField('do sprzedaży', default=True)
  66     can_sell_mp3 = models.BooleanField('do sprzedaży mp3', default=False)
  67     isbn_mp3 = models.CharField('ISBN audiobooka', max_length=32, blank=True)
  68
  69     # files generated during publication
  70     xml_file = fields.XmlField(storage=bofh_storage, with_etag=False)
  71     html_file = fields.HtmlField(storage=bofh_storage)
  72     html_nonotes_file = fields.HtmlNonotesField(storage=bofh_storage)
  73     fb2_file = fields.Fb2Field(storage=bofh_storage)
  74     txt_file = fields.TxtField(storage=bofh_storage)
  75     epub_file = fields.EpubField(storage=bofh_storage)
  76     mobi_file = fields.MobiField(storage=bofh_storage)
  77     pdf_file = fields.PdfField(storage=bofh_storage)
  78
  79     cover = fields.CoverField('okładka', storage=bofh_storage)
  80     # Cleaner version of cover for thumbs
  81     cover_clean = fields.CoverCleanField('czysta okładka')
  82     cover_thumb = fields.CoverThumbField('miniatura okładki')
  83     cover_api_thumb = fields.CoverApiThumbField(
  84         'mniaturka okładki dla aplikacji')
  85     simple_cover = fields.SimpleCoverField('okładka dla aplikacji')
  86     cover_ebookpoint = fields.CoverEbookpointField(
  87         'okładka dla Ebookpoint')
  88
  89     ebook_formats = constants.EBOOK_FORMATS
  90     formats = ebook_formats + ['html', 'xml', 'html_nonotes']
  91
  92     parent = models.ForeignKey('self', models.CASCADE, blank=True, null=True, related_name='children')
  93     ancestor = models.ManyToManyField('self', blank=True, editable=False, related_name='descendant', symmetrical=False)
  94
  95     cached_author = models.CharField(blank=True, max_length=240, db_index=True)
  96     has_audience = models.BooleanField(default=False)
  97
  98     objects = models.Manager()
  99     tagged = managers.ModelTaggedItemManager(Tag)
 100     tags = managers.TagDescriptor(Tag)
 101     tag_relations = GenericRelation(Tag.intermediary_table_model, related_query_name='tagged_book')
 102     translators = models.ManyToManyField(Tag, blank=True)
 103     narrators = models.ManyToManyField(Tag, blank=True, related_name='narrated')
 104     has_audio = models.BooleanField(default=False)
 105     read_time = models.IntegerField(blank=True, null=True)
 106     pages = models.IntegerField(blank=True, null=True)
 107
 108     html_built = django.dispatch.Signal()
 109     published = django.dispatch.Signal()
 110
 111     SORT_KEY_SEP = '$'
 112
 113     is_book = True
 114
 115     class AlreadyExists(Exception):
 116         pass
 117
 118     class Meta:
 119         ordering = ('sort_key_author', 'sort_key')
 120         verbose_name = 'książka'
 121         verbose_name_plural = 'książki'
 122         app_label = 'catalogue'
 123
 124     def __str__(self):
 125         return self.title
 126
 127     def get_extra_info_json(self):
 128         return json.loads(self.extra_info or '{}')
 129
 130     def get_initial(self):
 131         try:
 132             return re.search(r'\w', self.title, re.U).group(0)
 133         except AttributeError:
 134             return ''
 135
 136     def authors(self):
 137         return self.tags.filter(category='author')
 138
 139     def epochs(self):
 140         return self.tags.filter(category='epoch')
 141
 142     def genres(self):
 143         return self.tags.filter(category='genre')
 144
 145     def kinds(self):
 146         return self.tags.filter(category='kind')
 147
 148     def tag_unicode(self, category):
 149         relations = prefetched_relations(self, category)
 150         if relations:
 151             return ', '.join(rel.tag.name for rel in relations)
 152         else:
 153             return ', '.join(self.tags.filter(category=category).values_list('name', flat=True))
 154
 155     def tags_by_category(self):
 156         return split_tags(self.tags.exclude(category__in=('set', 'theme')))
 157
 158     def author_unicode(self):
 159         return self.cached_author
 160
 161     def kind_unicode(self):
 162         return self.tag_unicode('kind')
 163
 164     def epoch_unicode(self):
 165         return self.tag_unicode('epoch')
 166
 167     def genre_unicode(self):
 168         return self.tag_unicode('genre')
 169
 170     def translator(self):
 171         translators = self.get_extra_info_json().get('translators')
 172         if not translators:
 173             return None
 174         if len(translators) > 3:
 175             translators = translators[:2]
 176             others = ' i inni'
 177         else:
 178             others = ''
 179         return ', '.join('\xa0'.join(reversed(translator.split(', ', 1))) for translator in translators) + others
 180
 181     def cover_source(self):
 182         return self.get_extra_info_json().get('cover_source', self.parent.cover_source() if self.parent else '')
 183
 184     @property
 185     def isbn_pdf(self):
 186         return self.get_extra_info_json().get('isbn_pdf')
 187
 188     @property
 189     def isbn_epub(self):
 190         return self.get_extra_info_json().get('isbn_epub')
 191
 192     @property
 193     def isbn_mobi(self):
 194         return self.get_extra_info_json().get('isbn_mobi')
 195
 196     @property
 197     def redakcja(self):
 198         return self.get_extra_info_json().get('about')
 199
 200     def is_accessible_to(self, user):
 201         if not self.preview:
 202             return True
 203         if not user.is_authenticated:
 204             return False
 205         Membership = apps.get_model('club', 'Membership')
 206         if Membership.is_active_for(user):
 207             return True
 208         Funding = apps.get_model('funding', 'Funding')
 209         if Funding.objects.filter(user=user, offer__book=self):
 210             return True
 211         return False
 212
 213     def save(self, force_insert=False, force_update=False, **kwargs):
 214         from sortify import sortify
 215
 216         self.sort_key = sortify(self.title)[:120]
 217         self.title = str(self.title)  # ???
 218
 219         try:
 220             author = self.authors().first().sort_key
 221         except AttributeError:
 222             author = ''
 223         self.sort_key_author = author
 224
 225         self.cached_author = self.tag_unicode('author')
 226         self.has_audience = 'audience' in self.get_extra_info_json()
 227
 228         if self.preview and not self.preview_key:
 229             self.preview_key = get_random_hash(self.slug)[:32]
 230
 231         ret = super(Book, self).save(
 232             force_insert=force_insert, force_update=force_update, **kwargs)
 233
 234         return ret
 235
 236     def get_absolute_url(self):
 237         return reverse('book_detail', args=[self.slug])
 238
 239     def gallery_path(self):
 240         return gallery_path(self.slug)
 241
 242     def gallery_url(self):
 243         return gallery_url(self.slug)
 244
 245     def get_first_text(self):
 246         if self.html_file:
 247             return self
 248         child = self.children.all().order_by('parent_number').first()
 249         if child is not None:
 250             return child.get_first_text()
 251
 252     def get_last_text(self):
 253         if self.html_file:
 254             return self
 255         child = self.children.all().order_by('parent_number').last()
 256         if child is not None:
 257             return child.get_last_text()
 258
 259     def get_prev_text(self):
 260         if not self.parent:
 261             return None
 262         sibling = self.parent.children.filter(parent_number__lt=self.parent_number).order_by('-parent_number').first()
 263         if sibling is not None:
 264             return sibling.get_last_text()
 265
 266         if self.parent.html_file:
 267             return self.parent
 268
 269         return self.parent.get_prev_text()
 270
 271     def get_next_text(self, inside=True):
 272         if inside:
 273             child = self.children.order_by('parent_number').first()
 274             if child is not None:
 275                 return child.get_first_text()
 276
 277         if not self.parent:
 278             return None
 279         sibling = self.parent.children.filter(parent_number__gt=self.parent_number).order_by('parent_number').first()
 280         if sibling is not None:
 281             return sibling.get_first_text()
 282         return self.parent.get_next_text(inside=False)
 283
 284     def get_siblings(self):
 285         if not self.parent:
 286             return []
 287         return self.parent.children.all().order_by('parent_number')
 288
 289     def get_children(self):
 290         return self.children.all().order_by('parent_number')
 291
 292     @property
 293     def name(self):
 294         return self.title
 295
 296     def language_code(self):
 297         return constants.LANGUAGES_3TO2.get(self.language, self.language)
 298
 299     def language_name(self):
 300         return dict(settings.LANGUAGES).get(self.language_code(), "")
 301
 302     def is_foreign(self):
 303         return self.language_code() != settings.LANGUAGE_CODE
 304
 305     def set_audio_length(self):
 306         length = self.get_audio_length()
 307         if length > 0:
 308             self.audio_length = self.format_audio_length(length)
 309             self.save()
 310
 311     @staticmethod
 312     def format_audio_length(seconds):
 313         """
 314         >>> Book.format_audio_length(1)
 315         '0:01'
 316         >>> Book.format_audio_length(3661)
 317         '1:01:01'
 318         """
 319         if seconds < 60*60:
 320             minutes = seconds // 60
 321             seconds = seconds % 60
 322             return '%d:%02d' % (minutes, seconds)
 323         else:
 324             hours = seconds // 3600
 325             minutes = seconds % 3600 // 60
 326             seconds = seconds % 60
 327             return '%d:%02d:%02d' % (hours, minutes, seconds)
 328
 329     def get_audio_length(self):
 330         total = 0
 331         for media in self.get_mp3() or ():
 332             total += app_settings.GET_MP3_LENGTH(media.file.path)
 333         return int(total)
 334
 335     def get_time(self):
 336         return round(self.xml_file.size / 1000 * 40)
 337
 338     def has_media(self, type_):
 339         if type_ in Book.formats:
 340             return bool(getattr(self, "%s_file" % type_))
 341         else:
 342             return self.media.filter(type=type_).exists()
 343
 344     def get_media(self, type_):
 345         if self.has_media(type_):
 346             if type_ in Book.formats:
 347                 return getattr(self, "%s_file" % type_)
 348             else:
 349                 return self.media.filter(type=type_)
 350         else:
 351             return None
 352
 353     def get_mp3(self):
 354         return self.get_media("mp3")
 355
 356     def get_odt(self):
 357         return self.get_media("odt")
 358
 359     def get_ogg(self):
 360         return self.get_media("ogg")
 361
 362     def get_daisy(self):
 363         return self.get_media("daisy")
 364
 365     def get_audio_epub(self):
 366         return self.get_media("audio.epub")
 367
 368     def media_url(self, format_):
 369         media = self.get_media(format_)
 370         if media:
 371             if self.preview:
 372                 return reverse('embargo_link', kwargs={'key': self.preview_key, 'slug': self.slug, 'format_': format_})
 373             else:
 374                 return media.url
 375         else:
 376             return None
 377
 378     def html_url(self):
 379         return self.media_url('html')
 380
 381     def html_nonotes_url(self):
 382         return self.media_url('html_nonotes')
 383
 384     def pdf_url(self):
 385         return self.media_url('pdf')
 386
 387     def epub_url(self):
 388         return self.media_url('epub')
 389
 390     def mobi_url(self):
 391         return self.media_url('mobi')
 392
 393     def txt_url(self):
 394         return self.media_url('txt')
 395
 396     def fb2_url(self):
 397         return self.media_url('fb2')
 398
 399     def xml_url(self):
 400         return self.media_url('xml')
 401
 402     def has_description(self):
 403         return len(self.description) > 0
 404     has_description.short_description = 'opis'
 405     has_description.boolean = True
 406
 407     def has_mp3_file(self):
 408         return self.has_media("mp3")
 409     has_mp3_file.short_description = 'MP3'
 410     has_mp3_file.boolean = True
 411
 412     def has_ogg_file(self):
 413         return self.has_media("ogg")
 414     has_ogg_file.short_description = 'OGG'
 415     has_ogg_file.boolean = True
 416
 417     def has_daisy_file(self):
 418         return self.has_media("daisy")
 419     has_daisy_file.short_description = 'DAISY'
 420     has_daisy_file.boolean = True
 421
 422     def has_sync_file(self):
 423         return settings.FEATURE_SYNCHRO and self.has_media("sync")
 424
 425     def build_sync_file(self):
 426         from lxml import html
 427         from django.core.files.base import ContentFile
 428         with self.html_file.open('rb') as f:
 429             h = html.fragment_fromstring(f.read().decode('utf-8'))
 430
 431         durations = [
 432             m['mp3'].duration
 433             for m in self.get_audiobooks()[0]
 434         ]
 435         if settings.MOCK_DURATIONS:
 436             durations = settings.MOCK_DURATIONS
 437
 438         sync = []
 439         ts = None
 440         sid = 1
 441         dirty = False
 442         for elem in h.iter():
 443             if elem.get('data-audio-ts'):
 444                 part, ts = int(elem.get('data-audio-part')), float(elem.get('data-audio-ts'))
 445                 ts = str(round(sum(durations[:part - 1]) + ts, 3))
 446                 # check if inside verse
 447                 p = elem.getparent()
 448                 while p is not None:
 449                     # Workaround for missing ids.
 450                     if 'verse' in p.get('class', ''):
 451                         if not p.get('id'):
 452                             p.set('id', f'syn{sid}')
 453                             dirty = True
 454                             sid += 1
 455                         sync.append((ts, p.get('id')))
 456                         ts = None
 457                         break
 458                     p = p.getparent()
 459             elif ts:
 460                 cls = elem.get('class', '')
 461                 # Workaround for missing ids.
 462                 if 'paragraph' in cls or 'verse' in cls or elem.tag in ('h1', 'h2', 'h3', 'h4'):
 463                     if not elem.get('id'):
 464                         elem.set('id', f'syn{sid}')
 465                         dirty = True
 466                         sid += 1
 467                     sync.append((ts, elem.get('id')))
 468                     ts = None
 469         if dirty:
 470             htext = html.tostring(h, encoding='utf-8')
 471             with open(self.html_file.path, 'wb') as f:
 472                 f.write(htext)
 473         try:
 474             bm = self.media.get(type='sync')
 475         except:
 476             bm = BookMedia(book=self, type='sync')
 477         sync = (
 478             '27\n' + '\n'.join(
 479                 f'{s[0]}\t{sync[i+1][0]}\t{s[1]}' for i, s in enumerate(sync[:-1])
 480             )).encode('latin1')
 481         bm.file.save(
 482             None, ContentFile(sync)
 483             )
 484
 485     def get_sync(self):
 486         if not self.has_sync_file():
 487             return []
 488         with self.get_media('sync').first().file.open('r') as f:
 489             sync = f.read().split('\n')
 490         offset = float(sync[0])
 491         items = []
 492         for line in sync[1:]:
 493             if not line:
 494                 continue
 495             start, end, elid = line.split()
 496             items.append([elid, float(start) + offset])
 497         return items
 498
 499     def sync_ts(self, ts):
 500         elid = None
 501         for cur_id, t in self.get_sync():
 502             if ts >= t:
 503                 elid = cur_id
 504             else:
 505                 break
 506         return elid
 507
 508     def sync_elid(self, elid):
 509         for cur_id, t in self.get_sync():
 510             if cur_id == elid:
 511                 return t
 512
 513     def has_audio_epub_file(self):
 514         return self.has_media("audio.epub")
 515
 516     @property
 517     def media_daisy(self):
 518         return self.get_media('daisy')
 519
 520     @property
 521     def media_audio_epub(self):
 522         return self.get_media('audio.epub')
 523
 524     def get_audiobooks(self, with_children=False, processing=False):
 525         ogg_files = {}
 526         for m in self.media.filter(type='ogg').order_by().iterator():
 527             ogg_files[m.name] = m
 528
 529         audiobooks = []
 530         projects = set()
 531         total_duration = 0
 532         for mp3 in self.media.filter(type='mp3').iterator():
 533             # ogg files are always from the same project
 534             meta = mp3.get_extra_info_json()
 535             project = meta.get('project')
 536             if not project:
 537                 # temporary fallback
 538                 project = 'CzytamySłuchając'
 539
 540             projects.add((project, meta.get('funded_by', '')))
 541             total_duration += mp3.duration or 0
 542
 543             media = {'mp3': mp3}
 544
 545             ogg = ogg_files.get(mp3.name)
 546             if ogg:
 547                 media['ogg'] = ogg
 548             audiobooks.append(media)
 549
 550         if with_children:
 551             for child in self.get_children():
 552                 ch_audiobooks, ch_projects, ch_duration = child.get_audiobooks(
 553                     with_children=True, processing=True)
 554                 audiobooks.append({'part': child})
 555                 audiobooks += ch_audiobooks
 556                 projects.update(ch_projects)
 557                 total_duration += ch_duration
 558
 559         if not processing:
 560             projects = sorted(projects)
 561             total_duration = '%d:%02d' % (
 562                 total_duration // 60,
 563                 total_duration % 60
 564             )
 565
 566         return audiobooks, projects, total_duration
 567
 568     def get_audiobooks_with_children(self):
 569         return self.get_audiobooks(with_children=True)
 570
 571     def wldocument(self, parse_dublincore=True, inherit=True):
 572         from catalogue.import_utils import ORMDocProvider
 573         from librarian.parser import WLDocument
 574
 575         if inherit and self.parent:
 576             meta_fallbacks = self.parent.cover_info()
 577         else:
 578             meta_fallbacks = None
 579
 580         return WLDocument.from_file(
 581             self.xml_file.path,
 582             provider=ORMDocProvider(self),
 583             parse_dublincore=parse_dublincore,
 584             meta_fallbacks=meta_fallbacks)
 585
 586     def wldocument2(self):
 587         from catalogue.import_utils import ORMDocProvider
 588         from librarian.document import WLDocument
 589         doc = WLDocument(
 590             self.xml_file.path,
 591             provider=ORMDocProvider(self)
 592         )
 593         doc.meta.update(self.cover_info())
 594         return doc
 595
 596
 597     @staticmethod
 598     def zip_format(format_):
 599         def pretty_file_name(book):
 600             return "%s/%s.%s" % (
 601                 book.get_extra_info_json()['author'],
 602                 book.slug,
 603                 format_)
 604
 605         field_name = "%s_file" % format_
 606         field = getattr(Book, field_name)
 607         books = Book.objects.filter(parent=None).exclude(**{field_name: ""}).exclude(preview=True).exclude(findable=False)
 608         paths = [(pretty_file_name(b), getattr(b, field_name).path) for b in books.iterator()]
 609         return create_zip(paths, field.ZIP)
 610
 611     def zip_audiobooks(self, format_):
 612         bm = BookMedia.objects.filter(book=self, type=format_)
 613         paths = map(lambda bm: (bm.get_nice_filename(), bm.file.path), bm)
 614         licenses = set()
 615         for m in bm:
 616             license = constants.LICENSES.get(
 617                 m.get_extra_info_json().get('license'), {}
 618             ).get('locative')
 619             if license:
 620                 licenses.add(license)
 621         readme = render_to_string('catalogue/audiobook_zip_readme.txt', {
 622             'licenses': licenses,
 623             'meta': self.wldocument2().meta,
 624         })
 625         return create_zip(paths, "%s_%s" % (self.slug, format_), {'informacje.txt': readme})
 626
 627     def search_index(self, index=None):
 628         if not self.findable:
 629             return
 630         from search.index import Index
 631         Index.index_book(self)
 632
 633     # will make problems in conjunction with paid previews
 634     def download_pictures(self, remote_gallery_url):
 635         # This is only needed for legacy relative image paths.
 636         gallery_path = self.gallery_path()
 637         # delete previous files, so we don't include old files in ebooks
 638         if os.path.isdir(gallery_path):
 639             for filename in os.listdir(gallery_path):
 640                 file_path = os.path.join(gallery_path, filename)
 641                 os.unlink(file_path)
 642         ilustr_elements = list(self.wldocument().edoc.findall('//ilustr'))
 643         if ilustr_elements:
 644             makedirs(gallery_path)
 645             for ilustr in ilustr_elements:
 646                 ilustr_src = ilustr.get('src')
 647                 if '/' in ilustr_src:
 648                     continue
 649                 ilustr_path = os.path.join(gallery_path, ilustr_src)
 650                 urlretrieve('%s/%s' % (remote_gallery_url, ilustr_src), ilustr_path)
 651
 652     def load_abstract(self):
 653         abstract = self.wldocument(parse_dublincore=False).edoc.getroot().find('.//abstrakt')
 654         if abstract is not None:
 655             self.abstract = transform_abstrakt(abstract)
 656         else:
 657             self.abstract = ''
 658
 659     def load_toc(self):
 660         self.toc = ''
 661         if self.html_file:
 662             parser = html.HTMLParser(encoding='utf-8')
 663             tree = html.parse(self.html_file.path, parser=parser)
 664             toc = tree.find('//div[@id="toc"]/ol')
 665             if toc is None or not len(toc):
 666                 return
 667             html_link = reverse('book_text', args=[self.slug])
 668             for a in toc.findall('.//a'):
 669                 a.attrib['href'] = html_link + a.attrib['href']
 670             self.toc = html.tostring(toc, encoding='unicode')
 671             # div#toc
 672
 673     @classmethod
 674     def from_xml_file(cls, xml_file, **kwargs):
 675         from django.core.files import File
 676         from librarian import dcparser
 677
 678         # use librarian to parse meta-data
 679         book_info = dcparser.parse(xml_file)
 680
 681         if not isinstance(xml_file, File):
 682             xml_file = File(open(xml_file))
 683
 684         try:
 685             return cls.from_text_and_meta(xml_file, book_info, **kwargs)
 686         finally:
 687             xml_file.close()
 688
 689     @classmethod
 690     def from_text_and_meta(cls, raw_file, book_info, overwrite=False, dont_build=None, search_index=True,
 691                            remote_gallery_url=None, days=0, findable=True, logo=None, logo_mono=None, logo_alt=None, can_sell=None, isbn_mp3=None):
 692         from catalogue import tasks
 693
 694         if dont_build is None:
 695             dont_build = set()
 696         dont_build = set.union(set(dont_build), set(app_settings.DONT_BUILD))
 697
 698         # check for parts before we do anything
 699         children = []
 700         if hasattr(book_info, 'parts'):
 701             for part_url in book_info.parts:
 702                 try:
 703                     children.append(Book.objects.get(slug=part_url.slug))
 704                 except Book.DoesNotExist:
 705                     raise Book.DoesNotExist('Książka "%s" nie istnieje.' % part_url.slug)
 706
 707         # Read book metadata
 708         book_slug = book_info.url.slug
 709         if re.search(r'[^a-z0-9-]', book_slug):
 710             raise ValueError('Invalid characters in slug')
 711         book, created = Book.objects.get_or_create(slug=book_slug)
 712
 713         if created:
 714             book_shelves = []
 715             old_cover = None
 716             book.preview = bool(days)
 717             if book.preview:
 718                 book.preview_until = date.today() + timedelta(days)
 719         else:
 720             if not overwrite:
 721                 raise Book.AlreadyExists('Książka %s już istnieje' % book_slug)
 722             # Save shelves for this book
 723             book_shelves = list(book.tags.filter(category='set'))
 724             old_cover = book.cover_info()
 725
 726         # Save XML file
 727         book.xml_file.save('%s.xml' % book.slug, raw_file, save=False)
 728         if book.preview:
 729             book.xml_file.set_readable(False)
 730
 731         book.findable = findable
 732         book.language = book_info.language
 733         book.title = book_info.title
 734         book.license = book_info.license or ''
 735         if book_info.variant_of:
 736             book.common_slug = book_info.variant_of.slug
 737         else:
 738             book.common_slug = book.slug
 739         extra = book_info.to_dict()
 740         if logo:
 741             extra['logo'] = logo
 742         if logo_mono:
 743             extra['logo_mono'] = logo_mono
 744         if logo_alt:
 745             extra['logo_alt'] = logo_alt
 746         if can_sell is not None:
 747             book.can_sell = can_sell
 748         if isbn_mp3 is not None:
 749             book.isbn_mp3 = isbn_mp3
 750         book.extra_info = json.dumps(extra)
 751         book.load_abstract()
 752         book.load_toc()
 753         book.save()
 754
 755         book.update_stats()
 756
 757         meta_tags = Tag.tags_from_info(book_info)
 758
 759         just_tags = [t for (t, rel) in meta_tags if not rel]
 760         book.tags = set(just_tags + book_shelves)
 761         book.save()  # update sort_key_author
 762
 763         book.translators.set([t for (t, rel) in meta_tags if rel == 'translator'])
 764
 765         cover_changed = old_cover != book.cover_info()
 766         obsolete_children = set(b for b in book.children.all()
 767                                 if b not in children)
 768         notify_cover_changed = []
 769         for n, child_book in enumerate(children):
 770             new_child = child_book.parent != book
 771             child_book.parent = book
 772             child_book.parent_number = n
 773             child_book.save()
 774             if new_child or cover_changed:
 775                 notify_cover_changed.append(child_book)
 776         # Disown unfaithful children and let them cope on their own.
 777         for child in obsolete_children:
 778             child.parent = None
 779             child.parent_number = 0
 780             child.save()
 781             if old_cover:
 782                 notify_cover_changed.append(child)
 783
 784         cls.repopulate_ancestors()
 785         tasks.update_counters.delay()
 786
 787         if remote_gallery_url:
 788             book.download_pictures(remote_gallery_url)
 789
 790         # No saves beyond this point.
 791
 792         # Build cover.
 793         if 'cover' not in dont_build:
 794             book.cover.build_delay()
 795             book.cover_clean.build_delay()
 796             book.cover_thumb.build_delay()
 797             book.cover_api_thumb.build_delay()
 798             book.simple_cover.build_delay()
 799             book.cover_ebookpoint.build_delay()
 800
 801         # Build HTML and ebooks.
 802         book.html_file.build_delay()
 803         if not children:
 804             for format_ in constants.EBOOK_FORMATS_WITHOUT_CHILDREN:
 805                 if format_ not in dont_build:
 806                     getattr(book, '%s_file' % format_).build_delay()
 807         for format_ in constants.EBOOK_FORMATS_WITH_CHILDREN:
 808             if format_ not in dont_build:
 809                 getattr(book, '%s_file' % format_).build_delay()
 810         if 'html_nonotes' not in dont_build:
 811             book.html_nonotes_file.build_delay()
 812
 813         if not settings.NO_SEARCH_INDEX and search_index and findable:
 814             tasks.index_book.delay(book.id)
 815
 816         for child in notify_cover_changed:
 817             child.parent_cover_changed()
 818
 819         book.update_popularity()
 820         if 'references' not in dont_build:
 821             tasks.update_references.delay(book.id)
 822
 823         cls.published.send(sender=cls, instance=book)
 824         return book
 825
 826     def update_stats(self):
 827         stats = self.wldocument2().get_statistics()['total']
 828         self.pages = round(
 829             stats.get('verses_with_fn', 0) / 30 +
 830             stats.get('chars_out_verse_with_fn', 0) / 1800)
 831         self.read_time = round(self.get_time())
 832         self.save(update_fields=['pages', 'read_time'])
 833         if self.parent is not None:
 834             self.parent.update_stats()
 835
 836     def update_references(self):
 837         Entity = apps.get_model('references', 'Entity')
 838         doc = self.wldocument2()
 839         doc._compat_assign_section_ids()
 840         doc._compat_assign_ordered_ids()
 841         refs = {}
 842         for ref_elem in doc.references():
 843             uri = ref_elem.attrib.get('href', '')
 844             if not uri:
 845                 continue
 846             if uri in refs:
 847                 ref = refs[uri]
 848             else:
 849                 entity, entity_created = Entity.objects.get_or_create(uri=uri)
 850                 if entity_created:
 851                     try:
 852                         entity.populate()
 853                     except:
 854                         pass
 855                     else:
 856                         entity.save()
 857                 ref, ref_created = entity.reference_set.get_or_create(book=self)
 858                 refs[uri] = ref
 859                 if not ref_created:
 860                     ref.occurence_set.all().delete()
 861             sec = ref_elem.get_link()
 862             m = re.match(r'sec(\d+)', sec)
 863             assert m is not None
 864             sec = int(m.group(1))
 865             snippet = ref_elem.get_snippet()
 866             b = builders['html-snippet']()
 867             for s in snippet:
 868                 s.html_build(b)
 869             html = b.output().get_bytes().decode('utf-8')
 870
 871             ref.occurence_set.create(
 872                 section=sec,
 873                 html=html
 874             )
 875         self.reference_set.exclude(entity__uri__in=refs).delete()
 876
 877     @property
 878     def references(self):
 879         return self.reference_set.all().select_related('entity')
 880
 881     def update_has_audio(self):
 882         self.has_audio = False
 883         if self.media.filter(type='mp3').exists():
 884             self.has_audio = True
 885         if self.descendant.filter(has_audio=True).exists():
 886             self.has_audio = True
 887         self.save(update_fields=['has_audio'])
 888         if self.parent is not None:
 889             self.parent.update_has_audio()
 890
 891     def update_narrators(self):
 892         narrator_names = set()
 893         for bm in self.media.filter(type='mp3'):
 894             narrator_names.update(set(
 895                 a.strip() for a in re.split(r',|\si\s', bm.artist)
 896             ))
 897         narrators = []
 898
 899         for name in narrator_names:
 900             if not name: continue
 901             slug = slugify(name)
 902             try:
 903                 t = Tag.objects.get(category='author', slug=slug)
 904             except Tag.DoesNotExist:
 905                 sort_key = sortify(
 906                     ' '.join(name.rsplit(' ', 1)[::-1]).lower()
 907                 )
 908                 t = Tag.objects.create(
 909                     category='author',
 910                     name_pl=name,
 911                     slug=slug,
 912                     sort_key=sort_key,
 913                 )
 914             narrators.append(t)
 915         self.narrators.set(narrators)
 916
 917     def update_can_sell_mp3(self):
 918         ret = True
 919         for child in self.get_children():
 920             child.update_can_sell_mp3()
 921             if not child.can_sell_mp3:
 922                 ret = False
 923         if self.has_mp3_file():
 924             audio_items = requests.get(f'https://audio.wolnelektury.pl/archive/book/{self.slug}.json').json()['items']
 925             if not all(x['project']['can_sell'] for x in audio_items):
 926                 ret = False
 927         self.can_sell_mp3 = ret
 928         self.save(update_fields=['can_sell_mp3'])
 929
 930     @classmethod
 931     @transaction.atomic
 932     def repopulate_ancestors(cls):
 933         """Fixes the ancestry cache."""
 934         # TODO: table names
 935         cursor = connection.cursor()
 936         if connection.vendor == 'postgres':
 937             cursor.execute("TRUNCATE catalogue_book_ancestor")
 938             cursor.execute("""
 939                 WITH RECURSIVE ancestry AS (
 940                     SELECT book.id, book.parent_id
 941                     FROM catalogue_book AS book
 942                     WHERE book.parent_id IS NOT NULL
 943                     UNION
 944                     SELECT ancestor.id, book.parent_id
 945                     FROM ancestry AS ancestor, catalogue_book AS book
 946                     WHERE ancestor.parent_id = book.id
 947                         AND book.parent_id IS NOT NULL
 948                     )
 949                 INSERT INTO catalogue_book_ancestor
 950                     (from_book_id, to_book_id)
 951                     SELECT id, parent_id
 952                     FROM ancestry
 953                     ORDER BY id;
 954                 """)
 955         else:
 956             cursor.execute("DELETE FROM catalogue_book_ancestor")
 957             for b in cls.objects.exclude(parent=None):
 958                 parent = b.parent
 959                 while parent is not None:
 960                     b.ancestor.add(parent)
 961                     parent = parent.parent
 962
 963     @property
 964     def ancestors(self):
 965         if self.parent:
 966             for anc in self.parent.ancestors:
 967                 yield anc
 968             yield self.parent
 969         else:
 970             return []
 971
 972     def clear_cache(self):
 973         clear_cached_renders(self.mini_box)
 974         clear_cached_renders(self.mini_box_nolink)
 975
 976     def cover_info(self, inherit=True):
 977         """Returns a dictionary to serve as fallback for BookInfo.
 978
 979         For now, the only thing inherited is the cover image.
 980         """
 981         need = False
 982         info = {}
 983         for field in ('cover_url', 'cover_by', 'cover_source'):
 984             val = self.get_extra_info_json().get(field)
 985             if val:
 986                 info[field] = val
 987             else:
 988                 need = True
 989         if inherit and need and self.parent is not None:
 990             parent_info = self.parent.cover_info()
 991             parent_info.update(info)
 992             info = parent_info
 993         return info
 994
 995     def related_themes(self):
 996         return Tag.objects.usage_for_queryset(
 997             Fragment.objects.filter(models.Q(book=self) | models.Q(book__ancestor=self)),
 998             counts=True).filter(category='theme').order_by('-count')
 999
1000     def parent_cover_changed(self):
1001         """Called when parent book's cover image is changed."""
1002         if not self.cover_info(inherit=False):
1003             if 'cover' not in app_settings.DONT_BUILD:
1004                 self.cover.build_delay()
1005                 self.cover_clean.build_delay()
1006                 self.cover_thumb.build_delay()
1007                 self.cover_api_thumb.build_delay()
1008                 self.simple_cover.build_delay()
1009                 self.cover_ebookpoint.build_delay()
1010             for format_ in constants.EBOOK_FORMATS_WITH_COVERS:
1011                 if format_ not in app_settings.DONT_BUILD:
1012                     getattr(self, '%s_file' % format_).build_delay()
1013             for child in self.children.all():
1014                 child.parent_cover_changed()
1015
1016     def other_versions(self):
1017         """Find other versions (i.e. in other languages) of the book."""
1018         return type(self).objects.filter(common_slug=self.common_slug, findable=True).exclude(pk=self.pk)
1019
1020     def parents(self):
1021         books = []
1022         parent = self.parent
1023         while parent is not None:
1024             books.insert(0, parent)
1025             parent = parent.parent
1026         return books
1027
1028     def pretty_title(self, html_links=False):
1029         names = [(tag.name, tag.get_absolute_url()) for tag in self.authors().only('name', 'category', 'slug')]
1030         books = self.parents() + [self]
1031         names.extend([(b.title, b.get_absolute_url()) for b in books])
1032
1033         if html_links:
1034             names = ['<a href="%s">%s</a>' % (tag[1], tag[0]) for tag in names]
1035         else:
1036             names = [tag[0] for tag in names]
1037         return ', '.join(names)
1038
1039     def publisher(self):
1040         publisher = self.get_extra_info_json()['publisher']
1041         if isinstance(publisher, str):
1042             return publisher
1043         elif isinstance(publisher, list):
1044             return ', '.join(publisher)
1045
1046     def get_recommended(self, limit=4):
1047         books_qs = type(self).objects.filter(findable=True)
1048         books_qs = books_qs.exclude(common_slug=self.common_slug).exclude(ancestor=self)
1049         books = type(self).tagged.related_to(self, books_qs)[:limit]
1050         return books
1051
1052     @classmethod
1053     def tagged_top_level(cls, tags):
1054         """ Returns top-level books tagged with `tags`.
1055
1056         It only returns those books which don't have ancestors which are
1057         also tagged with those tags.
1058
1059         """
1060         objects = cls.tagged.with_all(tags)
1061         return objects.filter(findable=True).exclude(ancestor__in=objects)
1062
1063     @classmethod
1064     def book_list(cls, book_filter=None):
1065         """Generates a hierarchical listing of all books.
1066
1067         Books are optionally filtered with a test function.
1068
1069         """
1070
1071         books_by_parent = {}
1072         books = cls.objects.filter(findable=True).order_by('parent_number', 'sort_key').only('title', 'parent', 'slug', 'extra_info')
1073         if book_filter:
1074             books = books.filter(book_filter).distinct()
1075
1076             book_ids = set(b['pk'] for b in books.values("pk").iterator())
1077             for book in books.iterator():
1078                 parent = book.parent_id
1079                 if parent not in book_ids:
1080                     parent = None
1081                 books_by_parent.setdefault(parent, []).append(book)
1082         else:
1083             for book in books.iterator():
1084                 books_by_parent.setdefault(book.parent_id, []).append(book)
1085
1086         orphans = []
1087         books_by_author = OrderedDict()
1088         for tag in Tag.objects.filter(category='author').iterator():
1089             books_by_author[tag] = []
1090
1091         for book in books_by_parent.get(None, ()):
1092             authors = list(book.authors().only('pk'))
1093             if authors:
1094                 for author in authors:
1095                     books_by_author[author].append(book)
1096             else:
1097                 orphans.append(book)
1098
1099         return books_by_author, orphans, books_by_parent
1100
1101     _audiences_pl = {
1102         "SP": (1, "szkoła podstawowa"),
1103         "SP1": (1, "szkoła podstawowa"),
1104         "SP2": (1, "szkoła podstawowa"),
1105         "SP3": (1, "szkoła podstawowa"),
1106         "P": (1, "szkoła podstawowa"),
1107         "G": (2, "gimnazjum"),
1108         "L": (3, "liceum"),
1109         "LP": (3, "liceum"),
1110     }
1111
1112     def audiences_pl(self):
1113         audiences = self.get_extra_info_json().get('audiences', [])
1114         audiences = sorted(set([self._audiences_pl.get(a, (99, a)) for a in audiences]))
1115         return [a[1] for a in audiences]
1116
1117     def stage_note(self):
1118         stage = self.get_extra_info_json().get('stage')
1119         if stage and stage < '0.4':
1120             return (_('Ten utwór wymaga uwspółcześnienia'),
1121                     reverse('infopage', args=['wymagajace-uwspolczesnienia']))
1122         else:
1123             return None, None
1124
1125     def choose_fragments(self, number):
1126         fragments = self.fragments.order_by()
1127         fragments_count = fragments.count()
1128         if not fragments_count and self.children.exists():
1129             fragments = Fragment.objects.filter(book__ancestor=self).order_by()
1130             fragments_count = fragments.count()
1131         if fragments_count:
1132             if fragments_count > number:
1133                 offset = randint(0, fragments_count - number)
1134             else:
1135                 offset = 0
1136             return fragments[offset : offset + number]
1137         elif self.parent:
1138             return self.parent.choose_fragments(number)
1139         else:
1140             return []
1141
1142     def choose_fragment(self):
1143         fragments = self.choose_fragments(1)
1144         if fragments:
1145             return fragments[0]
1146         else:
1147             return None
1148
1149     def fragment_data(self):
1150         fragment = self.choose_fragment()
1151         if fragment:
1152             return {
1153                 'title': fragment.book.pretty_title(),
1154                 'html': re.sub('</?blockquote[^>]*>', '', fragment.get_short_text()),
1155             }
1156         else:
1157             return None
1158
1159     def update_popularity(self):
1160         count = self.userlistitem_set.values('list__user').order_by('list__user').distinct().count()
1161         try:
1162             pop = self.popularity
1163             pop.count = count
1164             pop.save()
1165         except BookPopularity.DoesNotExist:
1166             BookPopularity.objects.create(book=self, count=count)
1167
1168     def ridero_link(self):
1169         return 'https://ridero.eu/%s/books/wl_%s/' % (get_language(), self.slug.replace('-', '_'))
1170
1171     def elevenreader_link(self):
1172         first_text = self.get_first_text()
1173         if first_text is None:
1174             return None
1175         return 'https://elevenreader.io/audiobooks/wolnelektury:' + first_text.slug
1176
1177     def content_warnings(self):
1178         warnings_def = {
1179             'wulgaryzmy': _('wulgaryzmy'),
1180         }
1181         warnings = self.get_extra_info_json().get('content_warnings', [])
1182         warnings = [
1183             warnings_def.get(w, w)
1184             for w in warnings
1185         ]
1186         warnings.sort()
1187         return warnings
1188
1189     def full_sort_key(self):
1190         return self.SORT_KEY_SEP.join((self.sort_key_author, self.sort_key, str(self.id)))
1191
1192     def cover_color(self):
1193         return WLCover.epoch_colors.get(self.get_extra_info_json().get('epoch'), '#000000')
1194
1195     @cached_render('catalogue/book_mini_box.html')
1196     def mini_box(self):
1197         return {
1198             'book': self
1199         }
1200
1201     @cached_render('catalogue/book_mini_box.html')
1202     def mini_box_nolink(self):
1203         return {
1204             'book': self,
1205             'no_link': True,
1206         }
1207
1208
1209 class BookPopularity(models.Model):
1210     book = models.OneToOneField(Book, models.CASCADE, related_name='popularity')
1211     count = models.IntegerField(default=0, db_index=True)