fnp / wolnelektury.git / blobdiff
? search:
summary | shortlog | log | commit | commitdiff | tree
raw | inline | side by side
remove the banner
[wolnelektury.git] / src / catalogue / models / book.py
diff --git a/src/catalogue/models/book.py b/src/catalogue/models/book.py
index 5a0f169..9e0ec50 100644 (file)
--- a/src/catalogue/models/book.py
+++ b/src/catalogue/models/book.py
@@ -1,5 +1,5 @@
-# This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
-# Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
+# This file is part of Wolne Lektury, licensed under GNU Affero GPLv3 or later.
+# Copyright © Fundacja Wolne Lektury. See NOTICE for more information.
 #
 from collections import OrderedDict
 import json
 #
 from collections import OrderedDict
 import json
@@ -8,107 +8,76 @@ from random import randint
 import os.path
 import re
 from urllib.request import urlretrieve
 import os.path
 import re
 from urllib.request import urlretrieve
+from django.apps import apps
 from django.conf import settings
 from django.db import connection, models, transaction
 import django.dispatch
 from django.contrib.contenttypes.fields import GenericRelation
 from django.conf import settings
 from django.db import connection, models, transaction
 import django.dispatch
 from django.contrib.contenttypes.fields import GenericRelation
+from django.template.loader import render_to_string
 from django.urls import reverse
 from django.urls import reverse
-from django.utils.translation import ugettext_lazy as _, get_language
-from django.utils.deconstruct import deconstructible
+from django.utils.translation import gettext_lazy as _, get_language
 from fnpdjango.storage import BofhFileSystemStorage
 from fnpdjango.storage import BofhFileSystemStorage
-
+from lxml import html
 from librarian.cover import WLCover
 from librarian.html import transform_abstrakt
 from librarian.cover import WLCover
 from librarian.html import transform_abstrakt
+from librarian.builders import builders
 from newtagging import managers
 from catalogue import constants
 from newtagging import managers
 from catalogue import constants
-from catalogue.fields import EbookField
+from catalogue import fields
 from catalogue.models import Tag, Fragment, BookMedia
 from catalogue.utils import create_zip, gallery_url, gallery_path, split_tags, get_random_hash
 from catalogue.models.tag import prefetched_relations
 from catalogue import app_settings
 from catalogue.models import Tag, Fragment, BookMedia
 from catalogue.utils import create_zip, gallery_url, gallery_path, split_tags, get_random_hash
 from catalogue.models.tag import prefetched_relations
 from catalogue import app_settings
-from catalogue import tasks
 from wolnelektury.utils import makedirs, cached_render, clear_cached_renders
 
 bofh_storage = BofhFileSystemStorage()
 
 
 from wolnelektury.utils import makedirs, cached_render, clear_cached_renders
 
 bofh_storage = BofhFileSystemStorage()
 
 
-@deconstructible
-class UploadToPath(object):
-    def __init__(self, path):
-        self.path = path
-
-    def __call__(self, instance, filename):
-        return self.path % instance.slug
-
-
-_cover_upload_to = UploadToPath('book/cover/%s.jpg')
-_cover_thumb_upload_to = UploadToPath('book/cover_thumb/%s.jpg')
-_cover_api_thumb_upload_to = UploadToPath('book/cover_api_thumb/%s.jpg')
-_simple_cover_upload_to = UploadToPath('book/cover_simple/%s.jpg')
-_cover_ebookpoint_upload_to = UploadToPath('book/cover_ebookpoint/%s.jpg')
-
-
-def _ebook_upload_to(upload_path):
-    return UploadToPath(upload_path)
-
-
 class Book(models.Model):
     """Represents a book imported from WL-XML."""
 class Book(models.Model):
     """Represents a book imported from WL-XML."""
-    title = models.CharField(_('title'), max_length=32767)
-    sort_key = models.CharField(_('sort key'), max_length=120, db_index=True, editable=False)
+    title = models.CharField('tytuł', max_length=32767)
+    sort_key = models.CharField('klucz sortowania', max_length=120, db_index=True, editable=False)
     sort_key_author = models.CharField(
     sort_key_author = models.CharField(
-        _('sort key by author'), max_length=120, db_index=True, editable=False, default='')
-    slug = models.SlugField(_('slug'), max_length=120, db_index=True, unique=True)
-    common_slug = models.SlugField(_('slug'), max_length=120, db_index=True)
-    language = models.CharField(_('language code'), max_length=3, db_index=True, default=app_settings.DEFAULT_LANGUAGE)
-    description = models.TextField(_('description'), blank=True)
-    abstract = models.TextField(_('abstract'), blank=True)
-    created_at = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True)
-    changed_at = models.DateTimeField(_('change date'), auto_now=True, db_index=True)
-    parent_number = models.IntegerField(_('parent number'), default=0)
-    extra_info = models.TextField(_('extra information'), default='{}')
+        'klucz sortowania wg autora', max_length=120, db_index=True, editable=False, default='')
+    slug = models.SlugField('slug', max_length=120, db_index=True, unique=True)
+    common_slug = models.SlugField('wspólny slug', max_length=120, db_index=True)
+    language = models.CharField('kod języka', max_length=3, db_index=True, default=app_settings.DEFAULT_LANGUAGE)
+    description = models.TextField('opis', blank=True)
+    abstract = models.TextField('abstrakt', blank=True)
+    toc = models.TextField('spis treści', blank=True)
+    created_at = models.DateTimeField('data utworzenia', auto_now_add=True, db_index=True)
+    changed_at = models.DateTimeField('data motyfikacji', auto_now=True, db_index=True)
+    parent_number = models.IntegerField('numer w ramach rodzica', default=0)
+    extra_info = models.TextField('dodatkowe informacje', default='{}')
     gazeta_link = models.CharField(blank=True, max_length=240)
     wiki_link = models.CharField(blank=True, max_length=240)
     gazeta_link = models.CharField(blank=True, max_length=240)
     wiki_link = models.CharField(blank=True, max_length=240)
-    print_on_demand = models.BooleanField(_('print on demand'), default=False)
-    recommended = models.BooleanField(_('recommended'), default=False)
-    audio_length = models.CharField(_('audio length'), blank=True, max_length=8)
-    preview = models.BooleanField(_('preview'), default=False)
-    preview_until = models.DateField(_('preview until'), blank=True, null=True)
+    print_on_demand = models.BooleanField('druk na żądanie', default=False)
+    recommended = models.BooleanField('polecane', default=False)
+    audio_length = models.CharField('długość audio', blank=True, max_length=8)
+    preview = models.BooleanField('prapremiera', default=False)
+    preview_until = models.DateField('prapremiera do', blank=True, null=True)
     preview_key = models.CharField(max_length=32, blank=True, null=True)
     preview_key = models.CharField(max_length=32, blank=True, null=True)
-    findable = models.BooleanField(_('findable'), default=True, db_index=True)
+    findable = models.BooleanField('wyszukiwalna', default=True, db_index=True)
 
     # files generated during publication
 
     # files generated during publication
-    cover = EbookField(
-        'cover', _('cover'),
-        null=True, blank=True,
-        upload_to=_cover_upload_to,
-        storage=bofh_storage, max_length=255)
-    cover_etag = models.CharField(max_length=255, editable=False, default='', db_index=True)
+    xml_file = fields.XmlField(storage=bofh_storage, with_etag=False)
+    html_file = fields.HtmlField(storage=bofh_storage)
+    fb2_file = fields.Fb2Field(storage=bofh_storage)
+    txt_file = fields.TxtField(storage=bofh_storage)
+    epub_file = fields.EpubField(storage=bofh_storage)
+    mobi_file = fields.MobiField(storage=bofh_storage)
+    pdf_file = fields.PdfField(storage=bofh_storage)
+
+    cover = fields.CoverField('okładka', storage=bofh_storage)
     # Cleaner version of cover for thumbs
     # Cleaner version of cover for thumbs
-    cover_thumb = EbookField(
-        'cover_thumb', _('cover thumbnail'),
-        null=True, blank=True,
-        upload_to=_cover_thumb_upload_to,
-        max_length=255)
-    cover_thumb_etag = models.CharField(max_length=255, editable=False, default='', db_index=True)
-    cover_api_thumb = EbookField(
-        'cover_api_thumb', _('cover thumbnail for mobile app'),
-        null=True, blank=True,
-        upload_to=_cover_api_thumb_upload_to,
-        max_length=255)
-    cover_api_thumb_etag = models.CharField(max_length=255, editable=False, default='', db_index=True)
-    simple_cover = EbookField(
-        'simple_cover', _('cover for mobile app'),
-        null=True, blank=True,
-        upload_to=_simple_cover_upload_to,
-        max_length=255)
-    simple_cover_etag = models.CharField(max_length=255, editable=False, default='', db_index=True)
-    cover_ebookpoint = EbookField(
-        'cover_ebookpoint', _('cover for Ebookpoint'),
-        null=True, blank=True,
-        upload_to=_cover_ebookpoint_upload_to,
-        max_length=255)
-    cover_ebookpoint_etag = models.CharField(max_length=255, editable=False, default='', db_index=True)
+    cover_clean = fields.CoverCleanField('czysta okładka')
+    cover_thumb = fields.CoverThumbField('miniatura okładki')
+    cover_api_thumb = fields.CoverApiThumbField(
+        'mniaturka okładki dla aplikacji')
+    simple_cover = fields.SimpleCoverField('okładka dla aplikacji')
+    cover_ebookpoint = fields.CoverEbookpointField(
+        'okładka dla Ebookpoint')
+
     ebook_formats = constants.EBOOK_FORMATS
     formats = ebook_formats + ['html', 'xml']
 
     ebook_formats = constants.EBOOK_FORMATS
     formats = ebook_formats + ['html', 'xml']
 
@@ -122,19 +91,22 @@ class Book(models.Model):
     tagged = managers.ModelTaggedItemManager(Tag)
     tags = managers.TagDescriptor(Tag)
     tag_relations = GenericRelation(Tag.intermediary_table_model)
     tagged = managers.ModelTaggedItemManager(Tag)
     tags = managers.TagDescriptor(Tag)
     tag_relations = GenericRelation(Tag.intermediary_table_model)
+    translators = models.ManyToManyField(Tag)
 
     html_built = django.dispatch.Signal()
     published = django.dispatch.Signal()
 
     SORT_KEY_SEP = '$'
 
 
     html_built = django.dispatch.Signal()
     published = django.dispatch.Signal()
 
     SORT_KEY_SEP = '$'
 
+    is_book = True
+
     class AlreadyExists(Exception):
         pass
 
     class Meta:
         ordering = ('sort_key_author', 'sort_key')
     class AlreadyExists(Exception):
         pass
 
     class Meta:
         ordering = ('sort_key_author', 'sort_key')
-        verbose_name = _('book')
-        verbose_name_plural = _('books')
+        verbose_name = 'książka'
+        verbose_name_plural = 'książki'
         app_label = 'catalogue'
 
     def __str__(self):
         app_label = 'catalogue'
 
     def __str__(self):
@@ -183,12 +155,6 @@ class Book(models.Model):
     def genre_unicode(self):
         return self.tag_unicode('genre')
 
     def genre_unicode(self):
         return self.tag_unicode('genre')
 
-    def translators(self):
-        translators = self.get_extra_info_json().get('translators') or []
-        return [
-            '\xa0'.join(reversed(translator.split(', ', 1))) for translator in translators
-        ]
-
     def translator(self):
         translators = self.get_extra_info_json().get('translators')
         if not translators:
     def translator(self):
         translators = self.get_extra_info_json().get('translators')
         if not translators:
@@ -215,6 +181,18 @@ class Book(models.Model):
     def isbn_mobi(self):
         return self.get_extra_info_json().get('isbn_mobi')
 
     def isbn_mobi(self):
         return self.get_extra_info_json().get('isbn_mobi')
 
+    def is_accessible_to(self, user):
+        if not self.preview:
+            return True
+        if not user.is_authenticated:
+            return False
+        Membership = apps.get_model('club', 'Membership')
+        if Membership.is_active_for(user):
+            return True
+        Funding = apps.get_model('funding', 'Funding')
+        if Funding.objects.filter(user=user, offer__book=self):
+            return True
+        return False
 
     def save(self, force_insert=False, force_update=False, **kwargs):
         from sortify import sortify
 
     def save(self, force_insert=False, force_update=False, **kwargs):
         from sortify import sortify
@@ -267,21 +245,44 @@ class Book(models.Model):
         sibling = self.parent.children.filter(parent_number__lt=self.parent_number).order_by('-parent_number').first()
         if sibling is not None:
             return sibling.get_last_text()
         sibling = self.parent.children.filter(parent_number__lt=self.parent_number).order_by('-parent_number').first()
         if sibling is not None:
             return sibling.get_last_text()
+
+        if self.parent.html_file:
+            return self.parent
+
         return self.parent.get_prev_text()
 
         return self.parent.get_prev_text()
 
-    def get_next_text(self):
+    def get_next_text(self, inside=True):
+        if inside:
+            child = self.children.order_by('parent_number').first()
+            if child is not None:
+                return child.get_first_text()
+
         if not self.parent:
             return None
         sibling = self.parent.children.filter(parent_number__gt=self.parent_number).order_by('parent_number').first()
         if sibling is not None:
             return sibling.get_first_text()
         if not self.parent:
             return None
         sibling = self.parent.children.filter(parent_number__gt=self.parent_number).order_by('parent_number').first()
         if sibling is not None:
             return sibling.get_first_text()
-        return self.parent.get_next_text()
+        return self.parent.get_next_text(inside=False)
+
+    def get_child_audiobook(self):
+        BookMedia = apps.get_model('catalogue', 'BookMedia')
+        if not BookMedia.objects.filter(book__ancestor=self).exists():
+            return None
+        for child in self.children.order_by('parent_number').all():
+            if child.has_mp3_file():
+                return child
+            child_sub = child.get_child_audiobook()
+            if child_sub is not None:
+                return child_sub
 
     def get_siblings(self):
         if not self.parent:
             return []
         return self.parent.children.all().order_by('parent_number')
 
 
     def get_siblings(self):
         if not self.parent:
             return []
         return self.parent.children.all().order_by('parent_number')
 
+    def get_children(self):
+        return self.children.all().order_by('parent_number')
+
     @property
     def name(self):
         return self.title
     @property
     def name(self):
         return self.title
@@ -325,6 +326,9 @@ class Book(models.Model):
             total += app_settings.GET_MP3_LENGTH(media.file.path)
         return int(total)
 
             total += app_settings.GET_MP3_LENGTH(media.file.path)
         return int(total)
 
+    def get_time(self):
+        return round(self.xml_file.size / 1000 * 40)
+    
     def has_media(self, type_):
         if type_ in Book.formats:
             return bool(getattr(self, "%s_file" % type_))
     def has_media(self, type_):
         if type_ in Book.formats:
             return bool(getattr(self, "%s_file" % type_))
@@ -355,6 +359,9 @@ class Book(models.Model):
     def get_daisy(self):
         return self.get_media("daisy")
 
     def get_daisy(self):
         return self.get_media("daisy")
 
+    def get_audio_epub(self):
+        return self.get_media("audio.epub")
+
     def media_url(self, format_):
         media = self.get_media(format_)
         if media:
     def media_url(self, format_):
         media = self.get_media(format_)
         if media:
@@ -388,7 +395,7 @@ class Book(models.Model):
 
     def has_description(self):
         return len(self.description) > 0
 
     def has_description(self):
         return len(self.description) > 0
-    has_description.short_description = _('description')
+    has_description.short_description = 'opis'
     has_description.boolean = True
 
     def has_mp3_file(self):
     has_description.boolean = True
 
     def has_mp3_file(self):
@@ -406,6 +413,32 @@ class Book(models.Model):
     has_daisy_file.short_description = 'DAISY'
     has_daisy_file.boolean = True
 
     has_daisy_file.short_description = 'DAISY'
     has_daisy_file.boolean = True
 
+    def has_sync_file(self):
+        return self.has_media("sync")
+
+    def get_sync(self):
+        with self.get_media('sync').first().file.open('r') as f:
+            sync = f.read().split('\n')
+        offset = float(sync[0])
+        items = []
+        for line in sync[1:]:
+            if not line:
+                continue
+            start, end, elid = line.split()
+            items.append([elid, float(start) + offset])
+        return json.dumps(items)
+    
+    def has_audio_epub_file(self):
+        return self.has_media("audio.epub")
+
+    @property
+    def media_daisy(self):
+        return self.get_media('daisy')
+
+    @property
+    def media_audio_epub(self):
+        return self.get_media('audio.epub')
+
     def get_audiobooks(self):
         ogg_files = {}
         for m in self.media.filter(type='ogg').order_by().iterator():
     def get_audiobooks(self):
         ogg_files = {}
         for m in self.media.filter(type='ogg').order_by().iterator():
@@ -413,6 +446,7 @@ class Book(models.Model):
 
         audiobooks = []
         projects = set()
 
         audiobooks = []
         projects = set()
+        total_duration = 0
         for mp3 in self.media.filter(type='mp3').iterator():
             # ogg files are always from the same project
             meta = mp3.get_extra_info_json()
         for mp3 in self.media.filter(type='mp3').iterator():
             # ogg files are always from the same project
             meta = mp3.get_extra_info_json()
@@ -422,6 +456,7 @@ class Book(models.Model):
                 project = 'CzytamySłuchając'
 
             projects.add((project, meta.get('funded_by', '')))
                 project = 'CzytamySłuchając'
 
             projects.add((project, meta.get('funded_by', '')))
+            total_duration += mp3.duration or 0
 
             media = {'mp3': mp3}
 
 
             media = {'mp3': mp3}
 
@@ -431,7 +466,11 @@ class Book(models.Model):
             audiobooks.append(media)
 
         projects = sorted(projects)
             audiobooks.append(media)
 
         projects = sorted(projects)
-        return audiobooks, projects
+        total_duration = '%d:%02d' % (
+            total_duration // 60,
+            total_duration % 60
+        )
+        return audiobooks, projects, total_duration
 
     def wldocument(self, parse_dublincore=True, inherit=True):
         from catalogue.import_utils import ORMDocProvider
 
     def wldocument(self, parse_dublincore=True, inherit=True):
         from catalogue.import_utils import ORMDocProvider
@@ -448,6 +487,17 @@ class Book(models.Model):
             parse_dublincore=parse_dublincore,
             meta_fallbacks=meta_fallbacks)
 
             parse_dublincore=parse_dublincore,
             meta_fallbacks=meta_fallbacks)
 
+    def wldocument2(self):
+        from catalogue.import_utils import ORMDocProvider
+        from librarian.document import WLDocument
+        doc = WLDocument(
+            self.xml_file.path,
+            provider=ORMDocProvider(self)
+        )
+        doc.meta.update(self.cover_info())
+        return doc
+
+
     @staticmethod
     def zip_format(format_):
         def pretty_file_name(book):
     @staticmethod
     def zip_format(format_):
         def pretty_file_name(book):
@@ -457,33 +507,36 @@ class Book(models.Model):
                 format_)
 
         field_name = "%s_file" % format_
                 format_)
 
         field_name = "%s_file" % format_
+        field = getattr(Book, field_name)
         books = Book.objects.filter(parent=None).exclude(**{field_name: ""}).exclude(preview=True).exclude(findable=False)
         paths = [(pretty_file_name(b), getattr(b, field_name).path) for b in books.iterator()]
         books = Book.objects.filter(parent=None).exclude(**{field_name: ""}).exclude(preview=True).exclude(findable=False)
         paths = [(pretty_file_name(b), getattr(b, field_name).path) for b in books.iterator()]
-        return create_zip(paths, app_settings.FORMAT_ZIPS[format_])
+        return create_zip(paths, field.ZIP)
 
     def zip_audiobooks(self, format_):
         bm = BookMedia.objects.filter(book=self, type=format_)
 
     def zip_audiobooks(self, format_):
         bm = BookMedia.objects.filter(book=self, type=format_)
-        paths = map(lambda bm: (None, bm.file.path), bm)
-        return create_zip(paths, "%s_%s" % (self.slug, format_))
-
-    def search_index(self, book_info=None, index=None, index_tags=True, commit=True):
+        paths = map(lambda bm: (bm.get_nice_filename(), bm.file.path), bm)
+        licenses = set()
+        for m in bm:
+            license = constants.LICENSES.get(
+                m.get_extra_info_json().get('license'), {}
+            ).get('locative')
+            if license:
+                licenses.add(license)
+        readme = render_to_string('catalogue/audiobook_zip_readme.txt', {
+            'licenses': licenses,
+            'meta': self.wldocument2().meta,
+        })
+        return create_zip(paths, "%s_%s" % (self.slug, format_), {'informacje.txt': readme})
+
+    def search_index(self, index=None):
         if not self.findable:
             return
         if not self.findable:
             return
-        if index is None:
-            from search.index import Index
-            index = Index()
-        try:
-            index.index_book(self, book_info)
-            if index_tags:
-                index.index_tags()
-            if commit:
-                index.index.commit()
-        except Exception as e:
-            index.index.rollback()
-            raise e
+        from search.index import Index
+        Index.index_book(self)
 
     # will make problems in conjunction with paid previews
     def download_pictures(self, remote_gallery_url):
 
     # will make problems in conjunction with paid previews
     def download_pictures(self, remote_gallery_url):
+        # This is only needed for legacy relative image paths.
         gallery_path = self.gallery_path()
         # delete previous files, so we don't include old files in ebooks
         if os.path.isdir(gallery_path):
         gallery_path = self.gallery_path()
         # delete previous files, so we don't include old files in ebooks
         if os.path.isdir(gallery_path):
@@ -495,6 +548,8 @@ class Book(models.Model):
             makedirs(gallery_path)
             for ilustr in ilustr_elements:
                 ilustr_src = ilustr.get('src')
             makedirs(gallery_path)
             for ilustr in ilustr_elements:
                 ilustr_src = ilustr.get('src')
+                if '/' in ilustr_src:
+                    continue
                 ilustr_path = os.path.join(gallery_path, ilustr_src)
                 urlretrieve('%s/%s' % (remote_gallery_url, ilustr_src), ilustr_path)
 
                 ilustr_path = os.path.join(gallery_path, ilustr_src)
                 urlretrieve('%s/%s' % (remote_gallery_url, ilustr_src), ilustr_path)
 
@@ -505,6 +560,20 @@ class Book(models.Model):
         else:
             self.abstract = ''
 
         else:
             self.abstract = ''
 
+    def load_toc(self):
+        self.toc = ''
+        if self.html_file:
+            parser = html.HTMLParser(encoding='utf-8')
+            tree = html.parse(self.html_file.path, parser=parser)
+            toc = tree.find('//div[@id="toc"]/ol')
+            if toc is None or not len(toc):
+                return
+            html_link = reverse('book_text', args=[self.slug])
+            for a in toc.findall('.//a'):
+                a.attrib['href'] = html_link + a.attrib['href']
+            self.toc = html.tostring(toc, encoding='unicode')
+            # div#toc
+
     @classmethod
     def from_xml_file(cls, xml_file, **kwargs):
         from django.core.files import File
     @classmethod
     def from_xml_file(cls, xml_file, **kwargs):
         from django.core.files import File
@@ -523,7 +592,9 @@ class Book(models.Model):
 
     @classmethod
     def from_text_and_meta(cls, raw_file, book_info, overwrite=False, dont_build=None, search_index=True,
 
     @classmethod
     def from_text_and_meta(cls, raw_file, book_info, overwrite=False, dont_build=None, search_index=True,
-                           search_index_tags=True, remote_gallery_url=None, days=0, findable=True):
+                           remote_gallery_url=None, days=0, findable=True, logo=None, logo_mono=None, logo_alt=None):
+        from catalogue import tasks
+
         if dont_build is None:
             dont_build = set()
         dont_build = set.union(set(dont_build), set(app_settings.DONT_BUILD))
         if dont_build is None:
             dont_build = set()
         dont_build = set.union(set(dont_build), set(app_settings.DONT_BUILD))
@@ -535,7 +606,7 @@ class Book(models.Model):
                 try:
                     children.append(Book.objects.get(slug=part_url.slug))
                 except Book.DoesNotExist:
                 try:
                     children.append(Book.objects.get(slug=part_url.slug))
                 except Book.DoesNotExist:
-                    raise Book.DoesNotExist(_('Book "%s" does not exist.') % part_url.slug)
+                    raise Book.DoesNotExist('Książka "%s" nie istnieje.' % part_url.slug)
 
         # Read book metadata
         book_slug = book_info.url.slug
 
         # Read book metadata
         book_slug = book_info.url.slug
@@ -551,7 +622,7 @@ class Book(models.Model):
                 book.preview_until = date.today() + timedelta(days)
         else:
             if not overwrite:
                 book.preview_until = date.today() + timedelta(days)
         else:
             if not overwrite:
-                raise Book.AlreadyExists(_('Book %s already exists') % book_slug)
+                raise Book.AlreadyExists('Książka %s już istnieje' % book_slug)
             # Save shelves for this book
             book_shelves = list(book.tags.filter(category='set'))
             old_cover = book.cover_info()
             # Save shelves for this book
             book_shelves = list(book.tags.filter(category='set'))
             old_cover = book.cover_info()
@@ -568,20 +639,31 @@ class Book(models.Model):
             book.common_slug = book_info.variant_of.slug
         else:
             book.common_slug = book.slug
             book.common_slug = book_info.variant_of.slug
         else:
             book.common_slug = book.slug
-        book.extra_info = json.dumps(book_info.to_dict())
+        extra = book_info.to_dict()
+        if logo:
+            extra['logo'] = logo
+        if logo_mono:
+            extra['logo_mono'] = logo_mono
+        if logo_alt:
+            extra['logo_alt'] = logo_alt
+        book.extra_info = json.dumps(extra)
         book.load_abstract()
         book.load_abstract()
+        book.load_toc()
         book.save()
 
         meta_tags = Tag.tags_from_info(book_info)
 
         book.save()
 
         meta_tags = Tag.tags_from_info(book_info)
 
-        for tag in meta_tags:
+        for tag, relationship in meta_tags:
             if not tag.for_books:
                 tag.for_books = True
                 tag.save()
 
             if not tag.for_books:
                 tag.for_books = True
                 tag.save()
 
-        book.tags = set(meta_tags + book_shelves)
+        just_tags = [t for (t, rel) in meta_tags if not rel]
+        book.tags = set(just_tags + book_shelves)
         book.save()  # update sort_key_author
 
         book.save()  # update sort_key_author
 
+        book.translators.set([t for (t, rel) in meta_tags if rel == 'translator'])
+
         cover_changed = old_cover != book.cover_info()
         obsolete_children = set(b for b in book.children.all()
                                 if b not in children)
         cover_changed = old_cover != book.cover_info()
         obsolete_children = set(b for b in book.children.all()
                                 if b not in children)
@@ -612,6 +694,7 @@ class Book(models.Model):
         # Build cover.
         if 'cover' not in dont_build:
             book.cover.build_delay()
         # Build cover.
         if 'cover' not in dont_build:
             book.cover.build_delay()
+            book.cover_clean.build_delay()
             book.cover_thumb.build_delay()
             book.cover_api_thumb.build_delay()
             book.simple_cover.build_delay()
             book.cover_thumb.build_delay()
             book.cover_api_thumb.build_delay()
             book.simple_cover.build_delay()
@@ -628,7 +711,7 @@ class Book(models.Model):
                 getattr(book, '%s_file' % format_).build_delay()
 
         if not settings.NO_SEARCH_INDEX and search_index and findable:
                 getattr(book, '%s_file' % format_).build_delay()
 
         if not settings.NO_SEARCH_INDEX and search_index and findable:
-            tasks.index_book.delay(book.id, book_info=book_info, index_tags=search_index_tags)
+            tasks.index_book.delay(book.id)
 
         for child in notify_cover_changed:
             child.parent_cover_changed()
 
         for child in notify_cover_changed:
             child.parent_cover_changed()
@@ -639,44 +722,51 @@ class Book(models.Model):
         cls.published.send(sender=cls, instance=book)
         return book
 
         cls.published.send(sender=cls, instance=book)
         return book
 
-    def get_master(self):
-        master_tags = [
-            'opowiadanie',
-            'powiesc',
-            'dramat_wierszowany_l',
-            'dramat_wierszowany_lp',
-            'dramat_wspolczesny', 'liryka_l', 'liryka_lp',
-            'wywiad',
-        ]
-        from librarian.parser import WLDocument
-        wld = WLDocument.from_file(self.xml_file.path, parse_dublincore=False)
-        root = wld.edoc.getroot()
-        for master in root.iter():
-            if master.tag in master_tags:
-                return master
-    
     def update_references(self):
     def update_references(self):
-        from references.models import Entity, Reference
-        master = self.get_master()
-        found = set()
-        for i, sec in enumerate(master):
-            for ref in sec.findall('.//ref'):
-                href = ref.attrib.get('href', '')
-                if not href or href in found:
-                    continue
-                found.add(href)
-                entity, created = Entity.objects.get_or_create(
-                    uri=href
-                )
-                ref, created = Reference.objects.get_or_create(
-                    book=self,
-                    entity=entity
-                )
-                ref.first_section = 'sec%d' % (i + 1)
-                entity.populate()
-                entity.save()
-        Reference.objects.filter(book=self).exclude(entity__uri__in=found).delete()
-    
+        Entity = apps.get_model('references', 'Entity')
+        doc = self.wldocument2()
+        doc._compat_assign_section_ids()
+        doc._compat_assign_ordered_ids()
+        refs = {}
+        for ref_elem in doc.references():
+            uri = ref_elem.attrib.get('href', '')
+            if not uri:
+                continue
+            if uri in refs:
+                ref = refs[uri]
+            else:
+                entity, entity_created = Entity.objects.get_or_create(uri=uri)
+                if entity_created:
+                    try:
+                        entity.populate()
+                    except:
+                        pass
+                    else:
+                        entity.save()
+                ref, ref_created = entity.reference_set.get_or_create(book=self)
+                refs[uri] = ref
+                if not ref_created:
+                    ref.occurence_set.all().delete()
+            sec = ref_elem.get_link()
+            m = re.match(r'sec(\d+)', sec)
+            assert m is not None
+            sec = int(m.group(1))
+            snippet = ref_elem.get_snippet()
+            b = builders['html-snippet']()
+            for s in snippet:
+                s.html_build(b)
+            html = b.output().get_bytes().decode('utf-8')
+
+            ref.occurence_set.create(
+                section=sec,
+                html=html
+            )
+        self.reference_set.exclude(entity__uri__in=refs).delete()
+
+    @property
+    def references(self):
+        return self.reference_set.all().select_related('entity')
+
     @classmethod
     @transaction.atomic
     def repopulate_ancestors(cls):
     @classmethod
     @transaction.atomic
     def repopulate_ancestors(cls):
@@ -710,6 +800,15 @@ class Book(models.Model):
                     b.ancestor.add(parent)
                     parent = parent.parent
 
                     b.ancestor.add(parent)
                     parent = parent.parent
 
+    @property
+    def ancestors(self):
+        if self.parent:
+            for anc in self.parent.ancestors:
+                yield anc
+            yield self.parent
+        else:
+            return []
+
     def clear_cache(self):
         clear_cached_renders(self.mini_box)
         clear_cached_renders(self.mini_box_nolink)
     def clear_cache(self):
         clear_cached_renders(self.mini_box)
         clear_cached_renders(self.mini_box_nolink)
@@ -736,16 +835,18 @@ class Book(models.Model):
     def related_themes(self):
         return Tag.objects.usage_for_queryset(
             Fragment.objects.filter(models.Q(book=self) | models.Q(book__ancestor=self)),
     def related_themes(self):
         return Tag.objects.usage_for_queryset(
             Fragment.objects.filter(models.Q(book=self) | models.Q(book__ancestor=self)),
-            counts=True).filter(category='theme')
+            counts=True).filter(category='theme').order_by('-count')
 
     def parent_cover_changed(self):
         """Called when parent book's cover image is changed."""
         if not self.cover_info(inherit=False):
             if 'cover' not in app_settings.DONT_BUILD:
                 self.cover.build_delay()
 
     def parent_cover_changed(self):
         """Called when parent book's cover image is changed."""
         if not self.cover_info(inherit=False):
             if 'cover' not in app_settings.DONT_BUILD:
                 self.cover.build_delay()
+                self.cover_clean.build_delay()
                 self.cover_thumb.build_delay()
                 self.cover_api_thumb.build_delay()
                 self.simple_cover.build_delay()
                 self.cover_thumb.build_delay()
                 self.cover_api_thumb.build_delay()
                 self.simple_cover.build_delay()
+                self.cover_ebookpoint.build_delay()
             for format_ in constants.EBOOK_FORMATS_WITH_COVERS:
                 if format_ not in app_settings.DONT_BUILD:
                     getattr(self, '%s_file' % format_).build_delay()
             for format_ in constants.EBOOK_FORMATS_WITH_COVERS:
                 if format_ not in app_settings.DONT_BUILD:
                     getattr(self, '%s_file' % format_).build_delay()
@@ -850,21 +951,32 @@ class Book(models.Model):
     def stage_note(self):
         stage = self.get_extra_info_json().get('stage')
         if stage and stage < '0.4':
     def stage_note(self):
         stage = self.get_extra_info_json().get('stage')
         if stage and stage < '0.4':
-            return (_('This work needs modernisation'),
+            return (_('Ten utwór wymaga uwspółcześnienia'),
                     reverse('infopage', args=['wymagajace-uwspolczesnienia']))
         else:
             return None, None
 
                     reverse('infopage', args=['wymagajace-uwspolczesnienia']))
         else:
             return None, None
 
-    def choose_fragment(self):
+    def choose_fragments(self, number):
         fragments = self.fragments.order_by()
         fragments_count = fragments.count()
         if not fragments_count and self.children.exists():
             fragments = Fragment.objects.filter(book__ancestor=self).order_by()
             fragments_count = fragments.count()
         if fragments_count:
         fragments = self.fragments.order_by()
         fragments_count = fragments.count()
         if not fragments_count and self.children.exists():
             fragments = Fragment.objects.filter(book__ancestor=self).order_by()
             fragments_count = fragments.count()
         if fragments_count:
-            return fragments[randint(0, fragments_count - 1)]
+            if fragments_count > number:
+                offset = randint(0, fragments_count - number)
+            else:
+                offset = 0
+            return fragments[offset : offset + number]
         elif self.parent:
         elif self.parent:
-            return self.parent.choose_fragment()
+            return self.parent.choose_fragments(number)
+        else:
+            return []
+
+    def choose_fragment(self):
+        fragments = self.choose_fragments(1)
+        if fragments:
+            return fragments[0]
         else:
             return None
 
         else:
             return None
 
@@ -920,28 +1032,6 @@ class Book(models.Model):
             'no_link': True,
         }
 
             'no_link': True,
         }
 
-def add_file_fields():
-    for format_ in Book.formats:
-        field_name = "%s_file" % format_
-        # This weird globals() assignment makes Django migrations comfortable.
-        _upload_to = _ebook_upload_to('book/%s/%%s.%s' % (format_, format_))
-        _upload_to.__name__ = '_%s_upload_to' % format_
-        globals()[_upload_to.__name__] = _upload_to
-
-        EbookField(
-            format_, _("%s file" % format_.upper()),
-            upload_to=_upload_to,
-            storage=bofh_storage,
-            max_length=255,
-            blank=True,
-            default=''
-        ).contribute_to_class(Book, field_name)
-        if format_ != 'xml':
-            models.CharField(max_length=255, editable=False, default='', db_index=True).contribute_to_class(Book, f'{field_name}_etag')
-
-
-add_file_fields()
-
 
 class BookPopularity(models.Model):
     book = models.OneToOneField(Book, models.CASCADE, related_name='popularity')
 
 class BookPopularity(models.Model):
     book = models.OneToOneField(Book, models.CASCADE, related_name='popularity')