Fetch thumbnails from wikidata.

[wolnelektury.git] / src / catalogue / models / book.py

diff --git a/src/catalogue/models/book.py b/src/catalogue/models/book.py
index d3e2a77..978bca4 100644 (file)
--- a/src/catalogue/models/book.py
+++ b/src/catalogue/models/book.py
@@ -1,46 +1,55 @@
-# -*- coding: utf-8 -*-

 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
 #
 from collections import OrderedDict
 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
 #
 from collections import OrderedDict

+import json
+from datetime import date, timedelta

 from random import randint
 import os.path
 import re
 from random import randint
 import os.path
 import re

-import urllib
+from urllib.request import urlretrieve

 from django.conf import settings
 from django.db import connection, models, transaction
 from django.conf import settings
 from django.db import connection, models, transaction

-from django.db.models import permalink

 import django.dispatch
 from django.contrib.contenttypes.fields import GenericRelation
 import django.dispatch
 from django.contrib.contenttypes.fields import GenericRelation

-from django.core.urlresolvers import reverse
-from django.utils.translation import ugettext_lazy as _
-import jsonfield
+from django.urls import reverse
+from django.utils.translation import ugettext_lazy as _, get_language
+from django.utils.deconstruct import deconstructible

 from fnpdjango.storage import BofhFileSystemStorage
 from fnpdjango.storage import BofhFileSystemStorage

-from ssify import flush_ssi_includes
+
+from librarian.cover import WLCover
+from librarian.html import transform_abstrakt

 from newtagging import managers
 from catalogue import constants
 from catalogue.fields import EbookField
 from catalogue.models import Tag, Fragment, BookMedia
 from newtagging import managers
 from catalogue import constants
 from catalogue.fields import EbookField
 from catalogue.models import Tag, Fragment, BookMedia

-from catalogue.utils import create_zip, gallery_url, gallery_path
+from catalogue.utils import create_zip, gallery_url, gallery_path, split_tags, get_random_hash
+from catalogue.models.tag import prefetched_relations

 from catalogue import app_settings
 from catalogue import tasks
 from catalogue import app_settings
 from catalogue import tasks

-from wolnelektury.utils import makedirs
+from wolnelektury.utils import makedirs, cached_render, clear_cached_renders

 
 bofh_storage = BofhFileSystemStorage()
 
 
 
 bofh_storage = BofhFileSystemStorage()
 
 

-def _make_upload_to(path):
-    def _upload_to(i, n):
-        return path % i.slug
-    return _upload_to
+@deconstructible
+class UploadToPath(object):
+    def __init__(self, path):
+        self.path = path
+
+    def __call__(self, instance, filename):
+        return self.path % instance.slug

 
 
 
 

-_cover_upload_to = _make_upload_to('book/cover/%s.jpg')
-_cover_thumb_upload_to = _make_upload_to('book/cover_thumb/%s.jpg')
+_cover_upload_to = UploadToPath('book/cover/%s.jpg')
+_cover_thumb_upload_to = UploadToPath('book/cover_thumb/%s.jpg')
+_cover_api_thumb_upload_to = UploadToPath('book/cover_api_thumb/%s.jpg')
+_simple_cover_upload_to = UploadToPath('book/cover_simple/%s.jpg')
+_cover_ebookpoint_upload_to = UploadToPath('book/cover_ebookpoint/%s.jpg')

 
 
 def _ebook_upload_to(upload_path):
 
 
 def _ebook_upload_to(upload_path):

-    return _make_upload_to(upload_path)
+    return UploadToPath(upload_path)

 
 
 class Book(models.Model):
 
 
 class Book(models.Model):


@@ -48,17 +57,25 @@ class Book(models.Model):
     title = models.CharField(_('title'), max_length=32767)
     sort_key = models.CharField(_('sort key'), max_length=120, db_index=True, editable=False)
     sort_key_author = models.CharField(
     title = models.CharField(_('title'), max_length=32767)
     sort_key = models.CharField(_('sort key'), max_length=120, db_index=True, editable=False)
     sort_key_author = models.CharField(

-        _('sort key by author'), max_length=120, db_index=True, editable=False, default=u'')
+        _('sort key by author'), max_length=120, db_index=True, editable=False, default='')

     slug = models.SlugField(_('slug'), max_length=120, db_index=True, unique=True)
     common_slug = models.SlugField(_('slug'), max_length=120, db_index=True)
     language = models.CharField(_('language code'), max_length=3, db_index=True, default=app_settings.DEFAULT_LANGUAGE)
     description = models.TextField(_('description'), blank=True)
     slug = models.SlugField(_('slug'), max_length=120, db_index=True, unique=True)
     common_slug = models.SlugField(_('slug'), max_length=120, db_index=True)
     language = models.CharField(_('language code'), max_length=3, db_index=True, default=app_settings.DEFAULT_LANGUAGE)
     description = models.TextField(_('description'), blank=True)

+    abstract = models.TextField(_('abstract'), blank=True)

     created_at = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True)
     created_at = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True)

-    changed_at = models.DateTimeField(_('creation date'), auto_now=True, db_index=True)
+    changed_at = models.DateTimeField(_('change date'), auto_now=True, db_index=True)

     parent_number = models.IntegerField(_('parent number'), default=0)
     parent_number = models.IntegerField(_('parent number'), default=0)

-    extra_info = jsonfield.JSONField(_('extra information'), default={})
+    extra_info = models.TextField(_('extra information'), default='{}')

     gazeta_link = models.CharField(blank=True, max_length=240)
     wiki_link = models.CharField(blank=True, max_length=240)
     gazeta_link = models.CharField(blank=True, max_length=240)
     wiki_link = models.CharField(blank=True, max_length=240)

+    print_on_demand = models.BooleanField(_('print on demand'), default=False)
+    recommended = models.BooleanField(_('recommended'), default=False)
+    audio_length = models.CharField(_('audio length'), blank=True, max_length=8)
+    preview = models.BooleanField(_('preview'), default=False)
+    preview_until = models.DateField(_('preview until'), blank=True, null=True)
+    preview_key = models.CharField(max_length=32, blank=True, null=True)
+    findable = models.BooleanField(_('findable'), default=True, db_index=True)

 
     # files generated during publication
     cover = EbookField(
 
     # files generated during publication
     cover = EbookField(


@@ -66,18 +83,41 @@ class Book(models.Model):
         null=True, blank=True,
         upload_to=_cover_upload_to,
         storage=bofh_storage, max_length=255)
         null=True, blank=True,
         upload_to=_cover_upload_to,
         storage=bofh_storage, max_length=255)

+    cover_etag = models.CharField(max_length=255, editable=False, default='', db_index=True)

     # Cleaner version of cover for thumbs
     cover_thumb = EbookField(
         'cover_thumb', _('cover thumbnail'),
         null=True, blank=True,
         upload_to=_cover_thumb_upload_to,
         max_length=255)
     # Cleaner version of cover for thumbs
     cover_thumb = EbookField(
         'cover_thumb', _('cover thumbnail'),
         null=True, blank=True,
         upload_to=_cover_thumb_upload_to,
         max_length=255)

+    cover_thumb_etag = models.CharField(max_length=255, editable=False, default='', db_index=True)
+    cover_api_thumb = EbookField(
+        'cover_api_thumb', _('cover thumbnail for mobile app'),
+        null=True, blank=True,
+        upload_to=_cover_api_thumb_upload_to,
+        max_length=255)
+    cover_api_thumb_etag = models.CharField(max_length=255, editable=False, default='', db_index=True)
+    simple_cover = EbookField(
+        'simple_cover', _('cover for mobile app'),
+        null=True, blank=True,
+        upload_to=_simple_cover_upload_to,
+        max_length=255)
+    simple_cover_etag = models.CharField(max_length=255, editable=False, default='', db_index=True)
+    cover_ebookpoint = EbookField(
+        'cover_ebookpoint', _('cover for Ebookpoint'),
+        null=True, blank=True,
+        upload_to=_cover_ebookpoint_upload_to,
+        max_length=255)
+    cover_ebookpoint_etag = models.CharField(max_length=255, editable=False, default='', db_index=True)

     ebook_formats = constants.EBOOK_FORMATS
     formats = ebook_formats + ['html', 'xml']
 
     ebook_formats = constants.EBOOK_FORMATS
     formats = ebook_formats + ['html', 'xml']
 

-    parent = models.ForeignKey('self', blank=True, null=True, related_name='children')
+    parent = models.ForeignKey('self', models.CASCADE, blank=True, null=True, related_name='children')

     ancestor = models.ManyToManyField('self', blank=True, editable=False, related_name='descendant', symmetrical=False)
 
     ancestor = models.ManyToManyField('self', blank=True, editable=False, related_name='descendant', symmetrical=False)
 

+    cached_author = models.CharField(blank=True, max_length=240, db_index=True)
+    has_audience = models.BooleanField(default=False)
+

     objects = models.Manager()
     tagged = managers.ModelTaggedItemManager(Tag)
     tags = managers.TagDescriptor(Tag)
     objects = models.Manager()
     tagged = managers.ModelTaggedItemManager(Tag)
     tags = managers.TagDescriptor(Tag)


@@ -86,20 +126,23 @@ class Book(models.Model):
     html_built = django.dispatch.Signal()
     published = django.dispatch.Signal()
 
     html_built = django.dispatch.Signal()
     published = django.dispatch.Signal()
 

-    short_html_url_name = 'catalogue_book_short'
+    SORT_KEY_SEP = '$'

 
     class AlreadyExists(Exception):
         pass
 
     class Meta:
 
     class AlreadyExists(Exception):
         pass
 
     class Meta:

-        ordering = ('sort_key',)
+        ordering = ('sort_key_author', 'sort_key')

         verbose_name = _('book')
         verbose_name_plural = _('books')
         app_label = 'catalogue'
 
         verbose_name = _('book')
         verbose_name_plural = _('books')
         app_label = 'catalogue'
 

-    def __unicode__(self):
+    def __str__(self):

         return self.title
 
         return self.title
 

+    def get_extra_info_json(self):
+        return json.loads(self.extra_info or '{}')
+

     def get_initial(self):
         try:
             return re.search(r'\w', self.title, re.U).group(0)
     def get_initial(self):
         try:
             return re.search(r'\w', self.title, re.U).group(0)


@@ -109,33 +152,94 @@ class Book(models.Model):
     def authors(self):
         return self.tags.filter(category='author')
 
     def authors(self):
         return self.tags.filter(category='author')
 

+    def epochs(self):
+        return self.tags.filter(category='epoch')
+
+    def genres(self):
+        return self.tags.filter(category='genre')
+
+    def kinds(self):
+        return self.tags.filter(category='kind')
+
+    def tag_unicode(self, category):
+        relations = prefetched_relations(self, category)
+        if relations:
+            return ', '.join(rel.tag.name for rel in relations)
+        else:
+            return ', '.join(self.tags.filter(category=category).values_list('name', flat=True))
+
+    def tags_by_category(self):
+        return split_tags(self.tags.exclude(category__in=('set', 'theme')))
+

     def author_unicode(self):
     def author_unicode(self):

-        return ", ".join(self.authors().values_list('name', flat=True))
+        return self.cached_author
+
+    def kind_unicode(self):
+        return self.tag_unicode('kind')
+
+    def epoch_unicode(self):
+        return self.tag_unicode('epoch')
+
+    def genre_unicode(self):
+        return self.tag_unicode('genre')
+
+    def translators(self):
+        translators = self.get_extra_info_json().get('translators') or []
+        return [
+            '\xa0'.join(reversed(translator.split(', ', 1))) for translator in translators
+        ]
+
+    def translator(self):
+        translators = self.get_extra_info_json().get('translators')
+        if not translators:
+            return None
+        if len(translators) > 3:
+            translators = translators[:2]
+            others = ' i inni'
+        else:
+            others = ''
+        return ', '.join('\xa0'.join(reversed(translator.split(', ', 1))) for translator in translators) + others
+
+    def cover_source(self):
+        return self.get_extra_info_json().get('cover_source', self.parent.cover_source() if self.parent else '')
+
+    @property
+    def isbn_pdf(self):
+        return self.get_extra_info_json().get('isbn_pdf')
+
+    @property
+    def isbn_epub(self):
+        return self.get_extra_info_json().get('isbn_epub')
+
+    @property
+    def isbn_mobi(self):
+        return self.get_extra_info_json().get('isbn_mobi')
+

 
     def save(self, force_insert=False, force_update=False, **kwargs):
         from sortify import sortify
 
         self.sort_key = sortify(self.title)[:120]
 
     def save(self, force_insert=False, force_update=False, **kwargs):
         from sortify import sortify
 
         self.sort_key = sortify(self.title)[:120]

-        self.title = unicode(self.title)  # ???
+        self.title = str(self.title)  # ???

 
         try:
             author = self.authors().first().sort_key
         except AttributeError:
 
         try:
             author = self.authors().first().sort_key
         except AttributeError:

-            author = u''
+            author = ''

         self.sort_key_author = author
 
         self.sort_key_author = author
 

+        self.cached_author = self.tag_unicode('author')
+        self.has_audience = 'audience' in self.get_extra_info_json()
+
+        if self.preview and not self.preview_key:
+            self.preview_key = get_random_hash(self.slug)[:32]
+

         ret = super(Book, self).save(force_insert, force_update, **kwargs)
 
         return ret
 
         ret = super(Book, self).save(force_insert, force_update, **kwargs)
 
         return ret
 

-    @permalink

     def get_absolute_url(self):
     def get_absolute_url(self):

-        return 'catalogue.views.book_detail', [self.slug]
-
-    @staticmethod
-    @permalink
-    def create_url(slug):
-        return 'catalogue.views.book_detail', [slug]
+        return reverse('book_detail', args=[self.slug])

 
     def gallery_path(self):
         return gallery_path(self.slug)
 
     def gallery_path(self):
         return gallery_path(self.slug)


@@ -143,6 +247,41 @@ class Book(models.Model):
     def gallery_url(self):
         return gallery_url(self.slug)
 
     def gallery_url(self):
         return gallery_url(self.slug)
 

+    def get_first_text(self):
+        if self.html_file:
+            return self
+        child = self.children.all().order_by('parent_number').first()
+        if child is not None:
+            return child.get_first_text()
+
+    def get_last_text(self):
+        if self.html_file:
+            return self
+        child = self.children.all().order_by('parent_number').last()
+        if child is not None:
+            return child.get_last_text()
+
+    def get_prev_text(self):
+        if not self.parent:
+            return None
+        sibling = self.parent.children.filter(parent_number__lt=self.parent_number).order_by('-parent_number').first()
+        if sibling is not None:
+            return sibling.get_last_text()
+        return self.parent.get_prev_text()
+
+    def get_next_text(self):
+        if not self.parent:
+            return None
+        sibling = self.parent.children.filter(parent_number__gt=self.parent_number).order_by('parent_number').first()
+        if sibling is not None:
+            return sibling.get_first_text()
+        return self.parent.get_next_text()
+
+    def get_siblings(self):
+        if not self.parent:
+            return []
+        return self.parent.children.all().order_by('parent_number')
+

     @property
     def name(self):
         return self.title
     @property
     def name(self):
         return self.title


@@ -156,12 +295,45 @@ class Book(models.Model):
     def is_foreign(self):
         return self.language_code() != settings.LANGUAGE_CODE
 
     def is_foreign(self):
         return self.language_code() != settings.LANGUAGE_CODE
 

+    def set_audio_length(self):
+        length = self.get_audio_length()
+        if length > 0:
+            self.audio_length = self.format_audio_length(length)
+            self.save()
+
+    @staticmethod
+    def format_audio_length(seconds):
+        """
+        >>> Book.format_audio_length(1)
+        '0:01'
+        >>> Book.format_audio_length(3661)
+        '1:01:01'
+        """
+        if seconds < 60*60:
+            minutes = seconds // 60
+            seconds = seconds % 60
+            return '%d:%02d' % (minutes, seconds)
+        else:
+            hours = seconds // 3600
+            minutes = seconds % 3600 // 60
+            seconds = seconds % 60
+            return '%d:%02d:%02d' % (hours, minutes, seconds)
+
+    def get_audio_length(self):
+        total = 0
+        for media in self.get_mp3() or ():
+            total += app_settings.GET_MP3_LENGTH(media.file.path)
+        return int(total)
+

     def has_media(self, type_):
         if type_ in Book.formats:
             return bool(getattr(self, "%s_file" % type_))
         else:
             return self.media.filter(type=type_).exists()
 
     def has_media(self, type_):
         if type_ in Book.formats:
             return bool(getattr(self, "%s_file" % type_))
         else:
             return self.media.filter(type=type_).exists()
 

+    def has_audio(self):
+        return self.has_media('mp3')
+

     def get_media(self, type_):
         if self.has_media(type_):
             if type_ in Book.formats:
     def get_media(self, type_):
         if self.has_media(type_):
             if type_ in Book.formats:


@@ -183,27 +355,84 @@ class Book(models.Model):
     def get_daisy(self):
         return self.get_media("daisy")
 
     def get_daisy(self):
         return self.get_media("daisy")
 

+    def media_url(self, format_):
+        media = self.get_media(format_)
+        if media:
+            if self.preview:
+                return reverse('embargo_link', kwargs={'key': self.preview_key, 'slug': self.slug, 'format_': format_})
+            else:
+                return media.url
+        else:
+            return None
+
+    def html_url(self):
+        return self.media_url('html')
+
+    def pdf_url(self):
+        return self.media_url('pdf')
+
+    def epub_url(self):
+        return self.media_url('epub')
+
+    def mobi_url(self):
+        return self.media_url('mobi')
+
+    def txt_url(self):
+        return self.media_url('txt')
+
+    def fb2_url(self):
+        return self.media_url('fb2')
+
+    def xml_url(self):
+        return self.media_url('xml')
+

     def has_description(self):
         return len(self.description) > 0
     has_description.short_description = _('description')
     has_description.boolean = True
 
     def has_description(self):
         return len(self.description) > 0
     has_description.short_description = _('description')
     has_description.boolean = True
 

-    # ugly ugly ugly

     def has_mp3_file(self):
     def has_mp3_file(self):

-        return bool(self.has_media("mp3"))
+        return self.has_media("mp3")

     has_mp3_file.short_description = 'MP3'
     has_mp3_file.boolean = True
 
     def has_ogg_file(self):
     has_mp3_file.short_description = 'MP3'
     has_mp3_file.boolean = True
 
     def has_ogg_file(self):

-        return bool(self.has_media("ogg"))
+        return self.has_media("ogg")

     has_ogg_file.short_description = 'OGG'
     has_ogg_file.boolean = True
 
     def has_daisy_file(self):
     has_ogg_file.short_description = 'OGG'
     has_ogg_file.boolean = True
 
     def has_daisy_file(self):

-        return bool(self.has_media("daisy"))
+        return self.has_media("daisy")

     has_daisy_file.short_description = 'DAISY'
     has_daisy_file.boolean = True
 
     has_daisy_file.short_description = 'DAISY'
     has_daisy_file.boolean = True
 

+    def get_audiobooks(self):
+        ogg_files = {}
+        for m in self.media.filter(type='ogg').order_by().iterator():
+            ogg_files[m.name] = m
+
+        audiobooks = []
+        projects = set()
+        for mp3 in self.media.filter(type='mp3').iterator():
+            # ogg files are always from the same project
+            meta = mp3.get_extra_info_json()
+            project = meta.get('project')
+            if not project:
+                # temporary fallback
+                project = 'CzytamySłuchając'
+
+            projects.add((project, meta.get('funded_by', '')))
+
+            media = {'mp3': mp3}
+
+            ogg = ogg_files.get(mp3.name)
+            if ogg:
+                media['ogg'] = ogg
+            audiobooks.append(media)
+
+        projects = sorted(projects)
+        return audiobooks, projects
+

     def wldocument(self, parse_dublincore=True, inherit=True):
         from catalogue.import_utils import ORMDocProvider
         from librarian.parser import WLDocument
     def wldocument(self, parse_dublincore=True, inherit=True):
         from catalogue.import_utils import ORMDocProvider
         from librarian.parser import WLDocument


@@ -223,12 +452,12 @@ class Book(models.Model):
     def zip_format(format_):
         def pretty_file_name(book):
             return "%s/%s.%s" % (
     def zip_format(format_):
         def pretty_file_name(book):
             return "%s/%s.%s" % (

-                book.extra_info['author'],
+                book.get_extra_info_json()['author'],

                 book.slug,
                 format_)
 
         field_name = "%s_file" % format_
                 book.slug,
                 format_)
 
         field_name = "%s_file" % format_

-        books = Book.objects.filter(parent=None).exclude(**{field_name: ""})
+        books = Book.objects.filter(parent=None).exclude(**{field_name: ""}).exclude(preview=True).exclude(findable=False)

         paths = [(pretty_file_name(b), getattr(b, field_name).path) for b in books.iterator()]
         return create_zip(paths, app_settings.FORMAT_ZIPS[format_])
 
         paths = [(pretty_file_name(b), getattr(b, field_name).path) for b in books.iterator()]
         return create_zip(paths, app_settings.FORMAT_ZIPS[format_])
 


@@ -238,6 +467,8 @@ class Book(models.Model):
         return create_zip(paths, "%s_%s" % (self.slug, format_))
 
     def search_index(self, book_info=None, index=None, index_tags=True, commit=True):
         return create_zip(paths, "%s_%s" % (self.slug, format_))
 
     def search_index(self, book_info=None, index=None, index_tags=True, commit=True):

+        if not self.findable:
+            return

         if index is None:
             from search.index import Index
             index = Index()
         if index is None:
             from search.index import Index
             index = Index()


@@ -247,10 +478,11 @@ class Book(models.Model):
                 index.index_tags()
             if commit:
                 index.index.commit()
                 index.index_tags()
             if commit:
                 index.index.commit()

-        except Exception, e:
+        except Exception as e:

             index.index.rollback()
             raise e
 
             index.index.rollback()
             raise e
 

+    # will make problems in conjunction with paid previews

     def download_pictures(self, remote_gallery_url):
         gallery_path = self.gallery_path()
         # delete previous files, so we don't include old files in ebooks
     def download_pictures(self, remote_gallery_url):
         gallery_path = self.gallery_path()
         # delete previous files, so we don't include old files in ebooks


@@ -264,7 +496,14 @@ class Book(models.Model):
             for ilustr in ilustr_elements:
                 ilustr_src = ilustr.get('src')
                 ilustr_path = os.path.join(gallery_path, ilustr_src)
             for ilustr in ilustr_elements:
                 ilustr_src = ilustr.get('src')
                 ilustr_path = os.path.join(gallery_path, ilustr_src)

-                urllib.urlretrieve('%s/%s' % (remote_gallery_url, ilustr_src), ilustr_path)
+                urlretrieve('%s/%s' % (remote_gallery_url, ilustr_src), ilustr_path)
+
+    def load_abstract(self):
+        abstract = self.wldocument(parse_dublincore=False).edoc.getroot().find('.//abstrakt')
+        if abstract is not None:
+            self.abstract = transform_abstrakt(abstract)
+        else:
+            self.abstract = ''

 
     @classmethod
     def from_xml_file(cls, xml_file, **kwargs):
 
     @classmethod
     def from_xml_file(cls, xml_file, **kwargs):


@@ -284,7 +523,7 @@ class Book(models.Model):
 
     @classmethod
     def from_text_and_meta(cls, raw_file, book_info, overwrite=False, dont_build=None, search_index=True,
 
     @classmethod
     def from_text_and_meta(cls, raw_file, book_info, overwrite=False, dont_build=None, search_index=True,

-                           search_index_tags=True, remote_gallery_url=None):
+                           search_index_tags=True, remote_gallery_url=None, days=0, findable=True):

         if dont_build is None:
             dont_build = set()
         dont_build = set.union(set(dont_build), set(app_settings.DONT_BUILD))
         if dont_build is None:
             dont_build = set()
         dont_build = set.union(set(dont_build), set(app_settings.DONT_BUILD))


@@ -307,6 +546,9 @@ class Book(models.Model):
         if created:
             book_shelves = []
             old_cover = None
         if created:
             book_shelves = []
             old_cover = None

+            book.preview = bool(days)
+            if book.preview:
+                book.preview_until = date.today() + timedelta(days)

         else:
             if not overwrite:
                 raise Book.AlreadyExists(_('Book %s already exists') % book_slug)
         else:
             if not overwrite:
                 raise Book.AlreadyExists(_('Book %s already exists') % book_slug)


@@ -316,19 +558,29 @@ class Book(models.Model):
 
         # Save XML file
         book.xml_file.save('%s.xml' % book.slug, raw_file, save=False)
 
         # Save XML file
         book.xml_file.save('%s.xml' % book.slug, raw_file, save=False)

+        if book.preview:
+            book.xml_file.set_readable(False)

 
 

+        book.findable = findable

         book.language = book_info.language
         book.title = book_info.title
         if book_info.variant_of:
             book.common_slug = book_info.variant_of.slug
         else:
             book.common_slug = book.slug
         book.language = book_info.language
         book.title = book_info.title
         if book_info.variant_of:
             book.common_slug = book_info.variant_of.slug
         else:
             book.common_slug = book.slug

-        book.extra_info = book_info.to_dict()
+        book.extra_info = json.dumps(book_info.to_dict())
+        book.load_abstract()

         book.save()
 
         meta_tags = Tag.tags_from_info(book_info)
 
         book.save()
 
         meta_tags = Tag.tags_from_info(book_info)
 

+        for tag in meta_tags:
+            if not tag.for_books:
+                tag.for_books = True
+                tag.save()
+

         book.tags = set(meta_tags + book_shelves)
         book.tags = set(meta_tags + book_shelves)

+        book.save()  # update sort_key_author

 
         cover_changed = old_cover != book.cover_info()
         obsolete_children = set(b for b in book.children.all()
 
         cover_changed = old_cover != book.cover_info()
         obsolete_children = set(b for b in book.children.all()


@@ -361,6 +613,9 @@ class Book(models.Model):
         if 'cover' not in dont_build:
             book.cover.build_delay()
             book.cover_thumb.build_delay()
         if 'cover' not in dont_build:
             book.cover.build_delay()
             book.cover_thumb.build_delay()

+            book.cover_api_thumb.build_delay()
+            book.simple_cover.build_delay()
+            book.cover_ebookpoint.build_delay()

 
         # Build HTML and ebooks.
         book.html_file.build_delay()
 
         # Build HTML and ebooks.
         book.html_file.build_delay()


@@ -372,15 +627,60 @@ class Book(models.Model):
             if format_ not in dont_build:
                 getattr(book, '%s_file' % format_).build_delay()
 
             if format_ not in dont_build:
                 getattr(book, '%s_file' % format_).build_delay()
 

-        if not settings.NO_SEARCH_INDEX and search_index:
+        if not settings.NO_SEARCH_INDEX and search_index and findable:

             tasks.index_book.delay(book.id, book_info=book_info, index_tags=search_index_tags)
 
         for child in notify_cover_changed:
             child.parent_cover_changed()
 
             tasks.index_book.delay(book.id, book_info=book_info, index_tags=search_index_tags)
 
         for child in notify_cover_changed:
             child.parent_cover_changed()
 

+        book.update_popularity()
+        tasks.update_references.delay(book.id)
+

         cls.published.send(sender=cls, instance=book)
         return book
 
         cls.published.send(sender=cls, instance=book)
         return book
 

+    def get_master(self):
+        master_tags = [
+            'opowiadanie',
+            'powiesc',
+            'dramat_wierszowany_l',
+            'dramat_wierszowany_lp',
+            'dramat_wspolczesny', 'liryka_l', 'liryka_lp',
+            'wywiad',
+        ]
+        from librarian.parser import WLDocument
+        wld = WLDocument.from_file(self.xml_file.path, parse_dublincore=False)
+        root = wld.edoc.getroot()
+        for master in root.iter():
+            if master.tag in master_tags:
+                return master
+    
+    def update_references(self):
+        from references.models import Entity, Reference
+        master = self.get_master()
+        found = set()
+        for i, sec in enumerate(master):
+            for ref in sec.findall('.//ref'):
+                href = ref.attrib.get('href', '')
+                if not href or href in found:
+                    continue
+                found.add(href)
+                entity, created = Entity.objects.get_or_create(
+                    uri=href
+                )
+                ref, created = Reference.objects.get_or_create(
+                    book=self,
+                    entity=entity
+                )
+                ref.first_section = 'sec%d' % (i + 1)
+                entity.populate()
+                entity.save()
+        Reference.objects.filter(book=self).exclude(entity__uri__in=found).delete()
+    
+    @property
+    def references(self):
+        return self.reference_set.all().select_related('entity')
+

     @classmethod
     @transaction.atomic
     def repopulate_ancestors(cls):
     @classmethod
     @transaction.atomic
     def repopulate_ancestors(cls):


@@ -414,23 +714,9 @@ class Book(models.Model):
                     b.ancestor.add(parent)
                     parent = parent.parent
 
                     b.ancestor.add(parent)
                     parent = parent.parent
 

-    def flush_includes(self, languages=True):
-        if not languages:
-            return
-        if languages is True:
-            languages = [lc for (lc, _ln) in settings.LANGUAGES]
-        flush_ssi_includes([
-            template % (self.pk, lang)
-            for template in [
-                '/katalog/b/%d/mini.%s.html',
-                '/katalog/b/%d/mini_nolink.%s.html',
-                '/katalog/b/%d/short.%s.html',
-                '/katalog/b/%d/wide.%s.html',
-                '/api/include/book/%d.%s.json',
-                '/api/include/book/%d.%s.xml',
-                ]
-            for lang in languages
-            ])
+    def clear_cache(self):
+        clear_cached_renders(self.mini_box)
+        clear_cached_renders(self.mini_box_nolink)

 
     def cover_info(self, inherit=True):
         """Returns a dictionary to serve as fallback for BookInfo.
 
     def cover_info(self, inherit=True):
         """Returns a dictionary to serve as fallback for BookInfo.


@@ -440,7 +726,7 @@ class Book(models.Model):
         need = False
         info = {}
         for field in ('cover_url', 'cover_by', 'cover_source'):
         need = False
         info = {}
         for field in ('cover_url', 'cover_by', 'cover_source'):

-            val = self.extra_info.get(field)
+            val = self.get_extra_info_json().get(field)

             if val:
                 info[field] = val
             else:
             if val:
                 info[field] = val
             else:


@@ -462,6 +748,8 @@ class Book(models.Model):
             if 'cover' not in app_settings.DONT_BUILD:
                 self.cover.build_delay()
                 self.cover_thumb.build_delay()
             if 'cover' not in app_settings.DONT_BUILD:
                 self.cover.build_delay()
                 self.cover_thumb.build_delay()

+                self.cover_api_thumb.build_delay()
+                self.simple_cover.build_delay()

             for format_ in constants.EBOOK_FORMATS_WITH_COVERS:
                 if format_ not in app_settings.DONT_BUILD:
                     getattr(self, '%s_file' % format_).build_delay()
             for format_ in constants.EBOOK_FORMATS_WITH_COVERS:
                 if format_ not in app_settings.DONT_BUILD:
                     getattr(self, '%s_file' % format_).build_delay()


@@ -470,7 +758,7 @@ class Book(models.Model):
 
     def other_versions(self):
         """Find other versions (i.e. in other languages) of the book."""
 
     def other_versions(self):
         """Find other versions (i.e. in other languages) of the book."""

-        return type(self).objects.filter(common_slug=self.common_slug).exclude(pk=self.pk)
+        return type(self).objects.filter(common_slug=self.common_slug, findable=True).exclude(pk=self.pk)

 
     def parents(self):
         books = []
 
     def parents(self):
         books = []


@@ -491,6 +779,13 @@ class Book(models.Model):
             names = [tag[0] for tag in names]
         return ', '.join(names)
 
             names = [tag[0] for tag in names]
         return ', '.join(names)
 

+    def publisher(self):
+        publisher = self.get_extra_info_json()['publisher']
+        if isinstance(publisher, str):
+            return publisher
+        elif isinstance(publisher, list):
+            return ', '.join(publisher)
+

     @classmethod
     def tagged_top_level(cls, tags):
         """ Returns top-level books tagged with `tags`.
     @classmethod
     def tagged_top_level(cls, tags):
         """ Returns top-level books tagged with `tags`.


@@ -500,7 +795,7 @@ class Book(models.Model):
 
         """
         objects = cls.tagged.with_all(tags)
 
         """
         objects = cls.tagged.with_all(tags)

-        return objects.exclude(ancestor__in=objects)
+        return objects.filter(findable=True).exclude(ancestor__in=objects)

 
     @classmethod
     def book_list(cls, book_filter=None):
 
     @classmethod
     def book_list(cls, book_filter=None):


@@ -511,7 +806,7 @@ class Book(models.Model):
         """
 
         books_by_parent = {}
         """
 
         books_by_parent = {}

-        books = cls.objects.order_by('parent_number', 'sort_key').only('title', 'parent', 'slug')
+        books = cls.objects.filter(findable=True).order_by('parent_number', 'sort_key').only('title', 'parent', 'slug', 'extra_info')

         if book_filter:
             books = books.filter(book_filter).distinct()
 
         if book_filter:
             books = books.filter(book_filter).distinct()
 


@@ -541,22 +836,23 @@ class Book(models.Model):
         return books_by_author, orphans, books_by_parent
 
     _audiences_pl = {
         return books_by_author, orphans, books_by_parent
 
     _audiences_pl = {

-        "SP": (1, u"szkoła podstawowa"),
-        "SP1": (1, u"szkoła podstawowa"),
-        "SP2": (1, u"szkoła podstawowa"),
-        "P": (1, u"szkoła podstawowa"),
-        "G": (2, u"gimnazjum"),
-        "L": (3, u"liceum"),
-        "LP": (3, u"liceum"),
+        "SP": (1, "szkoła podstawowa"),
+        "SP1": (1, "szkoła podstawowa"),
+        "SP2": (1, "szkoła podstawowa"),
+        "SP3": (1, "szkoła podstawowa"),
+        "P": (1, "szkoła podstawowa"),
+        "G": (2, "gimnazjum"),
+        "L": (3, "liceum"),
+        "LP": (3, "liceum"),

     }
 
     def audiences_pl(self):
     }
 
     def audiences_pl(self):

-        audiences = self.extra_info.get('audiences', [])
+        audiences = self.get_extra_info_json().get('audiences', [])

         audiences = sorted(set([self._audiences_pl.get(a, (99, a)) for a in audiences]))
         return [a[1] for a in audiences]
 
     def stage_note(self):
         audiences = sorted(set([self._audiences_pl.get(a, (99, a)) for a in audiences]))
         return [a[1] for a in audiences]
 
     def stage_note(self):

-        stage = self.extra_info.get('stage')
+        stage = self.get_extra_info_json().get('stage')

         if stage and stage < '0.4':
             return (_('This work needs modernisation'),
                     reverse('infopage', args=['wymagajace-uwspolczesnienia']))
         if stage and stage < '0.4':
             return (_('This work needs modernisation'),
                     reverse('infopage', args=['wymagajace-uwspolczesnienia']))


@@ -576,6 +872,57 @@ class Book(models.Model):
         else:
             return None
 
         else:
             return None
 

+    def fragment_data(self):
+        fragment = self.choose_fragment()
+        if fragment:
+            return {
+                'title': fragment.book.pretty_title(),
+                'html': re.sub('</?blockquote[^>]*>', '', fragment.get_short_text()),
+            }
+        else:
+            return None
+
+    def update_popularity(self):
+        count = self.tags.filter(category='set').values('user').order_by('user').distinct().count()
+        try:
+            pop = self.popularity
+            pop.count = count
+            pop.save()
+        except BookPopularity.DoesNotExist:
+            BookPopularity.objects.create(book=self, count=count)
+
+    def ridero_link(self):
+        return 'https://ridero.eu/%s/books/wl_%s/' % (get_language(), self.slug.replace('-', '_'))
+
+    def like(self, user):
+        from social.utils import likes, get_set, set_sets
+        if not likes(user, self):
+            tag = get_set(user, '')
+            set_sets(user, self, [tag])
+
+    def unlike(self, user):
+        from social.utils import likes, set_sets
+        if likes(user, self):
+            set_sets(user, self, [])
+
+    def full_sort_key(self):
+        return self.SORT_KEY_SEP.join((self.sort_key_author, self.sort_key, str(self.id)))
+
+    def cover_color(self):
+        return WLCover.epoch_colors.get(self.get_extra_info_json().get('epoch'), '#000000')
+
+    @cached_render('catalogue/book_mini_box.html')
+    def mini_box(self):
+        return {
+            'book': self
+        }
+
+    @cached_render('catalogue/book_mini_box.html')
+    def mini_box_nolink(self):
+        return {
+            'book': self,
+            'no_link': True,
+        }

 
 def add_file_fields():
     for format_ in Book.formats:
 
 def add_file_fields():
     for format_ in Book.formats:


@@ -593,5 +940,13 @@ def add_file_fields():
             blank=True,
             default=''
         ).contribute_to_class(Book, field_name)
             blank=True,
             default=''
         ).contribute_to_class(Book, field_name)

+        if format_ != 'xml':
+            models.CharField(max_length=255, editable=False, default='', db_index=True).contribute_to_class(Book, f'{field_name}_etag')
+

 
 add_file_fields()
 
 add_file_fields()

+
+
+class BookPopularity(models.Model):
+    book = models.OneToOneField(Book, models.CASCADE, related_name='popularity')
+    count = models.IntegerField(default=0, db_index=True)