limited stopwords + better search by author + remove descendant results

[wolnelektury.git] / src / catalogue / models / book.py

diff --git a/src/catalogue/models/book.py b/src/catalogue/models/book.py
index 5194fe1..0f664c4 100644 (file)
--- a/src/catalogue/models/book.py
+++ b/src/catalogue/models/book.py
@@ -4,14 +4,17 @@
 #
 from collections import OrderedDict
 from random import randint
 #
 from collections import OrderedDict
 from random import randint

+import os.path

 import re
 import re

+import urllib

 from django.conf import settings
 from django.db import connection, models, transaction
 from django.db.models import permalink
 import django.dispatch
 from django.contrib.contenttypes.fields import GenericRelation
 from django.core.urlresolvers import reverse
 from django.conf import settings
 from django.db import connection, models, transaction
 from django.db.models import permalink
 import django.dispatch
 from django.contrib.contenttypes.fields import GenericRelation
 from django.core.urlresolvers import reverse

-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import ugettext_lazy as _, get_language
+from django.utils.deconstruct import deconstructible

 import jsonfield
 from fnpdjango.storage import BofhFileSystemStorage
 from ssify import flush_ssi_includes
 import jsonfield
 from fnpdjango.storage import BofhFileSystemStorage
 from ssify import flush_ssi_includes


@@ -19,25 +22,32 @@ from newtagging import managers
 from catalogue import constants
 from catalogue.fields import EbookField
 from catalogue.models import Tag, Fragment, BookMedia
 from catalogue import constants
 from catalogue.fields import EbookField
 from catalogue.models import Tag, Fragment, BookMedia

-from catalogue.utils import create_zip
+from catalogue.utils import create_zip, gallery_url, gallery_path, split_tags
+from catalogue.models.tag import prefetched_relations

 from catalogue import app_settings
 from catalogue import tasks
 from catalogue import app_settings
 from catalogue import tasks

+from wolnelektury.utils import makedirs

 
 bofh_storage = BofhFileSystemStorage()
 
 
 
 bofh_storage = BofhFileSystemStorage()
 
 

-def _make_upload_to(path):
-    def _upload_to(i, n):
-        return path % i.slug
-    return _upload_to
+@deconstructible
+class UploadToPath(object):
+    def __init__(self, path):
+        self.path = path

 
 

+    def __call__(self, instance, filename):
+        return self.path % instance.slug

 
 

-_cover_upload_to = _make_upload_to('book/cover/%s.jpg')
-_cover_thumb_upload_to = _make_upload_to('book/cover_thumb/%s.jpg')
+
+_cover_upload_to = UploadToPath('book/cover/%s.jpg')
+_cover_thumb_upload_to = UploadToPath('book/cover_thumb/%s.jpg')
+_cover_api_thumb_upload_to = UploadToPath('book/cover_api_thumb/%s.jpg')
+_simple_cover_upload_to = UploadToPath('book/cover_simple/%s.jpg')

 
 
 def _ebook_upload_to(upload_path):
 
 
 def _ebook_upload_to(upload_path):

-    return _make_upload_to(upload_path)
+    return UploadToPath(upload_path)

 
 
 class Book(models.Model):
 
 
 class Book(models.Model):


@@ -51,11 +61,13 @@ class Book(models.Model):
     language = models.CharField(_('language code'), max_length=3, db_index=True, default=app_settings.DEFAULT_LANGUAGE)
     description = models.TextField(_('description'), blank=True)
     created_at = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True)
     language = models.CharField(_('language code'), max_length=3, db_index=True, default=app_settings.DEFAULT_LANGUAGE)
     description = models.TextField(_('description'), blank=True)
     created_at = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True)

-    changed_at = models.DateTimeField(_('creation date'), auto_now=True, db_index=True)
+    changed_at = models.DateTimeField(_('change date'), auto_now=True, db_index=True)

     parent_number = models.IntegerField(_('parent number'), default=0)
     extra_info = jsonfield.JSONField(_('extra information'), default={})
     gazeta_link = models.CharField(blank=True, max_length=240)
     wiki_link = models.CharField(blank=True, max_length=240)
     parent_number = models.IntegerField(_('parent number'), default=0)
     extra_info = jsonfield.JSONField(_('extra information'), default={})
     gazeta_link = models.CharField(blank=True, max_length=240)
     wiki_link = models.CharField(blank=True, max_length=240)

+    print_on_demand = models.BooleanField(_('print on demand'), default=False)
+    recommended = models.BooleanField(_('recommended'), default=False)

 
     # files generated during publication
     cover = EbookField(
 
     # files generated during publication
     cover = EbookField(


@@ -69,12 +81,25 @@ class Book(models.Model):
         null=True, blank=True,
         upload_to=_cover_thumb_upload_to,
         max_length=255)
         null=True, blank=True,
         upload_to=_cover_thumb_upload_to,
         max_length=255)

+    cover_api_thumb = EbookField(
+        'cover_api_thumb', _('cover thumbnail for mobile app'),
+        null=True, blank=True,
+        upload_to=_cover_api_thumb_upload_to,
+        max_length=255)
+    simple_cover = EbookField(
+        'simple_cover', _('cover for mobile app'),
+        null=True, blank=True,
+        upload_to=_simple_cover_upload_to,
+        max_length=255)

     ebook_formats = constants.EBOOK_FORMATS
     formats = ebook_formats + ['html', 'xml']
 
     parent = models.ForeignKey('self', blank=True, null=True, related_name='children')
     ancestor = models.ManyToManyField('self', blank=True, editable=False, related_name='descendant', symmetrical=False)
 
     ebook_formats = constants.EBOOK_FORMATS
     formats = ebook_formats + ['html', 'xml']
 
     parent = models.ForeignKey('self', blank=True, null=True, related_name='children')
     ancestor = models.ManyToManyField('self', blank=True, editable=False, related_name='descendant', symmetrical=False)
 

+    cached_author = models.CharField(blank=True, max_length=240, db_index=True)
+    has_audience = models.BooleanField(default=False)
+

     objects = models.Manager()
     tagged = managers.ModelTaggedItemManager(Tag)
     tags = managers.TagDescriptor(Tag)
     objects = models.Manager()
     tagged = managers.ModelTaggedItemManager(Tag)
     tags = managers.TagDescriptor(Tag)


@@ -89,7 +114,7 @@ class Book(models.Model):
         pass
 
     class Meta:
         pass
 
     class Meta:

-        ordering = ('sort_key',)
+        ordering = ('sort_key_author', 'sort_key')

         verbose_name = _('book')
         verbose_name_plural = _('books')
         app_label = 'catalogue'
         verbose_name = _('book')
         verbose_name_plural = _('books')
         app_label = 'catalogue'


@@ -103,8 +128,35 @@ class Book(models.Model):
         except AttributeError:
             return ''
 
         except AttributeError:
             return ''
 

-    def author_str(self):
-        return ", ".join(str(t) for t in self.tags.filter(category='author'))
+    def authors(self):
+        return self.tags.filter(category='author')
+
+    def tag_unicode(self, category):
+        relations = prefetched_relations(self, category)
+        if relations:
+            return ', '.join(rel.tag.name for rel in relations)
+        else:
+            return ', '.join(self.tags.filter(category=category).values_list('name', flat=True))
+
+    def tags_by_category(self):
+        return split_tags(self.tags.exclude(category__in=('set', 'theme')))
+
+    def author_unicode(self):
+        return self.cached_author
+
+    def translator(self):
+        translators = self.extra_info.get('translators')
+        if not translators:
+            return None
+        if len(translators) > 3:
+            translators = translators[:2]
+            others = ' i inni'
+        else:
+            others = ''
+        return ', '.join(u'\xa0'.join(reversed(translator.split(', ', 1))) for translator in translators) + others
+
+    def cover_source(self):
+        return self.extra_info.get('cover_source', self.parent.cover_source() if self.parent else '')

 
     def save(self, force_insert=False, force_update=False, **kwargs):
         from sortify import sortify
 
     def save(self, force_insert=False, force_update=False, **kwargs):
         from sortify import sortify


@@ -113,11 +165,14 @@ class Book(models.Model):
         self.title = unicode(self.title)  # ???
 
         try:
         self.title = unicode(self.title)  # ???
 
         try:

-            author = self.tags.filter(category='author')[0].sort_key
-        except IndexError:
+            author = self.authors().first().sort_key
+        except AttributeError:

             author = u''
         self.sort_key_author = author
 
             author = u''
         self.sort_key_author = author
 

+        self.cached_author = self.tag_unicode('author')
+        self.has_audience = 'audience' in self.extra_info
+

         ret = super(Book, self).save(force_insert, force_update, **kwargs)
 
         return ret
         ret = super(Book, self).save(force_insert, force_update, **kwargs)
 
         return ret


@@ -131,6 +186,12 @@ class Book(models.Model):
     def create_url(slug):
         return 'catalogue.views.book_detail', [slug]
 
     def create_url(slug):
         return 'catalogue.views.book_detail', [slug]
 

+    def gallery_path(self):
+        return gallery_path(self.slug)
+
+    def gallery_url(self):
+        return gallery_url(self.slug)
+

     @property
     def name(self):
         return self.title
     @property
     def name(self):
         return self.title


@@ -141,12 +202,18 @@ class Book(models.Model):
     def language_name(self):
         return dict(settings.LANGUAGES).get(self.language_code(), "")
 
     def language_name(self):
         return dict(settings.LANGUAGES).get(self.language_code(), "")
 

+    def is_foreign(self):
+        return self.language_code() != settings.LANGUAGE_CODE
+

     def has_media(self, type_):
         if type_ in Book.formats:
             return bool(getattr(self, "%s_file" % type_))
         else:
             return self.media.filter(type=type_).exists()
 
     def has_media(self, type_):
         if type_ in Book.formats:
             return bool(getattr(self, "%s_file" % type_))
         else:
             return self.media.filter(type=type_).exists()
 

+    def has_audio(self):
+        return self.has_media('mp3')
+

     def get_media(self, type_):
         if self.has_media(type_):
             if type_ in Book.formats:
     def get_media(self, type_):
         if self.has_media(type_):
             if type_ in Book.formats:


@@ -189,6 +256,33 @@ class Book(models.Model):
     has_daisy_file.short_description = 'DAISY'
     has_daisy_file.boolean = True
 
     has_daisy_file.short_description = 'DAISY'
     has_daisy_file.boolean = True
 

+    def get_audiobooks(self):
+        ogg_files = {}
+        for m in self.media.filter(type='ogg').order_by().iterator():
+            ogg_files[m.name] = m
+
+        audiobooks = []
+        projects = set()
+        for mp3 in self.media.filter(type='mp3').iterator():
+            # ogg files are always from the same project
+            meta = mp3.extra_info
+            project = meta.get('project')
+            if not project:
+                # temporary fallback
+                project = u'CzytamySłuchając'
+
+            projects.add((project, meta.get('funded_by', '')))
+
+            media = {'mp3': mp3}
+
+            ogg = ogg_files.get(mp3.name)
+            if ogg:
+                media['ogg'] = ogg
+            audiobooks.append(media)
+
+        projects = sorted(projects)
+        return audiobooks, projects
+

     def wldocument(self, parse_dublincore=True, inherit=True):
         from catalogue.import_utils import ORMDocProvider
         from librarian.parser import WLDocument
     def wldocument(self, parse_dublincore=True, inherit=True):
         from catalogue.import_utils import ORMDocProvider
         from librarian.parser import WLDocument


@@ -236,6 +330,21 @@ class Book(models.Model):
             index.index.rollback()
             raise e
 
             index.index.rollback()
             raise e
 

+    def download_pictures(self, remote_gallery_url):
+        gallery_path = self.gallery_path()
+        # delete previous files, so we don't include old files in ebooks
+        if os.path.isdir(gallery_path):
+            for filename in os.listdir(gallery_path):
+                file_path = os.path.join(gallery_path, filename)
+                os.unlink(file_path)
+        ilustr_elements = list(self.wldocument().edoc.findall('//ilustr'))
+        if ilustr_elements:
+            makedirs(gallery_path)
+            for ilustr in ilustr_elements:
+                ilustr_src = ilustr.get('src')
+                ilustr_path = os.path.join(gallery_path, ilustr_src)
+                urllib.urlretrieve('%s/%s' % (remote_gallery_url, ilustr_src), ilustr_path)
+

     @classmethod
     def from_xml_file(cls, xml_file, **kwargs):
         from django.core.files import File
     @classmethod
     def from_xml_file(cls, xml_file, **kwargs):
         from django.core.files import File


@@ -254,7 +363,7 @@ class Book(models.Model):
 
     @classmethod
     def from_text_and_meta(cls, raw_file, book_info, overwrite=False, dont_build=None, search_index=True,
 
     @classmethod
     def from_text_and_meta(cls, raw_file, book_info, overwrite=False, dont_build=None, search_index=True,

-                           search_index_tags=True):
+                           search_index_tags=True, remote_gallery_url=None):

         if dont_build is None:
             dont_build = set()
         dont_build = set.union(set(dont_build), set(app_settings.DONT_BUILD))
         if dont_build is None:
             dont_build = set()
         dont_build = set.union(set(dont_build), set(app_settings.DONT_BUILD))


@@ -298,6 +407,11 @@ class Book(models.Model):
 
         meta_tags = Tag.tags_from_info(book_info)
 
 
         meta_tags = Tag.tags_from_info(book_info)
 

+        for tag in meta_tags:
+            if not tag.for_books:
+                tag.for_books = True
+                tag.save()
+

         book.tags = set(meta_tags + book_shelves)
 
         cover_changed = old_cover != book.cover_info()
         book.tags = set(meta_tags + book_shelves)
 
         cover_changed = old_cover != book.cover_info()


@@ -322,12 +436,17 @@ class Book(models.Model):
         cls.repopulate_ancestors()
         tasks.update_counters.delay()
 
         cls.repopulate_ancestors()
         tasks.update_counters.delay()
 

+        if remote_gallery_url:
+            book.download_pictures(remote_gallery_url)
+

         # No saves beyond this point.
 
         # Build cover.
         if 'cover' not in dont_build:
             book.cover.build_delay()
             book.cover_thumb.build_delay()
         # No saves beyond this point.
 
         # Build cover.
         if 'cover' not in dont_build:
             book.cover.build_delay()
             book.cover_thumb.build_delay()

+            book.cover_api_thumb.build_delay()
+            book.simple_cover.build_delay()

 
         # Build HTML and ebooks.
         book.html_file.build_delay()
 
         # Build HTML and ebooks.
         book.html_file.build_delay()


@@ -345,6 +464,7 @@ class Book(models.Model):
         for child in notify_cover_changed:
             child.parent_cover_changed()
 
         for child in notify_cover_changed:
             child.parent_cover_changed()
 

+        book.save()  # update sort_key_author

         cls.published.send(sender=cls, instance=book)
         return book
 
         cls.published.send(sender=cls, instance=book)
         return book
 


@@ -429,6 +549,8 @@ class Book(models.Model):
             if 'cover' not in app_settings.DONT_BUILD:
                 self.cover.build_delay()
                 self.cover_thumb.build_delay()
             if 'cover' not in app_settings.DONT_BUILD:
                 self.cover.build_delay()
                 self.cover_thumb.build_delay()

+                self.cover_api_thumb.build_delay()
+                self.simple_cover.build_delay()

             for format_ in constants.EBOOK_FORMATS_WITH_COVERS:
                 if format_ not in app_settings.DONT_BUILD:
                     getattr(self, '%s_file' % format_).build_delay()
             for format_ in constants.EBOOK_FORMATS_WITH_COVERS:
                 if format_ not in app_settings.DONT_BUILD:
                     getattr(self, '%s_file' % format_).build_delay()


@@ -448,7 +570,7 @@ class Book(models.Model):
         return books
 
     def pretty_title(self, html_links=False):
         return books
 
     def pretty_title(self, html_links=False):

-        names = [(tag.name, tag.get_absolute_url()) for tag in self.tags.filter(category='author')]
+        names = [(tag.name, tag.get_absolute_url()) for tag in self.authors().only('name', 'category', 'slug')]

         books = self.parents() + [self]
         names.extend([(b.title, b.get_absolute_url()) for b in books])
 
         books = self.parents() + [self]
         names.extend([(b.title, b.get_absolute_url()) for b in books])
 


@@ -458,6 +580,13 @@ class Book(models.Model):
             names = [tag[0] for tag in names]
         return ', '.join(names)
 
             names = [tag[0] for tag in names]
         return ', '.join(names)
 

+    def publisher(self):
+        publisher = self.extra_info['publisher']
+        if isinstance(publisher, basestring):
+            return publisher
+        elif isinstance(publisher, list):
+            return ', '.join(publisher)
+

     @classmethod
     def tagged_top_level(cls, tags):
         """ Returns top-level books tagged with `tags`.
     @classmethod
     def tagged_top_level(cls, tags):
         """ Returns top-level books tagged with `tags`.


@@ -478,8 +607,7 @@ class Book(models.Model):
         """
 
         books_by_parent = {}
         """
 
         books_by_parent = {}

-        books = cls.objects.all().order_by('parent_number', 'sort_key').only(
-                'title', 'parent', 'slug')
+        books = cls.objects.order_by('parent_number', 'sort_key').only('title', 'parent', 'slug')

         if book_filter:
             books = books.filter(book_filter).distinct()
 
         if book_filter:
             books = books.filter(book_filter).distinct()
 


@@ -499,7 +627,7 @@ class Book(models.Model):
             books_by_author[tag] = []
 
         for book in books_by_parent.get(None, ()):
             books_by_author[tag] = []
 
         for book in books_by_parent.get(None, ()):

-            authors = list(book.tags.filter(category='author'))
+            authors = list(book.authors().only('pk'))

             if authors:
                 for author in authors:
                     books_by_author[author].append(book)
             if authors:
                 for author in authors:
                     books_by_author[author].append(book)


@@ -512,6 +640,7 @@ class Book(models.Model):
         "SP": (1, u"szkoła podstawowa"),
         "SP1": (1, u"szkoła podstawowa"),
         "SP2": (1, u"szkoła podstawowa"),
         "SP": (1, u"szkoła podstawowa"),
         "SP1": (1, u"szkoła podstawowa"),
         "SP2": (1, u"szkoła podstawowa"),

+        "SP3": (1, u"szkoła podstawowa"),

         "P": (1, u"szkoła podstawowa"),
         "G": (2, u"gimnazjum"),
         "L": (3, u"liceum"),
         "P": (1, u"szkoła podstawowa"),
         "G": (2, u"gimnazjum"),
         "L": (3, u"liceum"),


@@ -544,6 +673,25 @@ class Book(models.Model):
         else:
             return None
 
         else:
             return None
 

+    def fragment_data(self):
+        fragment = self.choose_fragment()
+        if fragment:
+            return {'title': fragment.book.pretty_title(), 'html': fragment.get_short_text()}
+        else:
+            return None
+
+    def update_popularity(self):
+        count = self.tags.filter(category='set').values('user').order_by('user').distinct().count()
+        try:
+            pop = self.popularity
+            pop.count = count
+            pop.save()
+        except BookPopularity.DoesNotExist:
+            BookPopularity.objects.create(book=self, count=count)
+
+    def ridero_link(self):
+        return 'https://ridero.eu/%s/books/wl_%s/' % (get_language(), self.slug.replace('-', '_'))
+

 
 def add_file_fields():
     for format_ in Book.formats:
 
 def add_file_fields():
     for format_ in Book.formats:


@@ -563,3 +711,8 @@ def add_file_fields():
         ).contribute_to_class(Book, field_name)
 
 add_file_fields()
         ).contribute_to_class(Book, field_name)
 
 add_file_fields()

+
+
+class BookPopularity(models.Model):
+    book = models.OneToOneField(Book, related_name='popularity')
+    count = models.IntegerField(default=0)