fix for checking paths in waiter

[wolnelektury.git] / apps / catalogue / models.py

diff --git a/apps/catalogue/models.py b/apps/catalogue/models.py
index ba1a5d2..29106b1 100644 (file)
--- a/apps/catalogue/models.py
+++ b/apps/catalogue/models.py
@@ -22,12 +22,11 @@ from django.conf import settings
 
 from newtagging.models import TagBase, tags_updated
 from newtagging import managers
 
 from newtagging.models import TagBase, tags_updated
 from newtagging import managers

-from catalogue.fields import JSONField, OverwritingFileField
+from catalogue.fields import OverwritingFileField

 from catalogue.utils import create_zip, split_tags, truncate_html_words
 from catalogue import tasks
 import re
 
 from catalogue.utils import create_zip, split_tags, truncate_html_words
 from catalogue import tasks
 import re
 

-import search

 
 # Those are hard-coded here so that makemessages sees them.
 TAG_CATEGORIES = (
 
 # Those are hard-coded here so that makemessages sees them.
 TAG_CATEGORIES = (


@@ -54,6 +53,10 @@ class TagSubcategoryManager(models.Manager):
 
 
 class Tag(TagBase):
 
 
 class Tag(TagBase):

+    """A tag attachable to books and fragments (and possibly anything).
+    
+    Used to represent searchable metadata (authors, epochs, genres, kinds),
+    fragment themes (motifs) and some book hierarchy related kludges."""

     name = models.CharField(_('name'), max_length=50, db_index=True)
     slug = models.SlugField(_('slug'), max_length=120, db_index=True)
     sort_key = models.CharField(_('sort key'), max_length=120, db_index=True)
     name = models.CharField(_('name'), max_length=50, db_index=True)
     slug = models.SlugField(_('slug'), max_length=120, db_index=True)
     sort_key = models.CharField(_('sort key'), max_length=120, db_index=True)


@@ -115,8 +118,8 @@ class Tag(TagBase):
             objects = Book.tagged.with_all((self,)).order_by()
             if self.category != 'set':
                 # eliminate descendants
             objects = Book.tagged.with_all((self,)).order_by()
             if self.category != 'set':
                 # eliminate descendants

-                l_tags = Tag.objects.filter(slug__in=[book.book_tag_slug() for book in objects])
-                descendants_keys = [book.pk for book in Book.tagged.with_any(l_tags)]
+                l_tags = Tag.objects.filter(slug__in=[book.book_tag_slug() for book in objects.iterator()])
+                descendants_keys = [book.pk for book in Book.tagged.with_any(l_tags).iterator()]

                 if descendants_keys:
                     objects = objects.exclude(pk__in=descendants_keys)
         return objects.count()
                 if descendants_keys:
                     objects = objects.exclude(pk__in=descendants_keys)
         return objects.count()


@@ -213,6 +216,7 @@ def book_upload_path(ext=None, maxlen=100):
 
 
 class BookMedia(models.Model):
 
 
 class BookMedia(models.Model):

+    """Represents media attached to a book."""

     FileFormat = namedtuple("FileFormat", "name ext")
     formats = SortedDict([
         ('mp3', FileFormat(name='MP3', ext='mp3')),
     FileFormat = namedtuple("FileFormat", "name ext")
     formats = SortedDict([
         ('mp3', FileFormat(name='MP3', ext='mp3')),


@@ -226,7 +230,7 @@ class BookMedia(models.Model):
     name        = models.CharField(_('name'), max_length="100")
     file        = OverwritingFileField(_('file'), upload_to=book_upload_path())
     uploaded_at = models.DateTimeField(_('creation date'), auto_now_add=True, editable=False)
     name        = models.CharField(_('name'), max_length="100")
     file        = OverwritingFileField(_('file'), upload_to=book_upload_path())
     uploaded_at = models.DateTimeField(_('creation date'), auto_now_add=True, editable=False)

-    extra_info  = JSONField(_('extra information'), default='{}', editable=False)
+    extra_info  = jsonfield.JSONField(_('extra information'), default='{}', editable=False)

     book = models.ForeignKey('Book', related_name='media')
     source_sha1 = models.CharField(null=True, blank=True, max_length=40, editable=False)
 
     book = models.ForeignKey('Book', related_name='media')
     source_sha1 = models.CharField(null=True, blank=True, max_length=40, editable=False)
 


@@ -258,9 +262,9 @@ class BookMedia(models.Model):
             remove_zip("%s_%s" % (old.book.slug, old.type))
         remove_zip("%s_%s" % (self.book.slug, self.type))
 
             remove_zip("%s_%s" % (old.book.slug, old.type))
         remove_zip("%s_%s" % (self.book.slug, self.type))
 

-        extra_info = self.get_extra_info_value()
+        extra_info = self.extra_info

         extra_info.update(self.read_meta())
         extra_info.update(self.read_meta())

-        self.set_extra_info_value(extra_info)
+        self.extra_info = extra_info

         self.source_sha1 = self.read_source_sha1(self.file.path, self.type)
         return super(BookMedia, self).save(*args, **kwargs)
 
         self.source_sha1 = self.read_source_sha1(self.file.path, self.type)
         return super(BookMedia, self).save(*args, **kwargs)
 


@@ -323,6 +327,7 @@ class BookMedia(models.Model):
 
 
 class Book(models.Model):
 
 
 class Book(models.Model):

+    """Represents a book imported from WL-XML."""

     title         = models.CharField(_('title'), max_length=120)
     sort_key = models.CharField(_('sort key'), max_length=120, db_index=True, editable=False)
     slug = models.SlugField(_('slug'), max_length=120, db_index=True,
     title         = models.CharField(_('title'), max_length=120)
     sort_key = models.CharField(_('sort key'), max_length=120, db_index=True, editable=False)
     slug = models.SlugField(_('slug'), max_length=120, db_index=True,


@@ -334,7 +339,7 @@ class Book(models.Model):
     created_at    = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True)
     changed_at    = models.DateTimeField(_('creation date'), auto_now=True, db_index=True)
     parent_number = models.IntegerField(_('parent number'), default=0)
     created_at    = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True)
     changed_at    = models.DateTimeField(_('creation date'), auto_now=True, db_index=True)
     parent_number = models.IntegerField(_('parent number'), default=0)

-    extra_info    = JSONField(_('extra information'), default='{}')
+    extra_info    = jsonfield.JSONField(_('extra information'), default='{}')

     gazeta_link   = models.CharField(blank=True, max_length=240)
     wiki_link     = models.CharField(blank=True, max_length=240)
     # files generated during publication
     gazeta_link   = models.CharField(blank=True, max_length=240)
     wiki_link     = models.CharField(blank=True, max_length=240)
     # files generated during publication


@@ -428,7 +433,7 @@ class Book(models.Model):
 
         type(self).objects.filter(pk=self.pk).update(_related_info=None)
         # Fragment.short_html relies on book's tags, so reset it here too
 
         type(self).objects.filter(pk=self.pk).update(_related_info=None)
         # Fragment.short_html relies on book's tags, so reset it here too

-        for fragm in self.fragments.all():
+        for fragm in self.fragments.all().iterator():

             fragm.reset_short_html()
 
     def has_description(self):
             fragm.reset_short_html()
 
     def has_description(self):


@@ -533,26 +538,30 @@ class Book(models.Model):
 
     # Thin wrappers for builder tasks
     def build_pdf(self, *args, **kwargs):
 
     # Thin wrappers for builder tasks
     def build_pdf(self, *args, **kwargs):

+        """(Re)builds PDF."""

         return tasks.build_pdf.delay(self.pk, *args, **kwargs)
     def build_epub(self, *args, **kwargs):
         return tasks.build_pdf.delay(self.pk, *args, **kwargs)
     def build_epub(self, *args, **kwargs):

+        """(Re)builds EPUB."""

         return tasks.build_epub.delay(self.pk, *args, **kwargs)
     def build_mobi(self, *args, **kwargs):
         return tasks.build_epub.delay(self.pk, *args, **kwargs)
     def build_mobi(self, *args, **kwargs):

+        """(Re)builds MOBI."""

         return tasks.build_mobi.delay(self.pk, *args, **kwargs)
     def build_txt(self, *args, **kwargs):
         return tasks.build_mobi.delay(self.pk, *args, **kwargs)
     def build_txt(self, *args, **kwargs):

+        """(Re)builds TXT."""

         return tasks.build_txt.delay(self.pk, *args, **kwargs)
 
     @staticmethod
     def zip_format(format_):
         def pretty_file_name(book):
             return "%s/%s.%s" % (
         return tasks.build_txt.delay(self.pk, *args, **kwargs)
 
     @staticmethod
     def zip_format(format_):
         def pretty_file_name(book):
             return "%s/%s.%s" % (

-                b.get_extra_info_value()['author'],
+                b.extra_info['author'],

                 b.slug,
                 format_)
 
         field_name = "%s_file" % format_
         books = Book.objects.filter(parent=None).exclude(**{field_name: ""})
         paths = [(pretty_file_name(b), getattr(b, field_name).path)
                 b.slug,
                 format_)
 
         field_name = "%s_file" % format_
         books = Book.objects.filter(parent=None).exclude(**{field_name: ""})
         paths = [(pretty_file_name(b), getattr(b, field_name).path)

-                    for b in books]
+                    for b in books.iterator()]

         return create_zip(paths,
                     getattr(settings, "ALL_%s_ZIP" % format_.upper()))
 
         return create_zip(paths,
                     getattr(settings, "ALL_%s_ZIP" % format_.upper()))
 


@@ -562,6 +571,7 @@ class Book(models.Model):
         return create_zip(paths, "%s_%s" % (self.slug, format_))
 
     def search_index(self, book_info=None, reuse_index=False, index_tags=True):
         return create_zip(paths, "%s_%s" % (self.slug, format_))
 
     def search_index(self, book_info=None, reuse_index=False, index_tags=True):

+        import search

         if reuse_index:
             idx = search.ReusableIndex()
         else:
         if reuse_index:
             idx = search.ReusableIndex()
         else:


@@ -628,7 +638,7 @@ class Book(models.Model):
             book.common_slug = book_info.variant_of.slug
         else:
             book.common_slug = book.slug
             book.common_slug = book_info.variant_of.slug
         else:
             book.common_slug = book.slug

-        book.set_extra_info_value(book_info.to_dict())
+        book.extra_info = book_info.to_dict()

         book.save()
 
         meta_tags = Tag.tags_from_info(book_info)
         book.save()
 
         meta_tags = Tag.tags_from_info(book_info)


@@ -675,7 +685,7 @@ class Book(models.Model):
             descendants_tags.update(child_book.tags)
             child_book.tags = list(child_book.tags) + [book_tag]
             child_book.save()
             descendants_tags.update(child_book.tags)
             child_book.tags = list(child_book.tags) + [book_tag]
             child_book.save()

-            for fragment in child_book.fragments.all():
+            for fragment in child_book.fragments.all().iterator():

                 fragment.tags = set(list(fragment.tags) + [book_tag])
             book_descendants += list(child_book.children.all())
 
                 fragment.tags = set(list(fragment.tags) + [book_tag])
             book_descendants += list(child_book.children.all())
 


@@ -723,7 +733,7 @@ class Book(models.Model):
 
     def related_themes(self):
         theme_counter = self.theme_counter
 
     def related_themes(self):
         theme_counter = self.theme_counter

-        book_themes = Tag.objects.filter(pk__in=theme_counter.keys())
+        book_themes = list(Tag.objects.filter(pk__in=theme_counter.keys()))

         for tag in book_themes:
             tag.count = theme_counter[tag.pk]
         return book_themes
         for tag in book_themes:
             tag.count = theme_counter[tag.pk]
         return book_themes


@@ -747,10 +757,10 @@ class Book(models.Model):
 
         if tags is None:
             tags = {}
 
         if tags is None:
             tags = {}

-            for child in self.children.all().order_by():
+            for child in self.children.all().order_by().iterator():

                 for tag_pk, value in child.tag_counter.iteritems():
                     tags[tag_pk] = tags.get(tag_pk, 0) + value
                 for tag_pk, value in child.tag_counter.iteritems():
                     tags[tag_pk] = tags.get(tag_pk, 0) + value

-            for tag in self.tags.exclude(category__in=('book', 'theme', 'set')).order_by():
+            for tag in self.tags.exclude(category__in=('book', 'theme', 'set')).order_by().iterator():

                 tags[tag.pk] = 1
 
             if self.id:
                 tags[tag.pk] = 1
 
             if self.id:


@@ -776,8 +786,8 @@ class Book(models.Model):
 
         if tags is None:
             tags = {}
 
         if tags is None:
             tags = {}

-            for fragment in Fragment.tagged.with_any([self.book_tag()]).order_by():
-                for tag in fragment.tags.filter(category='theme').order_by():
+            for fragment in Fragment.tagged.with_any([self.book_tag()]).order_by().iterator():
+                for tag in fragment.tags.filter(category='theme').order_by().iterator():

                     tags[tag.pk] = tags.get(tag.pk, 0) + 1
 
             if self.id:
                     tags[tag.pk] = tags.get(tag.pk, 0) + 1
 
             if self.id:


@@ -803,7 +813,7 @@ class Book(models.Model):
 
     @classmethod
     def tagged_top_level(cls, tags):
 
     @classmethod
     def tagged_top_level(cls, tags):

-        """ Returns top-level books tagged with `tags'.
+        """ Returns top-level books tagged with `tags`.

 
         It only returns those books which don't have ancestors which are
         also tagged with those tags.
 
         It only returns those books which don't have ancestors which are
         also tagged with those tags.


@@ -812,8 +822,9 @@ class Book(models.Model):
         # get relevant books and their tags
         objects = cls.tagged.with_all(tags)
         # eliminate descendants
         # get relevant books and their tags
         objects = cls.tagged.with_all(tags)
         # eliminate descendants

-        l_tags = Tag.objects.filter(category='book', slug__in=[book.book_tag_slug() for book in objects])
-        descendants_keys = [book.pk for book in cls.tagged.with_any(l_tags)]
+        l_tags = Tag.objects.filter(category='book',
+            slug__in=[book.book_tag_slug() for book in objects.iterator()])
+        descendants_keys = [book.pk for book in cls.tagged.with_any(l_tags).iterator()]

         if descendants_keys:
             objects = objects.exclude(pk__in=descendants_keys)
 
         if descendants_keys:
             objects = objects.exclude(pk__in=descendants_keys)
 


@@ -832,19 +843,20 @@ class Book(models.Model):
                 'title', 'parent', 'slug')
         if filter:
             books = books.filter(filter).distinct()
                 'title', 'parent', 'slug')
         if filter:
             books = books.filter(filter).distinct()

-            book_ids = set((book.pk for book in books))
-            for book in books:
+            
+            book_ids = set(b['pk'] for b in books.values("pk").iterator())
+            for book in books.iterator():

                 parent = book.parent_id
                 if parent not in book_ids:
                     parent = None
                 books_by_parent.setdefault(parent, []).append(book)
         else:
                 parent = book.parent_id
                 if parent not in book_ids:
                     parent = None
                 books_by_parent.setdefault(parent, []).append(book)
         else:

-            for book in books:
+            for book in books.iterator():

                 books_by_parent.setdefault(book.parent_id, []).append(book)
 
         orphans = []
         books_by_author = SortedDict()
                 books_by_parent.setdefault(book.parent_id, []).append(book)
 
         orphans = []
         books_by_author = SortedDict()

-        for tag in Tag.objects.filter(category='author'):
+        for tag in Tag.objects.filter(category='author').iterator():

             books_by_author[tag] = []
 
         for book in books_by_parent.get(None,()):
             books_by_author[tag] = []
 
         for book in books_by_parent.get(None,()):


@@ -866,7 +878,7 @@ class Book(models.Model):
         "LP": (3, u"liceum"),
     }
     def audiences_pl(self):
         "LP": (3, u"liceum"),
     }
     def audiences_pl(self):

-        audiences = self.get_extra_info_value().get('audiences', [])
+        audiences = self.extra_info.get('audiences', [])

         audiences = sorted(set([self._audiences_pl[a] for a in audiences]))
         return [a[1] for a in audiences]
 
         audiences = sorted(set([self._audiences_pl[a] for a in audiences]))
         return [a[1] for a in audiences]
 


@@ -883,7 +895,8 @@ class Book(models.Model):
 
 def _has_factory(ftype):
     has = lambda self: bool(getattr(self, "%s_file" % ftype))
 
 def _has_factory(ftype):
     has = lambda self: bool(getattr(self, "%s_file" % ftype))

-    has.short_description = t.upper()
+    has.short_description = ftype.upper()
+    has.__doc__ = None

     has.boolean = True
     has.__name__ = "has_%s_file" % ftype
     return has
     has.boolean = True
     has.__name__ = "has_%s_file" % ftype
     return has


@@ -900,6 +913,7 @@ for t in Book.formats:
 
 
 class Fragment(models.Model):
 
 
 class Fragment(models.Model):

+    """Represents a themed fragment of a book."""

     text = models.TextField()
     short_text = models.TextField(editable=False)
     anchor = models.CharField(max_length=120)
     text = models.TextField()
     short_text = models.TextField(editable=False)
     anchor = models.CharField(max_length=120)


@@ -1004,14 +1018,16 @@ def _post_save_handler(sender, instance, **kwargs):
 post_save.connect(_post_save_handler)
 
 
 post_save.connect(_post_save_handler)
 
 

-@django.dispatch.receiver(post_delete, sender=Book)
-def _remove_book_from_index_handler(sender, instance, **kwargs):
-    """ remove the book from search index, when it is deleted."""
-    search.JVM.attachCurrentThread()
-    idx = search.Index()
-    idx.open(timeout=10000)  # 10 seconds timeout.
-    try:
-        idx.remove_book(instance)
-        idx.index_tags()
-    finally:
-        idx.close()
+if not settings.NO_SEARCH_INDEX:
+    @django.dispatch.receiver(post_delete, sender=Book)
+    def _remove_book_from_index_handler(sender, instance, **kwargs):
+        """ remove the book from search index, when it is deleted."""
+        import search
+        search.JVM.attachCurrentThread()
+        idx = search.Index()
+        idx.open(timeout=10000)  # 10 seconds timeout.
+        try:
+            idx.remove_book(instance)
+            idx.index_tags()
+        finally:
+            idx.close()