log parallel job exceptions

[wolnelektury.git] / apps / catalogue / models.py

diff --git a/apps/catalogue/models.py b/apps/catalogue/models.py
index d0a620a..0863d5b 100644 (file)
--- a/apps/catalogue/models.py
+++ b/apps/catalogue/models.py
@@ -6,6 +6,7 @@ from datetime import datetime
 
 from django.db import models
 from django.db.models import permalink, Q
 
 from django.db import models
 from django.db.models import permalink, Q

+import django.dispatch

 from django.core.cache import cache
 from django.utils.translation import ugettext_lazy as _
 from django.contrib.auth.models import User
 from django.core.cache import cache
 from django.utils.translation import ugettext_lazy as _
 from django.contrib.auth.models import User


@@ -21,14 +22,16 @@ from django.conf import settings
 from newtagging.models import TagBase, tags_updated
 from newtagging import managers
 from catalogue.fields import JSONField, OverwritingFileField
 from newtagging.models import TagBase, tags_updated
 from newtagging import managers
 from catalogue.fields import JSONField, OverwritingFileField

-from catalogue.utils import ExistingFile
+from catalogue.utils import ExistingFile, ORMDocProvider, create_zip, remove_zip

 
 from librarian import dcparser, html, epub, NoDublinCore
 import mutagen
 from mutagen import id3
 from slughifi import slughifi
 from sortify import sortify
 
 from librarian import dcparser, html, epub, NoDublinCore
 import mutagen
 from mutagen import id3
 from slughifi import slughifi
 from sortify import sortify

+from os import unlink

 
 

+import search

 
 TAG_CATEGORIES = (
     ('author', _('author')),
 
 TAG_CATEGORIES = (
     ('author', _('author')),


@@ -50,6 +53,7 @@ MEDIA_FORMATS = (
 # not quite, but Django wants you to set a timeout
 CACHE_FOREVER = 2419200  # 28 days
 
 # not quite, but Django wants you to set a timeout
 CACHE_FOREVER = 2419200  # 28 days
 

+

 class TagSubcategoryManager(models.Manager):
     def __init__(self, subcategory):
         super(TagSubcategoryManager, self).__init__()
 class TagSubcategoryManager(models.Manager):
     def __init__(self, subcategory):
         super(TagSubcategoryManager, self).__init__()


@@ -219,6 +223,10 @@ class BookMedia(models.Model):
                 self.file.save(None, ExistingFile(self.file.path), save=False, leave=True)
 
         super(BookMedia, self).save(*args, **kwargs)
                 self.file.save(None, ExistingFile(self.file.path), save=False, leave=True)
 
         super(BookMedia, self).save(*args, **kwargs)

+
+        # remove the zip package for book with modified media
+        remove_zip(self.book.slug)
+

         extra_info = self.get_extra_info_value()
         extra_info.update(self.read_meta())
         self.set_extra_info_value(extra_info)
         extra_info = self.get_extra_info_value()
         extra_info.update(self.read_meta())
         self.set_extra_info_value(extra_info)


@@ -281,7 +289,7 @@ class BookMedia(models.Model):
 
 class Book(models.Model):
     title         = models.CharField(_('title'), max_length=120)
 
 class Book(models.Model):
     title         = models.CharField(_('title'), max_length=120)

-    sort_key = models.CharField(_('sort_key'), max_length=120, db_index=True, editable=False)
+    sort_key = models.CharField(_('sort key'), max_length=120, db_index=True, editable=False)

     slug          = models.SlugField(_('slug'), max_length=120, unique=True, db_index=True)
     description   = models.TextField(_('description'), blank=True)
     created_at    = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True)
     slug          = models.SlugField(_('slug'), max_length=120, unique=True, db_index=True)
     description   = models.TextField(_('description'), blank=True)
     created_at    = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True)


@@ -291,17 +299,16 @@ class Book(models.Model):
     gazeta_link   = models.CharField(blank=True, max_length=240)
     wiki_link     = models.CharField(blank=True, max_length=240)
     # files generated during publication
     gazeta_link   = models.CharField(blank=True, max_length=240)
     wiki_link     = models.CharField(blank=True, max_length=240)
     # files generated during publication

-    xml_file      = models.FileField(_('XML file'), upload_to=book_upload_path('xml'), blank=True)
-    html_file     = models.FileField(_('HTML file'), upload_to=book_upload_path('html'), blank=True)
-    pdf_file      = models.FileField(_('PDF file'), upload_to=book_upload_path('pdf'), blank=True)
-    epub_file     = models.FileField(_('EPUB file'), upload_to=book_upload_path('epub'), blank=True)    
-    txt_file      = models.FileField(_('TXT file'), upload_to=book_upload_path('txt'), blank=True)        

 
 

+    file_types = ['epub', 'html', 'mobi', 'pdf', 'txt', 'xml']
+    

     parent        = models.ForeignKey('self', blank=True, null=True, related_name='children')
     objects  = models.Manager()
     tagged   = managers.ModelTaggedItemManager(Tag)
     tags     = managers.TagDescriptor(Tag)
 
     parent        = models.ForeignKey('self', blank=True, null=True, related_name='children')
     objects  = models.Manager()
     tagged   = managers.ModelTaggedItemManager(Tag)
     tags     = managers.TagDescriptor(Tag)
 

+    html_built = django.dispatch.Signal()
+

     class AlreadyExists(Exception):
         pass
 
     class AlreadyExists(Exception):
         pass
 


@@ -344,49 +351,15 @@ class Book(models.Model):
         return book_tag
 
     def has_media(self, type):
         return book_tag
 
     def has_media(self, type):

-        if   type == 'xml':
-            if self.xml_file:
-                return True
-            else:
-                return False
-        elif type == 'html':
-            if self.html_file:
-                return True
-            else:
-                return False        
-        elif type == 'txt':
-            if self.txt_file:
-                return True
-            else:
-                return False        
-        elif type == 'pdf':
-            if self.pdf_file:
-                return True
-            else:
-                return False  
-        elif type == 'epub':
-            if self.epub_file:
-                return True
-            else:
-                return False                          
+        if type in Book.file_types:
+            return bool(getattr(self, "%s_file" % type))

         else:
         else:

-            if self.media.filter(type=type).exists():
-                return True
-            else:
-                return False
+            return self.media.filter(type=type).exists()

 
     def get_media(self, type):
         if self.has_media(type):
 
     def get_media(self, type):
         if self.has_media(type):

-            if   type == "xml":
-                return self.xml_file
-            elif type == "html":
-                return self.html_file
-            elif type == "epub":
-                return self.epub_file
-            elif type == "txt":
-                return self.txt_file
-            elif type == "pdf":
-                return self.pdf_file
+            if type in Book.file_types:
+                return getattr(self, "%s_file" % type)

             else:                                             
                 return self.media.filter(type=type)
         else:
             else:                                             
                 return self.media.filter(type=type)
         else:


@@ -426,11 +399,13 @@ class Book(models.Model):
             tags = [mark_safe(u'<a href="%s">%s</a>' % (tag.get_absolute_url(), tag.name)) for tag in tags]
 
             formats = []
             tags = [mark_safe(u'<a href="%s">%s</a>' % (tag.get_absolute_url(), tag.name)) for tag in tags]
 
             formats = []

-            # files generated during publication               
+            # files generated during publication

             if self.has_media("html"):
                 formats.append(u'<a href="%s">%s</a>' % (reverse('book_text', kwargs={'slug': self.slug}), _('Read online')))
             if self.has_media("pdf"):
                 formats.append(u'<a href="%s">PDF</a>' % self.get_media('pdf').url)
             if self.has_media("html"):
                 formats.append(u'<a href="%s">%s</a>' % (reverse('book_text', kwargs={'slug': self.slug}), _('Read online')))
             if self.has_media("pdf"):
                 formats.append(u'<a href="%s">PDF</a>' % self.get_media('pdf').url)

+            if self.has_media("mobi"):
+                formats.append(u'<a href="%s">MOBI</a>' % self.get_media('mobi').url)

             if self.root_ancestor.has_media("epub"):
                 formats.append(u'<a href="%s">EPUB</a>' % self.root_ancestor.get_media('epub').url)
             if self.has_media("txt"):
             if self.root_ancestor.has_media("epub"):
                 formats.append(u'<a href="%s">EPUB</a>' % self.root_ancestor.get_media('epub').url)
             if self.has_media("txt"):


@@ -466,26 +441,6 @@ class Book(models.Model):
     has_description.boolean = True
 
     # ugly ugly ugly
     has_description.boolean = True
 
     # ugly ugly ugly

-    def has_pdf_file(self):
-        return bool(self.pdf_file)
-    has_pdf_file.short_description = 'PDF'
-    has_pdf_file.boolean = True
-
-    def has_epub_file(self):
-        return bool(self.epub_file)
-    has_epub_file.short_description = 'EPUB'
-    has_epub_file.boolean = True
-
-    def has_txt_file(self):
-        return bool(self.txt_file)
-    has_txt_file.short_description = 'HTML'
-    has_txt_file.boolean = True
-
-    def has_html_file(self):
-        return bool(self.html_file)
-    has_html_file.short_description = 'HTML'
-    has_html_file.boolean = True
-

     def has_odt_file(self):
         return bool(self.has_media("odt"))
     has_odt_file.short_description = 'ODT'
     def has_odt_file(self):
         return bool(self.has_media("odt"))
     has_odt_file.short_description = 'ODT'


@@ -500,34 +455,64 @@ class Book(models.Model):
         return bool(self.has_media("ogg"))
     has_ogg_file.short_description = 'OGG'
     has_ogg_file.boolean = True
         return bool(self.has_media("ogg"))
     has_ogg_file.short_description = 'OGG'
     has_ogg_file.boolean = True

-    
+

     def has_daisy_file(self):
         return bool(self.has_media("daisy"))
     has_daisy_file.short_description = 'DAISY'
     def has_daisy_file(self):
         return bool(self.has_media("daisy"))
     has_daisy_file.short_description = 'DAISY'

-    has_daisy_file.boolean = True    
-    
+    has_daisy_file.boolean = True
+
+    def build_pdf(self):
+        """ (Re)builds the pdf file.
+
+        """
+        from librarian import pdf
+        from tempfile import NamedTemporaryFile
+        import os
+
+        try:
+            pdf_file = NamedTemporaryFile(delete=False)
+            pdf.transform(ORMDocProvider(self),
+                      file_path=str(self.xml_file.path),
+                      output_file=pdf_file,
+                      )
+
+            self.pdf_file.save('%s.pdf' % self.slug, File(open(pdf_file.name)))
+        finally:
+            unlink(pdf_file.name)
+
+        # remove zip with all pdf files
+        remove_zip(settings.ALL_PDF_ZIP)
+
+    def build_mobi(self):
+        """ (Re)builds the MOBI file.
+
+        """
+        from librarian import mobi
+        from tempfile import NamedTemporaryFile
+        import os
+
+        try:
+            mobi_file = NamedTemporaryFile(suffix='.mobi', delete=False)
+            mobi.transform(ORMDocProvider(self), verbose=1,
+                      file_path=str(self.xml_file.path),
+                      output_file=mobi_file.name,
+                      )
+
+            self.mobi_file.save('%s.mobi' % self.slug, File(open(mobi_file.name)))
+        finally:
+            unlink(mobi_file.name)
+
+        # remove zip with all mobi files
+        remove_zip(settings.ALL_MOBI_ZIP)
+

     def build_epub(self, remove_descendants=True):
         """ (Re)builds the epub file.
             If book has a parent, does nothing.
             Unless remove_descendants is False, descendants' epubs are removed.
         """
     def build_epub(self, remove_descendants=True):
         """ (Re)builds the epub file.
             If book has a parent, does nothing.
             Unless remove_descendants is False, descendants' epubs are removed.
         """

-    

         from StringIO import StringIO
         from hashlib import sha1
         from django.core.files.base import ContentFile
         from StringIO import StringIO
         from hashlib import sha1
         from django.core.files.base import ContentFile

-        from librarian import DocProvider
-
-        class BookImportDocProvider(DocProvider):
-            """ used for joined EPUBs """
-
-            def __init__(self, book):
-                self.book = book
-
-            def by_slug(self, slug):
-                if slug == self.book.slug:
-                    return self.book.xml_file
-                else:
-                    return Book.objects.get(slug=slug).xml_file

 
         if self.parent:
             # don't need an epub
 
         if self.parent:
             # don't need an epub


@@ -535,7 +520,7 @@ class Book(models.Model):
 
         epub_file = StringIO()
         try:
 
         epub_file = StringIO()
         try:

-            epub.transform(BookImportDocProvider(self), self.slug, output_file=epub_file)
+            epub.transform(ORMDocProvider(self), self.slug, output_file=epub_file)

             self.epub_file.save('%s.epub' % self.slug, ContentFile(epub_file.getvalue()))
             FileRecord(slug=self.slug, type='epub', sha1=sha1(epub_file.getvalue()).hexdigest()).save()
         except NoDublinCore:
             self.epub_file.save('%s.epub' % self.slug, ContentFile(epub_file.getvalue()))
             FileRecord(slug=self.slug, type='epub', sha1=sha1(epub_file.getvalue()).hexdigest()).save()
         except NoDublinCore:


@@ -550,6 +535,9 @@ class Book(models.Model):
             child_book.save()
             book_descendants += list(child_book.children.all())
 
             child_book.save()
             book_descendants += list(child_book.children.all())
 

+        # remove zip package with all epub files
+        remove_zip(settings.ALL_EPUB_ZIP)
+

     def build_txt(self):
         from StringIO import StringIO
         from django.core.files.base import ContentFile
     def build_txt(self):
         from StringIO import StringIO
         from django.core.files.base import ContentFile


@@ -611,9 +599,39 @@ class Book(models.Model):
                 new_fragment.save()
                 new_fragment.tags = set(meta_tags + themes + [book_tag] + ancestor_tags)
             self.save()
                 new_fragment.save()
                 new_fragment.tags = set(meta_tags + themes + [book_tag] + ancestor_tags)
             self.save()

+            self.html_built.send(sender=self)

             return True
         return False
 
             return True
         return False
 

+    @staticmethod
+    def zip_format(format_):
+        def pretty_file_name(book):
+            return "%s/%s.%s" % (
+                b.get_extra_info_value()['author'],
+                b.slug,
+                format_)
+
+        field_name = "%s_file" % format_
+        books = Book.objects.filter(parent=None).exclude(**{field_name: ""})
+        paths = [(pretty_file_name(b), getattr(b, field_name).path)
+                    for b in books]
+        result = create_zip.delay(paths,
+                    getattr(settings, "ALL_%s_ZIP" % format_.upper()))
+        return result.wait()
+
+    def zip_audiobooks(self):
+        bm = BookMedia.objects.filter(book=self, type='mp3')
+        paths = map(lambda bm: (None, bm.file.path), bm)
+        result = create_zip.delay(paths, self.slug)
+        return result.wait()
+
+    def search_index(self):
+        idx = search.ReusableIndex()
+        idx.open()
+        try:
+            idx.index_book(self)
+        finally:
+            idx.close()

 
     @classmethod
     def from_xml_file(cls, xml_file, **kwargs):
 
     @classmethod
     def from_xml_file(cls, xml_file, **kwargs):


@@ -629,7 +647,9 @@ class Book(models.Model):
             xml_file.close()
 
     @classmethod
             xml_file.close()
 
     @classmethod

-    def from_text_and_meta(cls, raw_file, book_info, overwrite=False, build_epub=True, build_txt=True):
+    def from_text_and_meta(cls, raw_file, book_info, overwrite=False,
+            build_epub=True, build_txt=True, build_pdf=True, build_mobi=True,
+            search_index=True):

         import re
 
         # check for parts before we do anything
         import re
 
         # check for parts before we do anything


@@ -702,6 +722,15 @@ class Book(models.Model):
         if not settings.NO_BUILD_EPUB and build_epub:
             book.root_ancestor.build_epub()
 
         if not settings.NO_BUILD_EPUB and build_epub:
             book.root_ancestor.build_epub()
 

+        if not settings.NO_BUILD_PDF and build_pdf:
+            book.root_ancestor.build_pdf()
+
+        if not settings.NO_BUILD_MOBI and build_mobi:
+            book.build_mobi()
+
+        if not settings.NO_SEARCH_INDEX and search_index:
+            book.search_index()
+

         book_descendants = list(book.children.all())
         # add l-tag to descendants and their fragments
         # delete unnecessary EPUB files
         book_descendants = list(book.children.all())
         # add l-tag to descendants and their fragments
         # delete unnecessary EPUB files


@@ -813,6 +842,24 @@ class Book(models.Model):
         return objects
 
 
         return objects
 
 

+def _has_factory(ftype):
+    has = lambda self: bool(getattr(self, "%s_file" % ftype))
+    has.short_description = t.upper()
+    has.boolean = True
+    has.__name__ = "has_%s_file" % ftype
+    return has
+
+    
+# add the file fields
+for t in Book.file_types:
+    field_name = "%s_file" % t
+    models.FileField(_("%s file" % t.upper()),
+            upload_to=book_upload_path(t),
+            blank=True).contribute_to_class(Book, field_name)
+
+    setattr(Book, "has_%s_file" % t, _has_factory(t))
+
+

 class Fragment(models.Model):
     text = models.TextField()
     short_text = models.TextField(editable=False)
 class Fragment(models.Model):
     text = models.TextField()
     short_text = models.TextField(editable=False)