zip package for epub/pdf/audiobooks

diff --git a/apps/catalogue/management/commands/importbooks.py b/apps/catalogue/management/commands/importbooks.py
index f51214d..62ead53 100644 (file)
--- a/apps/catalogue/management/commands/importbooks.py
+++ b/apps/catalogue/management/commands/importbooks.py
@@ -101,6 +101,9 @@ class Command(BaseCommand):
                                 print "Importing %s.txt" % file_base
 
                         book.save()
                                 print "Importing %s.txt" % file_base
 
                         book.save()

+                        
+                        # clean the generated zip packages. Is this the right place for this?
+                        book.clean_zip_files()

 
                     except Book.AlreadyExists, msg:
                         print self.style.ERROR('%s: Book already imported. Skipping. To overwrite use --force.' %
 
                     except Book.AlreadyExists, msg:
                         print self.style.ERROR('%s: Book already imported. Skipping. To overwrite use --force.' %

diff --git a/apps/catalogue/models.py b/apps/catalogue/models.py
index c5ca5df..471c661 100644 (file)
--- a/apps/catalogue/models.py
+++ b/apps/catalogue/models.py
@@ -22,7 +22,7 @@ from django.conf import settings
 from newtagging.models import TagBase, tags_updated
 from newtagging import managers
 from catalogue.fields import JSONField, OverwritingFileField
 from newtagging.models import TagBase, tags_updated
 from newtagging import managers
 from catalogue.fields import JSONField, OverwritingFileField

-from catalogue.utils import ExistingFile, BookImportDocProvider
+from catalogue.utils import ExistingFile, BookImportDocProvider, create_zip_task, remove_zip

 
 from librarian import dcparser, html, epub, NoDublinCore
 import mutagen
 
 from librarian import dcparser, html, epub, NoDublinCore
 import mutagen


@@ -521,7 +521,7 @@ class Book(models.Model):
         path, fname = os.path.realpath(self.xml_file.path).rsplit('/', 1)
         try:
             pdf_file = NamedTemporaryFile(delete=False)
         path, fname = os.path.realpath(self.xml_file.path).rsplit('/', 1)
         try:
             pdf_file = NamedTemporaryFile(delete=False)

-
+            print("%s -> %s" % (self.xml_file.path, pdf_file))

             pdf.transform(BookImportDocProvider(self),
                       file_path=str(self.xml_file.path),
                       output_file=pdf_file,
             pdf.transform(BookImportDocProvider(self),
                       file_path=str(self.xml_file.path),
                       output_file=pdf_file,


@@ -531,7 +531,6 @@ class Book(models.Model):
         finally:
             unlink(pdf_file.name)
 
         finally:
             unlink(pdf_file.name)
 

-

     def build_epub(self, remove_descendants=True):
         """ (Re)builds the epub file.
             If book has a parent, does nothing.
     def build_epub(self, remove_descendants=True):
         """ (Re)builds the epub file.
             If book has a parent, does nothing.


@@ -627,6 +626,35 @@ class Book(models.Model):
             return True
         return False
 
             return True
         return False
 

+    @staticmethod
+    def zip_epub():
+        books = Book.objects.all()
+
+        paths = filter(lambda x: x is not None,
+                       map(lambda b: b.epub_file and b.epub_file.path or None, books))
+        result = create_zip_task.delay(paths, settings.ALL_EPUB_ZIP)
+        return settings.MEDIA_URL + result.wait()
+
+    @staticmethod
+    def zip_pdf():
+        books = Book.objects.all()
+
+        paths = filter(lambda x: x is not None,
+                       map(lambda b: b.pdf_file and b.pdf_file.path or None, books))
+        result = create_zip_task.delay(paths, settings.ALL_PDF_ZIP)
+        return settings.MEDIA_URL + result.wait()
+
+    def zip_audiobooks(self):
+        bm = BookMedia.objects.filter(book=self)
+        paths = map(lambda bm: bm.file.path, bm)
+        result = create_zip_task.delay(paths, self.slug)
+
+        return settings.MEDIA_URL + result.wait()
+
+    def clean_zip_files(self):
+        remove_zip(self.slug)
+        remove_zip(settings.ALL_EPUB_ZIP)
+        remove_zip(settings.ALL_PDF_ZIP)

 
     @classmethod
     def from_xml_file(cls, xml_file, **kwargs):
 
     @classmethod
     def from_xml_file(cls, xml_file, **kwargs):

diff --git a/apps/catalogue/utils.py b/apps/catalogue/utils.py
index 368f96d..d5ef2b7 100644 (file)
--- a/apps/catalogue/utils.py
+++ b/apps/catalogue/utils.py
@@ -2,6 +2,8 @@
 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
 #
 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
 #

+from __future__ import with_statement
+

 import random
 import time
 from base64 import urlsafe_b64encode
 import random
 import time
 from base64 import urlsafe_b64encode


@@ -9,6 +11,11 @@ from base64 import urlsafe_b64encode
 from django.core.files.uploadedfile import UploadedFile
 from django.utils.hashcompat import sha_constructor
 from django.conf import settings
 from django.core.files.uploadedfile import UploadedFile
 from django.utils.hashcompat import sha_constructor
 from django.conf import settings

+from celery.task import task
+from os import mkdir, path, unlink
+from errno import EEXIST
+from fcntl import flock, LOCK_EX
+from zipfile import ZipFile

 
 from librarian import DocProvider
 
 
 from librarian import DocProvider
 


@@ -59,3 +66,50 @@ class BookImportDocProvider(DocProvider):
             return self.book.xml_file
         else:
             return type(self.book).objects.get(slug=slug).xml_file
             return self.book.xml_file
         else:
             return type(self.book).objects.get(slug=slug).xml_file

+
+
+class LockFile(object):
+    def __init__(self, dir, objname):
+        self.lockname = path.join(dir, objname + ".lock")
+
+    def __entry__(self):
+        self.lock = open(self.lockname, 'w')
+        flock(self.lock, LOCK_EX)
+
+    def __exit__(self, *err):
+        self.lock.close()
+        unlink(self.lockname)
+
+
+def create_zip(paths, zip_slug):
+    # directory to store zip files
+    zip_path = path.join(settings.MEDIA_ROOT, 'zip')
+
+    try:
+        mkdir(zip_path)
+    except OSError as oe:
+        if oe.errno != EEXIST:
+            raise oe
+    zip_filename = zip_slug + ".zip"
+
+    with LockFile(zip_path, zip_slug):
+        if not path.exists(path.join(zip_path, zip_filename)):
+            with ZipFile(path.join(zip_path, zip_filename), 'w') as zipf:
+                for p in paths:
+                    zipf.write(p, path.basename(p))
+
+        return 'zip/' + zip_filename
+
+
+def remove_zip(zip_slug):
+    zip_file = path.join(settings.MEDIA_ROOT, 'zip', zip_slug + '.zip')
+    try:
+        unlink(zip_file)
+    except OSError as oe:
+        if oe.errno != EEXIST:
+            raise oe
+
+
+@task
+def create_zip_task(*args):
+    return create_zip(*args)

diff --git a/wolnelektury/settings.py b/wolnelektury/settings.py
index 48ba8f8..e0f740f 100644 (file)
--- a/wolnelektury/settings.py
+++ b/wolnelektury/settings.py
@@ -128,6 +128,9 @@ INSTALLED_APPS = [
     'rosetta',
     'south',
     'sorl.thumbnail',
     'rosetta',
     'south',
     'sorl.thumbnail',

+    'djcelery',
+    'djkombu',
+    #    'django_nose',

 
     # included
     'compress',
 
     # included
     'compress',


@@ -225,9 +228,20 @@ NO_BUILD_EPUB = False
 NO_BUILD_TXT = False
 NO_BUILD_PDF = False
 
 NO_BUILD_TXT = False
 NO_BUILD_PDF = False
 

+ALL_EPUB_ZIP = 'wolnelektury_pl_epub'
+ALL_PDF_ZIP = 'wolnelektury_pl_pdf'

 
 PAGINATION_INVALID_PAGE_RAISES_404 = True
 
 
 PAGINATION_INVALID_PAGE_RAISES_404 = True
 

+import djcelery
+djcelery.setup_loader()
+
+BROKER_BACKEND = "djkombu.transport.DatabaseTransport"
+BROKER_HOST = "localhost"
+BROKER_PORT = 5672
+BROKER_USER = "guest"
+BROKER_PASSWORD = "guest"
+BROKER_VHOST = "/"

 
 # Load localsettings, if they exist
 try:
 
 # Load localsettings, if they exist
 try: