From f37981065265afbb98a729e9ed8d6eec2e2763a0 Mon Sep 17 00:00:00 2001 From: Marcin Koziej Date: Mon, 24 Oct 2011 15:25:34 +0200 Subject: [PATCH] zip package for epub/pdf/audiobooks --- .../management/commands/importbooks.py | 3 ++ apps/catalogue/models.py | 34 ++++++++++-- apps/catalogue/utils.py | 54 +++++++++++++++++++ wolnelektury/settings.py | 14 +++++ 4 files changed, 102 insertions(+), 3 deletions(-) diff --git a/apps/catalogue/management/commands/importbooks.py b/apps/catalogue/management/commands/importbooks.py index f51214d72..62ead5373 100644 --- a/apps/catalogue/management/commands/importbooks.py +++ b/apps/catalogue/management/commands/importbooks.py @@ -101,6 +101,9 @@ class Command(BaseCommand): print "Importing %s.txt" % file_base book.save() + + # clean the generated zip packages. Is this the right place for this? + book.clean_zip_files() except Book.AlreadyExists, msg: print self.style.ERROR('%s: Book already imported. Skipping. To overwrite use --force.' % diff --git a/apps/catalogue/models.py b/apps/catalogue/models.py index c5ca5df7f..471c6617c 100644 --- a/apps/catalogue/models.py +++ b/apps/catalogue/models.py @@ -22,7 +22,7 @@ from django.conf import settings from newtagging.models import TagBase, tags_updated from newtagging import managers from catalogue.fields import JSONField, OverwritingFileField -from catalogue.utils import ExistingFile, BookImportDocProvider +from catalogue.utils import ExistingFile, BookImportDocProvider, create_zip_task, remove_zip from librarian import dcparser, html, epub, NoDublinCore import mutagen @@ -521,7 +521,7 @@ class Book(models.Model): path, fname = os.path.realpath(self.xml_file.path).rsplit('/', 1) try: pdf_file = NamedTemporaryFile(delete=False) - + print("%s -> %s" % (self.xml_file.path, pdf_file)) pdf.transform(BookImportDocProvider(self), file_path=str(self.xml_file.path), output_file=pdf_file, @@ -531,7 +531,6 @@ class Book(models.Model): finally: unlink(pdf_file.name) - def build_epub(self, remove_descendants=True): """ (Re)builds the epub file. If book has a parent, does nothing. @@ -627,6 +626,35 @@ class Book(models.Model): return True return False + @staticmethod + def zip_epub(): + books = Book.objects.all() + + paths = filter(lambda x: x is not None, + map(lambda b: b.epub_file and b.epub_file.path or None, books)) + result = create_zip_task.delay(paths, settings.ALL_EPUB_ZIP) + return settings.MEDIA_URL + result.wait() + + @staticmethod + def zip_pdf(): + books = Book.objects.all() + + paths = filter(lambda x: x is not None, + map(lambda b: b.pdf_file and b.pdf_file.path or None, books)) + result = create_zip_task.delay(paths, settings.ALL_PDF_ZIP) + return settings.MEDIA_URL + result.wait() + + def zip_audiobooks(self): + bm = BookMedia.objects.filter(book=self) + paths = map(lambda bm: bm.file.path, bm) + result = create_zip_task.delay(paths, self.slug) + + return settings.MEDIA_URL + result.wait() + + def clean_zip_files(self): + remove_zip(self.slug) + remove_zip(settings.ALL_EPUB_ZIP) + remove_zip(settings.ALL_PDF_ZIP) @classmethod def from_xml_file(cls, xml_file, **kwargs): diff --git a/apps/catalogue/utils.py b/apps/catalogue/utils.py index 368f96d20..d5ef2b789 100644 --- a/apps/catalogue/utils.py +++ b/apps/catalogue/utils.py @@ -2,6 +2,8 @@ # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later. # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information. # +from __future__ import with_statement + import random import time from base64 import urlsafe_b64encode @@ -9,6 +11,11 @@ from base64 import urlsafe_b64encode from django.core.files.uploadedfile import UploadedFile from django.utils.hashcompat import sha_constructor from django.conf import settings +from celery.task import task +from os import mkdir, path, unlink +from errno import EEXIST +from fcntl import flock, LOCK_EX +from zipfile import ZipFile from librarian import DocProvider @@ -59,3 +66,50 @@ class BookImportDocProvider(DocProvider): return self.book.xml_file else: return type(self.book).objects.get(slug=slug).xml_file + + +class LockFile(object): + def __init__(self, dir, objname): + self.lockname = path.join(dir, objname + ".lock") + + def __entry__(self): + self.lock = open(self.lockname, 'w') + flock(self.lock, LOCK_EX) + + def __exit__(self, *err): + self.lock.close() + unlink(self.lockname) + + +def create_zip(paths, zip_slug): + # directory to store zip files + zip_path = path.join(settings.MEDIA_ROOT, 'zip') + + try: + mkdir(zip_path) + except OSError as oe: + if oe.errno != EEXIST: + raise oe + zip_filename = zip_slug + ".zip" + + with LockFile(zip_path, zip_slug): + if not path.exists(path.join(zip_path, zip_filename)): + with ZipFile(path.join(zip_path, zip_filename), 'w') as zipf: + for p in paths: + zipf.write(p, path.basename(p)) + + return 'zip/' + zip_filename + + +def remove_zip(zip_slug): + zip_file = path.join(settings.MEDIA_ROOT, 'zip', zip_slug + '.zip') + try: + unlink(zip_file) + except OSError as oe: + if oe.errno != EEXIST: + raise oe + + +@task +def create_zip_task(*args): + return create_zip(*args) diff --git a/wolnelektury/settings.py b/wolnelektury/settings.py index 48ba8f894..e0f740fff 100644 --- a/wolnelektury/settings.py +++ b/wolnelektury/settings.py @@ -128,6 +128,9 @@ INSTALLED_APPS = [ 'rosetta', 'south', 'sorl.thumbnail', + 'djcelery', + 'djkombu', + # 'django_nose', # included 'compress', @@ -225,9 +228,20 @@ NO_BUILD_EPUB = False NO_BUILD_TXT = False NO_BUILD_PDF = False +ALL_EPUB_ZIP = 'wolnelektury_pl_epub' +ALL_PDF_ZIP = 'wolnelektury_pl_pdf' PAGINATION_INVALID_PAGE_RAISES_404 = True +import djcelery +djcelery.setup_loader() + +BROKER_BACKEND = "djkombu.transport.DatabaseTransport" +BROKER_HOST = "localhost" +BROKER_PORT = 5672 +BROKER_USER = "guest" +BROKER_PASSWORD = "guest" +BROKER_VHOST = "/" # Load localsettings, if they exist try: -- 2.20.1