X-Git-Url: https://git.mdrn.pl/wolnelektury.git/blobdiff_plain/5c3d5246956dfe2abea0aa5c80f5e776cec72fa5..5a605e04abaa18a80e391c5cc26c48e0125d6f3c:/apps/catalogue/models.py diff --git a/apps/catalogue/models.py b/apps/catalogue/models.py index 95d3da120..b82879ba5 100644 --- a/apps/catalogue/models.py +++ b/apps/catalogue/models.py @@ -6,11 +6,13 @@ from datetime import datetime from django.db import models from django.db.models import permalink, Q +import django.dispatch from django.core.cache import cache +from django.core.files.storage import DefaultStorage from django.utils.translation import ugettext_lazy as _ from django.contrib.auth.models import User -from django.core.files import File from django.template.loader import render_to_string +from django.utils.datastructures import SortedDict from django.utils.safestring import mark_safe from django.utils.translation import get_language from django.core.urlresolvers import reverse @@ -21,13 +23,11 @@ from django.conf import settings from newtagging.models import TagBase, tags_updated from newtagging import managers from catalogue.fields import JSONField, OverwritingFileField -from catalogue.utils import ExistingFile - -from librarian import dcparser, html, epub, NoDublinCore -import mutagen -from mutagen import id3 -from slughifi import slughifi -from sortify import sortify +from catalogue.utils import create_zip +from shutil import copy +from glob import glob +import re +from os import path TAG_CATEGORIES = ( @@ -47,6 +47,10 @@ MEDIA_FORMATS = ( ('daisy', _('DAISY file')), ) +# not quite, but Django wants you to set a timeout +CACHE_FOREVER = 2419200 # 28 days + + class TagSubcategoryManager(models.Manager): def __init__(self, subcategory): super(TagSubcategoryManager, self).__init__() @@ -171,21 +175,46 @@ class Tag(TagBase): return '/'.join((Tag.categories_dict[self.category], self.slug)) +def get_dynamic_path(media, filename, ext=None, maxlen=100): + from slughifi import slughifi + + # how to put related book's slug here? + if not ext: + if media.type == 'daisy': + ext = 'daisy.zip' + else: + ext = media.type + if media is None or not media.name: + name = slughifi(filename.split(".")[0]) + else: + name = slughifi(media.name) + return 'book/%s/%s.%s' % (ext, name[:maxlen-len('book/%s/.%s' % (ext, ext))-4], ext) + + # TODO: why is this hard-coded ? def book_upload_path(ext=None, maxlen=100): - def get_dynamic_path(media, filename, ext=ext): - # how to put related book's slug here? - if not ext: - if media.type == 'daisy': - ext = 'daisy.zip' - else: - ext = media.type - if not media.name: - name = slughifi(filename.split(".")[0]) - else: - name = slughifi(media.name) - return 'book/%s/%s.%s' % (ext, name[:maxlen-len('book/%s/.%s' % (ext, ext))-4], ext) - return get_dynamic_path + return lambda *args: get_dynamic_path(*args, ext=ext, maxlen=maxlen) + + +def get_customized_pdf_path(book, customizations): + """ + Returns a MEDIA_ROOT relative path for a customized pdf. The name will contain a hash of customization options. + """ + customizations.sort() + h = hash(tuple(customizations)) + pdf_name = '%s-custom-%s' % (book.slug, h) + pdf_file = models.get_dynamic_path(None, pdf_name, ext='pdf') + return pdf_file + + +def get_existing_customized_pdf(book): + """ + Returns a list of paths to generated customized pdf of a book + """ + pdf_glob = '%s-custom-' % (book.slug,) + pdf_glob = get_dynamic_path(None, pdf_glob, ext='pdf') + pdf_glob = re.sub(r"[.]([a-z0-9]+)$", "*.\\1", pdf_glob) + return glob(path.join(settings.MEDIA_ROOT, pdf_glob)) class BookMedia(models.Model): @@ -206,6 +235,9 @@ class BookMedia(models.Model): verbose_name_plural = _('book media') def save(self, *args, **kwargs): + from slughifi import slughifi + from catalogue.utils import ExistingFile, remove_zip + try: old = BookMedia.objects.get(pk=self.pk) except BookMedia.DoesNotExist, e: @@ -216,6 +248,10 @@ class BookMedia(models.Model): self.file.save(None, ExistingFile(self.file.path), save=False, leave=True) super(BookMedia, self).save(*args, **kwargs) + + # remove the zip package for book with modified media + remove_zip(self.book.slug) + extra_info = self.get_extra_info_value() extra_info.update(self.read_meta()) self.set_extra_info_value(extra_info) @@ -226,6 +262,8 @@ class BookMedia(models.Model): """ Reads some metadata from the audiobook. """ + import mutagen + from mutagen import id3 artist_name = director_name = project = funded_by = '' if self.type == 'mp3': @@ -258,6 +296,8 @@ class BookMedia(models.Model): """ Reads source file SHA1 from audiobok metadata. """ + import mutagen + from mutagen import id3 if filetype == 'mp3': try: @@ -278,7 +318,7 @@ class BookMedia(models.Model): class Book(models.Model): title = models.CharField(_('title'), max_length=120) - sort_key = models.CharField(_('sort_key'), max_length=120, db_index=True, editable=False) + sort_key = models.CharField(_('sort key'), max_length=120, db_index=True, editable=False) slug = models.SlugField(_('slug'), max_length=120, unique=True, db_index=True) description = models.TextField(_('description'), blank=True) created_at = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True) @@ -288,17 +328,17 @@ class Book(models.Model): gazeta_link = models.CharField(blank=True, max_length=240) wiki_link = models.CharField(blank=True, max_length=240) # files generated during publication - xml_file = models.FileField(_('XML file'), upload_to=book_upload_path('xml'), blank=True) - html_file = models.FileField(_('HTML file'), upload_to=book_upload_path('html'), blank=True) - pdf_file = models.FileField(_('PDF file'), upload_to=book_upload_path('pdf'), blank=True) - epub_file = models.FileField(_('EPUB file'), upload_to=book_upload_path('epub'), blank=True) - txt_file = models.FileField(_('TXT file'), upload_to=book_upload_path('txt'), blank=True) + file_types = ['epub', 'html', 'mobi', 'pdf', 'txt', 'xml'] + parent = models.ForeignKey('self', blank=True, null=True, related_name='children') objects = models.Manager() tagged = managers.ModelTaggedItemManager(Tag) tags = managers.TagDescriptor(Tag) + html_built = django.dispatch.Signal() + published = django.dispatch.Signal() + class AlreadyExists(Exception): pass @@ -311,12 +351,16 @@ class Book(models.Model): return self.title def save(self, force_insert=False, force_update=False, reset_short_html=True, **kwargs): + from sortify import sortify + self.sort_key = sortify(self.title) + ret = super(Book, self).save(force_insert, force_update) + if reset_short_html: self.reset_short_html() - return super(Book, self).save(force_insert, force_update) + return ret @permalink def get_absolute_url(self): @@ -339,49 +383,15 @@ class Book(models.Model): return book_tag def has_media(self, type): - if type == 'xml': - if self.xml_file: - return True - else: - return False - elif type == 'html': - if self.html_file: - return True - else: - return False - elif type == 'txt': - if self.txt_file: - return True - else: - return False - elif type == 'pdf': - if self.pdf_file: - return True - else: - return False - elif type == 'epub': - if self.epub_file: - return True - else: - return False + if type in Book.file_types: + return bool(getattr(self, "%s_file" % type)) else: - if self.media.filter(type=type).exists(): - return True - else: - return False + return self.media.filter(type=type).exists() def get_media(self, type): if self.has_media(type): - if type == "xml": - return self.xml_file - elif type == "html": - return self.html_file - elif type == "epub": - return self.epub_file - elif type == "txt": - return self.txt_file - elif type == "pdf": - return self.pdf_file + if type in Book.file_types: + return getattr(self, "%s_file" % type) else: return self.media.filter(type=type) else: @@ -421,11 +431,13 @@ class Book(models.Model): tags = [mark_safe(u'%s' % (tag.get_absolute_url(), tag.name)) for tag in tags] formats = [] - # files generated during publication + # files generated during publication if self.has_media("html"): formats.append(u'%s' % (reverse('book_text', kwargs={'slug': self.slug}), _('Read online'))) if self.has_media("pdf"): formats.append(u'PDF' % self.get_media('pdf').url) + if self.has_media("mobi"): + formats.append(u'MOBI' % self.get_media('mobi').url) if self.root_ancestor.has_media("epub"): formats.append(u'EPUB' % self.root_ancestor.get_media('epub').url) if self.has_media("txt"): @@ -440,7 +452,7 @@ class Book(models.Model): {'book': self, 'tags': tags, 'formats': formats})) if self.id: - cache.set(cache_key, short_html) + cache.set(cache_key, short_html, CACHE_FOREVER) return mark_safe(short_html) @property @@ -461,26 +473,6 @@ class Book(models.Model): has_description.boolean = True # ugly ugly ugly - def has_pdf_file(self): - return bool(self.pdf_file) - has_pdf_file.short_description = 'PDF' - has_pdf_file.boolean = True - - def has_epub_file(self): - return bool(self.epub_file) - has_epub_file.short_description = 'EPUB' - has_epub_file.boolean = True - - def has_txt_file(self): - return bool(self.txt_file) - has_txt_file.short_description = 'HTML' - has_txt_file.boolean = True - - def has_html_file(self): - return bool(self.html_file) - has_html_file.short_description = 'HTML' - has_html_file.boolean = True - def has_odt_file(self): return bool(self.has_media("odt")) has_odt_file.short_description = 'ODT' @@ -495,34 +487,83 @@ class Book(models.Model): return bool(self.has_media("ogg")) has_ogg_file.short_description = 'OGG' has_ogg_file.boolean = True - + def has_daisy_file(self): return bool(self.has_media("daisy")) has_daisy_file.short_description = 'DAISY' - has_daisy_file.boolean = True - + has_daisy_file.boolean = True + + def build_pdf(self, customizations=None, file_name=None): + """ (Re)builds the pdf file. + customizations - customizations which are passed to LaTeX class file. + file_name - save the pdf file under a different name and DO NOT save it in db. + """ + from tempfile import NamedTemporaryFile + from os import unlink + from django.core.files import File + from librarian import pdf + from catalogue.utils import ORMDocProvider, remove_zip + + try: + pdf_file = NamedTemporaryFile(delete=False) + pdf.transform(ORMDocProvider(self), + file_path=str(self.xml_file.path), + output_file=pdf_file, + customizations=customizations + ) + + if file_name is None: + # we'd like to be sure not to overwrite changes happening while + # (timely) pdf generation is taking place (async celery scenario) + current_self = Book.objects.get(id=self.id) + current_self.pdf_file.save('%s.pdf' % self.slug, File(open(pdf_file.name))) + self.pdf_file = current_self.pdf_file + else: + print "safing %s" % file_name + print "to: %s" % DefaultStorage().path(file_name) + DefaultStorage().save(file_name, File(open(pdf_file.name))) + finally: + unlink(pdf_file.name) + + # remove cached downloadables + remove_zip(settings.ALL_PDF_ZIP) + for customized_pdf in get_existing_customized_pdf(self): + unlink(customized_pdf) + + def build_mobi(self): + """ (Re)builds the MOBI file. + + """ + from tempfile import NamedTemporaryFile + from os import unlink + from django.core.files import File + from librarian import mobi + from catalogue.utils import ORMDocProvider, remove_zip + + try: + mobi_file = NamedTemporaryFile(suffix='.mobi', delete=False) + mobi.transform(ORMDocProvider(self), verbose=1, + file_path=str(self.xml_file.path), + output_file=mobi_file.name, + ) + + self.mobi_file.save('%s.mobi' % self.slug, File(open(mobi_file.name))) + finally: + unlink(mobi_file.name) + + # remove zip with all mobi files + remove_zip(settings.ALL_MOBI_ZIP) + def build_epub(self, remove_descendants=True): """ (Re)builds the epub file. If book has a parent, does nothing. Unless remove_descendants is False, descendants' epubs are removed. """ - from StringIO import StringIO from hashlib import sha1 from django.core.files.base import ContentFile - from librarian import DocProvider - - class BookImportDocProvider(DocProvider): - """ used for joined EPUBs """ - - def __init__(self, book): - self.book = book - - def by_slug(self, slug): - if slug == self.book.slug: - return self.book.xml_file - else: - return Book.objects.get(slug=slug).xml_file + from librarian import epub, NoDublinCore + from catalogue.utils import ORMDocProvider, remove_zip if self.parent: # don't need an epub @@ -530,7 +571,7 @@ class Book(models.Model): epub_file = StringIO() try: - epub.transform(BookImportDocProvider(self), self.slug, output_file=epub_file) + epub.transform(ORMDocProvider(self), self.slug, output_file=epub_file) self.epub_file.save('%s.epub' % self.slug, ContentFile(epub_file.getvalue())) FileRecord(slug=self.slug, type='epub', sha1=sha1(epub_file.getvalue()).hexdigest()).save() except NoDublinCore: @@ -545,6 +586,9 @@ class Book(models.Model): child_book.save() book_descendants += list(child_book.children.all()) + # remove zip package with all epub files + remove_zip(settings.ALL_EPUB_ZIP) + def build_txt(self): from StringIO import StringIO from django.core.files.base import ContentFile @@ -558,6 +602,9 @@ class Book(models.Model): def build_html(self): from tempfile import NamedTemporaryFile from markupstring import MarkupString + from django.core.files import File + from slughifi import slughifi + from librarian import html meta_tags = list(self.tags.filter( category__in=('author', 'epoch', 'genre', 'kind'))) @@ -606,12 +653,37 @@ class Book(models.Model): new_fragment.save() new_fragment.tags = set(meta_tags + themes + [book_tag] + ancestor_tags) self.save() + self.html_built.send(sender=self) return True return False + @staticmethod + def zip_format(format_): + def pretty_file_name(book): + return "%s/%s.%s" % ( + b.get_extra_info_value()['author'], + b.slug, + format_) + + field_name = "%s_file" % format_ + books = Book.objects.filter(parent=None).exclude(**{field_name: ""}) + paths = [(pretty_file_name(b), getattr(b, field_name).path) + for b in books] + result = create_zip.delay(paths, + getattr(settings, "ALL_%s_ZIP" % format_.upper())) + return result.wait() + + def zip_audiobooks(self): + bm = BookMedia.objects.filter(book=self, type='mp3') + paths = map(lambda bm: (None, bm.file.path), bm) + result = create_zip.delay(paths, self.slug) + return result.wait() @classmethod def from_xml_file(cls, xml_file, **kwargs): + from django.core.files import File + from librarian import dcparser + # use librarian to parse meta-data book_info = dcparser.parse(xml_file) @@ -624,8 +696,11 @@ class Book(models.Model): xml_file.close() @classmethod - def from_text_and_meta(cls, raw_file, book_info, overwrite=False, build_epub=True, build_txt=True): + def from_text_and_meta(cls, raw_file, book_info, overwrite=False, + build_epub=True, build_txt=True, build_pdf=True, build_mobi=True): import re + from slughifi import slughifi + from sortify import sortify # check for parts before we do anything children = [] @@ -697,6 +772,12 @@ class Book(models.Model): if not settings.NO_BUILD_EPUB and build_epub: book.root_ancestor.build_epub() + if not settings.NO_BUILD_PDF and build_pdf: + book.root_ancestor.build_pdf() + + if not settings.NO_BUILD_MOBI and build_mobi: + book.build_mobi() + book_descendants = list(book.children.all()) # add l-tag to descendants and their fragments # delete unnecessary EPUB files @@ -708,11 +789,13 @@ class Book(models.Model): fragment.tags = set(list(fragment.tags) + [book_tag]) book_descendants += list(child_book.children.all()) + book.save() + # refresh cache book.reset_tag_counter() book.reset_theme_counter() - book.save() + cls.published.send(sender=book) return book def reset_tag_counter(self): @@ -741,7 +824,7 @@ class Book(models.Model): tags[tag.pk] = 1 if self.id: - cache.set(cache_key, tags) + cache.set(cache_key, tags, CACHE_FOREVER) return tags def reset_theme_counter(self): @@ -768,7 +851,7 @@ class Book(models.Model): tags[tag.pk] = tags.get(tag.pk, 0) + 1 if self.id: - cache.set(cache_key, tags) + cache.set(cache_key, tags, CACHE_FOREVER) return tags def pretty_title(self, html_links=False): @@ -806,6 +889,74 @@ class Book(models.Model): return objects + @classmethod + def book_list(cls, filter=None): + """Generates a hierarchical listing of all books. + + Books are optionally filtered with a test function. + + """ + + books_by_parent = {} + books = cls.objects.all().order_by('parent_number', 'sort_key').only('title', 'parent', 'slug') + if filter: + books = books.filter(filter).distinct() + book_ids = set((book.pk for book in books)) + for book in books: + parent = book.parent_id + if parent not in book_ids: + parent = None + books_by_parent.setdefault(parent, []).append(book) + else: + for book in books: + books_by_parent.setdefault(book.parent_id, []).append(book) + + orphans = [] + books_by_author = SortedDict() + for tag in Tag.objects.filter(category='author'): + books_by_author[tag] = [] + + for book in books_by_parent.get(None,()): + authors = list(book.tags.filter(category='author')) + if authors: + for author in authors: + books_by_author[author].append(book) + else: + orphans.append(book) + + return books_by_author, orphans, books_by_parent + + _audiences_pl = { + "SP1": (1, u"szkoła podstawowa"), + "SP2": (1, u"szkoła podstawowa"), + "P": (1, u"szkoła podstawowa"), + "G": (2, u"gimnazjum"), + "L": (3, u"liceum"), + "LP": (3, u"liceum"), + } + def audiences_pl(self): + audiences = self.get_extra_info_value().get('audiences', []) + audiences = sorted(set([self._audiences_pl[a] for a in audiences])) + return [a[1] for a in audiences] + + +def _has_factory(ftype): + has = lambda self: bool(getattr(self, "%s_file" % ftype)) + has.short_description = t.upper() + has.boolean = True + has.__name__ = "has_%s_file" % ftype + return has + + +# add the file fields +for t in Book.file_types: + field_name = "%s_file" % t + models.FileField(_("%s file" % t.upper()), + upload_to=book_upload_path(t), + blank=True).contribute_to_class(Book, field_name) + + setattr(Book, "has_%s_file" % t, _has_factory(t)) + class Fragment(models.Model): text = models.TextField() @@ -846,7 +997,7 @@ class Fragment(models.Model): short_html = unicode(render_to_string('catalogue/fragment_short.html', {'fragment': self})) if self.id: - cache.set(cache_key, short_html) + cache.set(cache_key, short_html, CACHE_FOREVER) return mark_safe(short_html)