- DRY: move build methods to relevant fields, simplify adding formats (in catalogue.constants)
- add CATALOGUE_{DONT_BUILD,FORMAT_ZIPS} instead of many NO_BUILD_* and ALL_*_ZIP settings, move to catalogue.app_settings
- catalogue.management.importbooks: one option for skipping formats
- use override_settings in tests
- minor test fixes
+# -*- coding: utf-8 -*-
+# This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
+# Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
+#
+import logging
+from django.conf import settings as settings
+from catalogue.utils import AppSettings
+
+
+class Settings(AppSettings):
+ """Default settings for catalogue app."""
+ DEFAULT_LANGUAGE = u'pol'
+ # PDF needs TeXML + XeLaTeX, MOBI needs Calibre.
+ DONT_BUILD = {'pdf', 'mobi'}
+ FORMAT_ZIPS = {
+ 'epub': 'wolnelektury_pl_epub',
+ 'pdf': 'wolnelektury_pl_pdf',
+ 'mobi': 'wolnelektury_pl_mobi',
+ 'fb2': 'wolnelektury_pl_fb2',
+ }
+
+ def _more_DONT_BUILD(self, value):
+ for format_ in ['cover', 'pdf', 'epub', 'mobi', 'fb2', 'txt']:
+ attname = 'NO_BUILD_%s' % format_.upper()
+ if hasattr(settings, attname):
+ logging.warn("%s is deprecated, "
+ "use CATALOGUE_DONT_BUILD instead", attname)
+ if getattr(settings, attname):
+ value.add(format_)
+ else:
+ value.remove(format_)
+ return value
+
+ def _more_FORMAT_ZIPS(self, value):
+ for format_ in ['epub', 'pdf', 'mobi', 'fb2']:
+ attname = 'ALL_%s_ZIP' % format_.upper()
+ if hasattr(settings, attname):
+ logging.warn("%s is deprecated, "
+ "use CATALOGUE_FORMAT_ZIPS[%s] instead",
+ attname, format_)
+ value[format_] = getattr(settings, attname)
+ return value
+
+
+app_settings = Settings('CATALOGUE')
'description': _('Creative Commons Attribution-ShareAlike 3.0 Unported'),
},
}
+
+# Those will be generated only for books with own HTML.
+EBOOK_FORMATS_WITHOUT_CHILDREN = ['txt', 'fb2']
+# Those will be generated for all books.
+EBOOK_FORMATS_WITH_CHILDREN = ['pdf', 'epub', 'mobi']
+# Those will be generated when inherited cover changes.
+EBOOK_FORMATS_WITH_COVERS = ['mobi']
+
+EBOOK_FORMATS = EBOOK_FORMATS_WITHOUT_CHILDREN + EBOOK_FORMATS_WITH_CHILDREN
# This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
# Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
#
+from django.conf import settings
+from django.core.files import File
from django.db import models
from django.db.models.fields.files import FieldFile
+from catalogue import app_settings
+from catalogue.utils import remove_zip, truncate_html_words
+from celery import Task
+from celery.task import task
+from waiter.utils import clear_cache
+
+
+class EbookFieldFile(FieldFile):
+ """Represents contents of an ebook file field."""
+
+ def build(self):
+ """Build the ebook immediately."""
+ return self.field.builder(self)
+
+ def build_delay(self):
+ """Builds the ebook in a delayed task."""
+ return self.field.builder.delay(self)
+
+
+class EbookField(models.FileField):
+ """Represents an ebook file field, attachable to a model."""
+ attr_class = EbookFieldFile
+
+ def __init__(self, format_name, *args, **kwargs):
+ super(EbookField, self).__init__(*args, **kwargs)
+ self.format_name = format_name
+
+ @property
+ def builder(self):
+ """Finds a celery task suitable for the format of the field."""
+ return BuildEbook.for_format(self.format_name)
+
+ def contribute_to_class(self, cls, name):
+ super(EbookField, self).contribute_to_class(cls, name)
+
+ def has(model_instance):
+ return bool(getattr(model_instance, self.attname, None))
+ has.__doc__ = None
+ has.__name__ = "has_%s" % self.attname
+ has.short_description = self.name
+ has.boolean = True
+ setattr(cls, 'has_%s' % self.attname, has)
+
+
+class BuildEbook(Task):
+ formats = {}
+
+ @classmethod
+ def register(cls, format_name):
+ """A decorator for registering subclasses for particular formats."""
+ def wrapper(builder):
+ cls.formats[format_name] = builder
+ return builder
+ return wrapper
+
+ @classmethod
+ def for_format(cls, format_name):
+ """Returns a celery task suitable for specified format."""
+ return cls.formats.get(format_name, BuildEbookTask)
+
+ @staticmethod
+ def transform(wldoc, fieldfile):
+ """Transforms an librarian.WLDocument into an librarian.OutputFile.
+
+ By default, it just calls relevant wldoc.as_??? method.
+
+ """
+ return getattr(wldoc, "as_%s" % fieldfile.field.format_name)()
+
+ def run(self, fieldfile):
+ book = fieldfile.instance
+ out = self.transform(book.wldocument(), fieldfile)
+ fieldfile.save(None, File(open(out.get_filename())), save=False)
+ if book.pk is not None:
+ type(book).objects.filter(pk=book.pk).update(**{
+ fieldfile.field.attname: fieldfile
+ })
+ if fieldfile.field.format_name in app_settings.FORMAT_ZIPS:
+ remove_zip(app_settings.FORMAT_ZIPS[fieldfile.field.format_name])
+# Don't decorate BuildEbook, because we want to subclass it.
+BuildEbookTask = task(BuildEbook, ignore_result=True)
+
+
+@BuildEbook.register('txt')
+@task(ignore_result=True)
+class BuildTxt(BuildEbook):
+ @staticmethod
+ def transform(wldoc, fieldfile):
+ return wldoc.as_text()
+
+
+@BuildEbook.register('pdf')
+@task(ignore_result=True)
+class BuildPdf(BuildEbook):
+ @staticmethod
+ def transform(wldoc, fieldfile):
+ return wldoc.as_pdf(morefloats=settings.LIBRARIAN_PDF_MOREFLOATS)
+
+ def run(self, fieldfile):
+ BuildEbook.run(self, fieldfile)
+ clear_cache(fieldfile.instance.slug)
+
+
+@BuildEbook.register('html')
+@task(ignore_result=True)
+class BuildHtml(BuildEbook):
+ def run(self, fieldfile):
+ from django.core.files.base import ContentFile
+ from slughifi import slughifi
+ from sortify import sortify
+ from librarian import html
+ from catalogue.models import Fragment, Tag
+
+ book = fieldfile.instance
+
+ meta_tags = list(book.tags.filter(
+ category__in=('author', 'epoch', 'genre', 'kind')))
+ book_tag = book.book_tag()
+
+ html_output = self.transform(
+ book.wldocument(parse_dublincore=False),
+ fieldfile)
+ if html_output:
+ fieldfile.save(None, ContentFile(html_output.get_string()),
+ save=False)
+ type(book).objects.filter(pk=book.pk).update(**{
+ fieldfile.field.attname: fieldfile
+ })
+
+ # get ancestor l-tags for adding to new fragments
+ ancestor_tags = []
+ p = book.parent
+ while p:
+ ancestor_tags.append(p.book_tag())
+ p = p.parent
+
+ # Delete old fragments and create them from scratch
+ book.fragments.all().delete()
+ # Extract fragments
+ closed_fragments, open_fragments = html.extract_fragments(fieldfile.path)
+ for fragment in closed_fragments.values():
+ try:
+ theme_names = [s.strip() for s in fragment.themes.split(',')]
+ except AttributeError:
+ continue
+ themes = []
+ for theme_name in theme_names:
+ if not theme_name:
+ continue
+ tag, created = Tag.objects.get_or_create(
+ slug=slughifi(theme_name),
+ category='theme')
+ if created:
+ tag.name = theme_name
+ tag.sort_key = sortify(theme_name.lower())
+ tag.save()
+ themes.append(tag)
+ if not themes:
+ continue
+
+ text = fragment.to_string()
+ short_text = truncate_html_words(text, 15)
+ if text == short_text:
+ short_text = ''
+ new_fragment = Fragment.objects.create(anchor=fragment.id,
+ book=book, text=text, short_text=short_text)
+
+ new_fragment.save()
+ new_fragment.tags = set(meta_tags + themes + [book_tag] + ancestor_tags)
+ book.html_built.send(sender=book)
+ return True
+ return False
class OverwritingFieldFile(FieldFile):
try:
# check for south
from south.modelsinspector import add_introspection_rules
-
- add_introspection_rules([], ["^catalogue\.fields\.OverwritingFileField"])
except ImportError:
pass
+else:
+ add_introspection_rules([
+ (
+ [EbookField],
+ [],
+ {'format_name': ('format_name', {})}
+ )
+ ], ["^catalogue\.fields\.EbookField"])
+ add_introspection_rules([], ["^catalogue\.fields\.OverwritingFileField"])
help='Verbosity level; 0=minimal output, 1=normal output, 2=all output'),
make_option('-f', '--force', action='store_true', dest='force', default=False,
help='Overwrite works already in the catalogue'),
- make_option('-E', '--no-build-epub', action='store_false', dest='build_epub', default=True,
- help='Don\'t build EPUB file'),
- make_option('-M', '--no-build-mobi', action='store_false', dest='build_mobi', default=True,
- help='Don\'t build MOBI file'),
- make_option('-T', '--no-build-txt', action='store_false', dest='build_txt', default=True,
- help='Don\'t build TXT file'),
- make_option('-P', '--no-build-pdf', action='store_false', dest='build_pdf', default=True,
- help='Don\'t build PDF file'),
+ make_option('-D', '--dont-build', dest='dont_build',
+ metavar="FORMAT,...",
+ help="Skip building specified formats"),
make_option('-S', '--no-search-index', action='store_false', dest='search_index', default=True,
help='Skip indexing imported works for search'),
make_option('-w', '--wait-until', dest='wait_until', metavar='TIME',
def import_book(self, file_path, options):
verbose = options.get('verbose')
+ if options.get('dont_build'):
+ dont_build = options.get('dont_build').lower().split(',')
+ else:
+ dont_build = None
file_base, ext = os.path.splitext(file_path)
book = Book.from_xml_file(file_path, overwrite=options.get('force'),
- build_epub=options.get('build_epub'),
- build_txt=options.get('build_txt'),
- build_pdf=options.get('build_pdf'),
- build_mobi=options.get('build_mobi'),
- search_index=options.get('search_index'),
- search_index_reuse=True, search_index_tags=False)
+ dont_build=dont_build,
+ search_index=options.get('search_index'),
+ search_index_reuse=True,
+ search_index_tags=False)
for ebook_format in Book.ebook_formats:
if os.path.isfile(file_base + '.' + ebook_format):
getattr(book, '%s_file' % ebook_format).save(
'%s.%s' % (book.slug, ebook_format),
- File(file(file_base + '.' + ebook_format)))
+ File(file(file_base + '.' + ebook_format)),
+ save=False
+ )
if verbose:
print "Importing %s.%s" % (file_base, ebook_format)
-
book.save()
def import_picture(self, file_path, options):
--- /dev/null
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+ def forwards(self, orm):
+
+ # Changing field 'Book.pdf_file'
+ db.alter_column('catalogue_book', 'pdf_file', self.gf('catalogue.fields.EbookField')(max_length=100, format_name='pdf'))
+
+ # Changing field 'Book.html_file'
+ db.alter_column('catalogue_book', 'html_file', self.gf('catalogue.fields.EbookField')(max_length=100, format_name='html'))
+
+ # Changing field 'Book.xml_file'
+ db.alter_column('catalogue_book', 'xml_file', self.gf('catalogue.fields.EbookField')(max_length=100, format_name='xml'))
+
+ # Changing field 'Book.txt_file'
+ db.alter_column('catalogue_book', 'txt_file', self.gf('catalogue.fields.EbookField')(max_length=100, format_name='txt'))
+
+ # Changing field 'Book.fb2_file'
+ db.alter_column('catalogue_book', 'fb2_file', self.gf('catalogue.fields.EbookField')(max_length=100, format_name='fb2'))
+
+ # Changing field 'Book.mobi_file'
+ db.alter_column('catalogue_book', 'mobi_file', self.gf('catalogue.fields.EbookField')(max_length=100, format_name='mobi'))
+
+ # Changing field 'Book.epub_file'
+ db.alter_column('catalogue_book', 'epub_file', self.gf('catalogue.fields.EbookField')(max_length=100, format_name='epub'))
+
+ # Changing field 'Book.cover'
+ db.alter_column('catalogue_book', 'cover', self.gf('catalogue.fields.EbookField')(max_length=100, null=True, format_name='cover'))
+
+ def backwards(self, orm):
+
+ # Changing field 'Book.pdf_file'
+ db.alter_column('catalogue_book', 'pdf_file', self.gf('django.db.models.fields.files.FileField')(max_length=100))
+
+ # Changing field 'Book.html_file'
+ db.alter_column('catalogue_book', 'html_file', self.gf('django.db.models.fields.files.FileField')(max_length=100))
+
+ # Changing field 'Book.xml_file'
+ db.alter_column('catalogue_book', 'xml_file', self.gf('django.db.models.fields.files.FileField')(max_length=100))
+
+ # Changing field 'Book.txt_file'
+ db.alter_column('catalogue_book', 'txt_file', self.gf('django.db.models.fields.files.FileField')(max_length=100))
+
+ # Changing field 'Book.fb2_file'
+ db.alter_column('catalogue_book', 'fb2_file', self.gf('django.db.models.fields.files.FileField')(max_length=100))
+
+ # Changing field 'Book.mobi_file'
+ db.alter_column('catalogue_book', 'mobi_file', self.gf('django.db.models.fields.files.FileField')(max_length=100))
+
+ # Changing field 'Book.epub_file'
+ db.alter_column('catalogue_book', 'epub_file', self.gf('django.db.models.fields.files.FileField')(max_length=100))
+
+ # Changing field 'Book.cover'
+ db.alter_column('catalogue_book', 'cover', self.gf('django.db.models.fields.files.FileField')(max_length=100, null=True))
+
+ models = {
+ 'auth.group': {
+ 'Meta': {'object_name': 'Group'},
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+ 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+ },
+ 'auth.permission': {
+ 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+ 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+ 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+ },
+ 'auth.user': {
+ 'Meta': {'object_name': 'User'},
+ 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+ 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+ 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+ 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+ 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+ 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+ 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+ 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+ },
+ 'catalogue.book': {
+ 'Meta': {'ordering': "('sort_key',)", 'object_name': 'Book'},
+ '_related_info': ('jsonfield.fields.JSONField', [], {'null': 'True', 'blank': 'True'}),
+ 'changed_at': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'db_index': 'True', 'blank': 'True'}),
+ 'common_slug': ('django.db.models.fields.SlugField', [], {'max_length': '120'}),
+ 'cover': ('catalogue.fields.EbookField', [], {'max_length': '100', 'null': 'True', 'format_name': "'cover'", 'blank': 'True'}),
+ 'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'db_index': 'True', 'blank': 'True'}),
+ 'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+ 'epub_file': ('catalogue.fields.EbookField', [], {'default': "''", 'max_length': '100', 'format_name': "'epub'", 'blank': 'True'}),
+ 'extra_info': ('jsonfield.fields.JSONField', [], {'default': "'{}'"}),
+ 'fb2_file': ('catalogue.fields.EbookField', [], {'default': "''", 'max_length': '100', 'format_name': "'fb2'", 'blank': 'True'}),
+ 'gazeta_link': ('django.db.models.fields.CharField', [], {'max_length': '240', 'blank': 'True'}),
+ 'html_file': ('catalogue.fields.EbookField', [], {'default': "''", 'max_length': '100', 'format_name': "'html'", 'blank': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'language': ('django.db.models.fields.CharField', [], {'default': "'pol'", 'max_length': '3', 'db_index': 'True'}),
+ 'mobi_file': ('catalogue.fields.EbookField', [], {'default': "''", 'max_length': '100', 'format_name': "'mobi'", 'blank': 'True'}),
+ 'parent': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'children'", 'null': 'True', 'to': "orm['catalogue.Book']"}),
+ 'parent_number': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+ 'pdf_file': ('catalogue.fields.EbookField', [], {'default': "''", 'max_length': '100', 'format_name': "'pdf'", 'blank': 'True'}),
+ 'slug': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '120'}),
+ 'sort_key': ('django.db.models.fields.CharField', [], {'max_length': '120', 'db_index': 'True'}),
+ 'title': ('django.db.models.fields.CharField', [], {'max_length': '120'}),
+ 'txt_file': ('catalogue.fields.EbookField', [], {'default': "''", 'max_length': '100', 'format_name': "'txt'", 'blank': 'True'}),
+ 'wiki_link': ('django.db.models.fields.CharField', [], {'max_length': '240', 'blank': 'True'}),
+ 'xml_file': ('catalogue.fields.EbookField', [], {'default': "''", 'max_length': '100', 'format_name': "'xml'", 'blank': 'True'})
+ },
+ 'catalogue.bookmedia': {
+ 'Meta': {'ordering': "('type', 'name')", 'object_name': 'BookMedia'},
+ 'book': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'media'", 'to': "orm['catalogue.Book']"}),
+ 'extra_info': ('jsonfield.fields.JSONField', [], {'default': "'{}'"}),
+ 'file': ('catalogue.fields.OverwritingFileField', [], {'max_length': '100'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': "'100'"}),
+ 'source_sha1': ('django.db.models.fields.CharField', [], {'max_length': '40', 'null': 'True', 'blank': 'True'}),
+ 'type': ('django.db.models.fields.CharField', [], {'max_length': "'100'"}),
+ 'uploaded_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'})
+ },
+ 'catalogue.collection': {
+ 'Meta': {'ordering': "('title',)", 'object_name': 'Collection'},
+ 'book_slugs': ('django.db.models.fields.TextField', [], {}),
+ 'description': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+ 'slug': ('django.db.models.fields.SlugField', [], {'max_length': '120', 'primary_key': 'True'}),
+ 'title': ('django.db.models.fields.CharField', [], {'max_length': '120', 'db_index': 'True'})
+ },
+ 'catalogue.fragment': {
+ 'Meta': {'ordering': "('book', 'anchor')", 'object_name': 'Fragment'},
+ 'anchor': ('django.db.models.fields.CharField', [], {'max_length': '120'}),
+ 'book': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'fragments'", 'to': "orm['catalogue.Book']"}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'short_text': ('django.db.models.fields.TextField', [], {}),
+ 'text': ('django.db.models.fields.TextField', [], {})
+ },
+ 'catalogue.tag': {
+ 'Meta': {'ordering': "('sort_key',)", 'unique_together': "(('slug', 'category'),)", 'object_name': 'Tag'},
+ 'book_count': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+ 'category': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}),
+ 'changed_at': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'db_index': 'True', 'blank': 'True'}),
+ 'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'db_index': 'True', 'blank': 'True'}),
+ 'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+ 'gazeta_link': ('django.db.models.fields.CharField', [], {'max_length': '240', 'blank': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}),
+ 'slug': ('django.db.models.fields.SlugField', [], {'max_length': '120'}),
+ 'sort_key': ('django.db.models.fields.CharField', [], {'max_length': '120', 'db_index': 'True'}),
+ 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True', 'blank': 'True'}),
+ 'wiki_link': ('django.db.models.fields.CharField', [], {'max_length': '240', 'blank': 'True'})
+ },
+ 'catalogue.tagrelation': {
+ 'Meta': {'unique_together': "(('tag', 'content_type', 'object_id'),)", 'object_name': 'TagRelation', 'db_table': "'catalogue_tag_relation'"},
+ 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'object_id': ('django.db.models.fields.PositiveIntegerField', [], {'db_index': 'True'}),
+ 'tag': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'items'", 'to': "orm['catalogue.Tag']"})
+ },
+ 'contenttypes.contenttype': {
+ 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+ 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+ }
+ }
+
+ complete_apps = ['catalogue']
\ No newline at end of file
# Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
#
import re
-from django.conf import settings
+from django.conf import settings as settings
from django.core.cache import get_cache
from django.db import models
from django.db.models import permalink
from django.utils.datastructures import SortedDict
from django.utils.translation import ugettext_lazy as _
import jsonfield
+from catalogue import constants
+from catalogue.fields import EbookField
from catalogue.models import Tag, Fragment, BookMedia
-from catalogue.utils import create_zip, split_tags, truncate_html_words, book_upload_path
+from catalogue.utils import create_zip, split_tags, book_upload_path
+from catalogue import app_settings
from catalogue import tasks
from newtagging import managers
unique=True)
common_slug = models.SlugField(_('slug'), max_length=120, db_index=True)
language = models.CharField(_('language code'), max_length=3, db_index=True,
- default=settings.CATALOGUE_DEFAULT_LANGUAGE)
+ default=app_settings.DEFAULT_LANGUAGE)
description = models.TextField(_('description'), blank=True)
created_at = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True)
changed_at = models.DateTimeField(_('creation date'), auto_now=True, db_index=True)
wiki_link = models.CharField(blank=True, max_length=240)
# files generated during publication
- cover = models.FileField(_('cover'), upload_to=book_upload_path('png'),
- null=True, blank=True)
- ebook_formats = ['pdf', 'epub', 'mobi', 'fb2', 'txt']
+ cover = EbookField('cover', _('cover'),
+ upload_to=book_upload_path('jpg'), null=True, blank=True)
+ ebook_formats = constants.EBOOK_FORMATS
formats = ebook_formats + ['html', 'xml']
- parent = models.ForeignKey('self', blank=True, null=True, related_name='children')
+ parent = models.ForeignKey('self', blank=True, null=True,
+ related_name='children')
_related_info = jsonfield.JSONField(blank=True, null=True, editable=False)
has_daisy_file.short_description = 'DAISY'
has_daisy_file.boolean = True
- def wldocument(self, parse_dublincore=True):
+ def wldocument(self, parse_dublincore=True, inherit=True):
from catalogue.import_utils import ORMDocProvider
from librarian.parser import WLDocument
+ if inherit and self.parent:
+ meta_fallbacks = self.parent.cover_info()
+ else:
+ meta_fallbacks = None
+
return WLDocument.from_file(self.xml_file.path,
provider=ORMDocProvider(self),
- parse_dublincore=parse_dublincore)
-
- def build_html(self):
- from django.core.files.base import ContentFile
- from slughifi import slughifi
- from sortify import sortify
- from librarian import html
-
- meta_tags = list(self.tags.filter(
- category__in=('author', 'epoch', 'genre', 'kind')))
- book_tag = self.book_tag()
-
- html_output = self.wldocument(parse_dublincore=False).as_html()
- if html_output:
- self.html_file.save('%s.html' % self.slug,
- ContentFile(html_output.get_string()))
-
- # get ancestor l-tags for adding to new fragments
- ancestor_tags = []
- p = self.parent
- while p:
- ancestor_tags.append(p.book_tag())
- p = p.parent
-
- # Delete old fragments and create them from scratch
- self.fragments.all().delete()
- # Extract fragments
- closed_fragments, open_fragments = html.extract_fragments(self.html_file.path)
- for fragment in closed_fragments.values():
- try:
- theme_names = [s.strip() for s in fragment.themes.split(',')]
- except AttributeError:
- continue
- themes = []
- for theme_name in theme_names:
- if not theme_name:
- continue
- tag, created = Tag.objects.get_or_create(slug=slughifi(theme_name), category='theme')
- if created:
- tag.name = theme_name
- tag.sort_key = sortify(theme_name.lower())
- tag.save()
- themes.append(tag)
- if not themes:
- continue
-
- text = fragment.to_string()
- short_text = truncate_html_words(text, 15)
- if text == short_text:
- short_text = ''
- new_fragment = Fragment.objects.create(anchor=fragment.id, book=self,
- text=text, short_text=short_text)
-
- new_fragment.save()
- new_fragment.tags = set(meta_tags + themes + [book_tag] + ancestor_tags)
- self.save()
- self.html_built.send(sender=self)
- return True
- return False
-
- # Thin wrappers for builder tasks
- def build_cover(self):
- """(Re)builds the cover image."""
- return tasks.build_cover.delay(self.pk)
- def build_pdf(self, *args, **kwargs):
- """(Re)builds PDF."""
- return tasks.build_pdf.delay(self.pk, *args, **kwargs)
- def build_epub(self, *args, **kwargs):
- """(Re)builds EPUB."""
- return tasks.build_epub.delay(self.pk, *args, **kwargs)
- def build_mobi(self, *args, **kwargs):
- """(Re)builds MOBI."""
- return tasks.build_mobi.delay(self.pk, *args, **kwargs)
- def build_fb2(self, *args, **kwargs):
- """(Re)build FB2"""
- return tasks.build_fb2.delay(self.pk, *args, **kwargs)
- def build_txt(self, *args, **kwargs):
- """(Re)builds TXT."""
- return tasks.build_txt.delay(self.pk, *args, **kwargs)
+ parse_dublincore=parse_dublincore,
+ meta_fallbacks=meta_fallbacks)
@staticmethod
def zip_format(format_):
def pretty_file_name(book):
return "%s/%s.%s" % (
- b.extra_info['author'],
- b.slug,
+ book.extra_info['author'],
+ book.slug,
format_)
field_name = "%s_file" % format_
books = Book.objects.filter(parent=None).exclude(**{field_name: ""})
paths = [(pretty_file_name(b), getattr(b, field_name).path)
for b in books.iterator()]
- return create_zip(paths,
- getattr(settings, "ALL_%s_ZIP" % format_.upper()))
+ return create_zip(paths, app_settings.FORMAT_ZIPS[format_])
def zip_audiobooks(self, format_):
bm = BookMedia.objects.filter(book=self, type=format_)
@classmethod
def from_text_and_meta(cls, raw_file, book_info, overwrite=False,
- build_epub=True, build_txt=True, build_pdf=True, build_mobi=True, build_fb2=True,
- search_index=True, search_index_tags=True, search_index_reuse=False):
+ dont_build=None, search_index=True,
+ search_index_tags=True, search_index_reuse=False):
+ if dont_build is None:
+ dont_build = set()
+ dont_build = set.union(set(dont_build), set(app_settings.DONT_BUILD))
# check for parts before we do anything
children = []
if created:
book_shelves = []
+ old_cover = None
else:
if not overwrite:
raise Book.AlreadyExists(_('Book %s already exists') % (
book_slug))
# Save shelves for this book
book_shelves = list(book.tags.filter(category='set'))
+ old_cover = book.cover_info()
+
+ # Save XML file
+ book.xml_file.save('%s.xml' % book.slug, raw_file, save=False)
book.language = book_info.language
book.title = book_info.title
book.tags = set(meta_tags + book_shelves)
- obsolete_children = set(b for b in book.children.all() if b not in children)
+ cover_changed = old_cover != book.cover_info()
+ obsolete_children = set(b for b in book.children.all()
+ if b not in children)
for n, child_book in enumerate(children):
child_book.parent = book
child_book.parent_number = n
child_book.save()
+ if cover_changed:
+ child_book.parent_cover_changed()
# Disown unfaithful children and let them cope on their own.
for child in obsolete_children:
child.parent = None
child.parent_number = 0
child.save()
tasks.fix_tree_tags.delay(child)
-
- # Save XML file
- book.xml_file.save('%s.xml' % book.slug, raw_file, save=False)
+ if old_cover:
+ child.parent_cover_changed()
# delete old fragments when overwriting
book.fragments.all().delete()
# Build HTML, fix the tree tags, build cover.
- has_own_text = bool(book.build_html())
+ has_own_text = bool(book.html_file.build())
tasks.fix_tree_tags.delay(book)
- book.build_cover(book_info)
+ if 'cover' not in dont_build:
+ book.cover.build_delay()
# No saves behind this point.
if has_own_text:
- if not settings.NO_BUILD_TXT and build_txt:
- book.build_txt()
- if not settings.NO_BUILD_FB2 and build_fb2:
- book.build_fb2()
-
- if not settings.NO_BUILD_EPUB and build_epub:
- book.build_epub()
-
- if not settings.NO_BUILD_PDF and build_pdf:
- book.build_pdf()
-
- if not settings.NO_BUILD_MOBI and build_mobi:
- book.build_mobi()
+ for format_ in constants.EBOOK_FORMATS_WITHOUT_CHILDREN:
+ if format_ not in dont_build:
+ getattr(book, '%s_file' % format_).build_delay()
+ for format_ in constants.EBOOK_FORMATS_WITH_CHILDREN:
+ if format_ not in dont_build:
+ getattr(book, '%s_file' % format_).build_delay()
if not settings.NO_SEARCH_INDEX and search_index:
book.search_index(index_tags=search_index_tags, reuse_index=search_index_reuse)
sub_parent_tags = parent_tags + [book.book_tag()]
for frag in book.fragments.all():
affected_tags.update(frag.tags)
- frag.tags = list(frag.tags.exclude(category='book')) + sub_parent_tags
+ frag.tags = list(frag.tags.exclude(category='book')
+ ) + sub_parent_tags
for child in book.children.all():
affected_tags.update(fix_subtree(child, sub_parent_tags))
return affected_tags
book.reset_theme_counter()
book = book.parent
+ def cover_info(self, inherit=True):
+ """Returns a dictionary to serve as fallback for BookInfo.
+
+ For now, the only thing inherited is the cover image.
+ """
+ need = False
+ info = {}
+ for field in ('cover_url', 'cover_by', 'cover_source'):
+ val = self.extra_info.get(field)
+ if val:
+ info[field] = val
+ else:
+ need = True
+ if inherit and need and self.parent is not None:
+ parent_info = self.parent.cover_info()
+ parent_info.update(info)
+ info = parent_info
+ return info
+
+ def parent_cover_changed(self):
+ """Called when parent book's cover image is changed."""
+ if not self.cover_info(inherit=False):
+ if 'cover' not in app_settings.DONT_BUILD:
+ self.cover.build_delay()
+ for format_ in constants.EBOOK_FORMATS_WITH_COVERS:
+ if format_ not in app_settings.DONT_BUILD:
+ getattr(self, '%s_file' % format_).build_delay()
+ for child in self.children.all():
+ child.parent_cover_changed()
+
def related_info(self):
"""Keeps info about related objects (tags, media) in cache field."""
if self._related_info is not None:
return None
-def _has_factory(ftype):
- has = lambda self: bool(getattr(self, "%s_file" % ftype))
- has.short_description = ftype.upper()
- has.__doc__ = None
- has.boolean = True
- has.__name__ = "has_%s_file" % ftype
- return has
-
-
# add the file fields
-for t in Book.formats:
- field_name = "%s_file" % t
- models.FileField(_("%s file" % t.upper()),
- upload_to=book_upload_path(t),
- blank=True).contribute_to_class(Book, field_name)
-
- setattr(Book, "has_%s_file" % t, _has_factory(t))
+for format_ in Book.formats:
+ field_name = "%s_file" % format_
+ EbookField(format_, _("%s file" % format_.upper()),
+ upload_to=book_upload_path(format_),
+ blank=True, default='').contribute_to_class(Book, field_name)
raise e
-def _build_ebook(book_id, ext, transform):
- """Generic ebook builder."""
- from django.core.files import File
- from catalogue.models import Book
-
- book = Book.objects.get(pk=book_id)
- out = transform(book.wldocument())
- field_name = '%s_file' % ext
- # Update instead of saving the model to avoid race condition.
- getattr(book, field_name).save('%s.%s' % (book.slug, ext),
- File(open(out.get_filename())),
- save=False
- )
- Book.objects.filter(pk=book_id).update(**{
- field_name: getattr(book, field_name)
- })
-
-
-@task(ignore_result=True)
-def build_txt(book_id):
- """(Re)builds the TXT file for a book."""
- _build_ebook(book_id, 'txt', lambda doc: doc.as_text())
-
-
-@task(ignore_result=True, rate_limit=settings.CATALOGUE_PDF_RATE_LIMIT)
-def build_pdf(book_id):
- """(Re)builds the pdf file for a book."""
- from catalogue.models import Book
- from catalogue.utils import remove_zip
- from waiter.utils import clear_cache
-
- _build_ebook(book_id, 'pdf',
- lambda doc: doc.as_pdf(morefloats=settings.LIBRARIAN_PDF_MOREFLOATS))
- # Remove cached downloadables
- remove_zip(settings.ALL_PDF_ZIP)
- book = Book.objects.get(pk=book_id)
- clear_cache(book.slug)
-
-
-@task(ignore_result=True, rate_limit=settings.CATALOGUE_EPUB_RATE_LIMIT)
-def build_epub(book_id):
- """(Re)builds the EPUB file for a book."""
- from catalogue.utils import remove_zip
-
- _build_ebook(book_id, 'epub', lambda doc: doc.as_epub())
- # remove zip with all epub files
- remove_zip(settings.ALL_EPUB_ZIP)
-
-
-@task(ignore_result=True, rate_limit=settings.CATALOGUE_MOBI_RATE_LIMIT)
-def build_mobi(book_id):
- """(Re)builds the MOBI file for a book."""
- from catalogue.utils import remove_zip
-
- _build_ebook(book_id, 'mobi', lambda doc: doc.as_mobi())
- # remove zip with all mobi files
- remove_zip(settings.ALL_MOBI_ZIP)
-
-
-@task(ignore_result=True, rate_limit=settings.CATALOGUE_FB2_RATE_LIMIT)
-def build_fb2(book_id, *args, **kwargs):
- """(Re)builds the FB2 file for a book."""
- from catalogue.utils import remove_zip
-
- _build_ebook(book_id, 'fb2', lambda doc: doc.as_fb2())
- # remove zip with all fb2 files
- remove_zip(settings.ALL_FB2_ZIP)
-
-
@task(ignore_result=True, rate_limit=settings.CATALOGUE_CUSTOMPDF_RATE_LIMIT)
def build_custom_pdf(book_id, customizations, file_name):
"""Builds a custom PDF file."""
customizations=customizations,
morefloats=settings.LIBRARIAN_PDF_MOREFLOATS)
DefaultStorage().save(file_name, File(open(pdf.get_filename())))
-
-
-@task(ignore_result=True)
-def build_cover(book_id):
- """(Re)builds the cover image."""
- from StringIO import StringIO
- from django.core.files.base import ContentFile
- from librarian.cover import WLCover
- from catalogue.models import Book
-
- book = Book.objects.get(pk=book_id)
- book_info = book.wldocument().book_info
- cover = WLCover(book_info).image()
- imgstr = StringIO()
- cover.save(imgstr, 'png')
- book.cover.save(None, ContentFile(imgstr.getvalue()), save=False)
- Book.objects.filter(pk=book_id).update(cover=book.cover)
-from django.conf import settings
+# -*- coding: utf-8 -*-
+# This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
+# Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
+#
from django.test import TestCase
-import shutil
+from django.test.utils import override_settings
import tempfile
from slughifi import slughifi
from librarian import WLURI
+@override_settings(
+ MEDIA_ROOT=tempfile.mkdtemp(prefix='djangotest_'),
+ CATALOGUE_DONT_BUILD={'pdf', 'mobi', 'epub', 'txt', 'fb2', 'cover'},
+ NO_SEARCH_INDEX = True,
+ CELERY_ALWAYS_EAGER = True,
+ CACHES={
+ 'api': {'BACKEND': 'django.core.cache.backends.dummy.DummyCache'},
+ 'default': {'BACKEND': 'django.core.cache.backends.dummy.DummyCache'},
+ 'permanent': {'BACKEND': 'django.core.cache.backends.dummy.DummyCache'},
+ },
+)
class WLTestCase(TestCase):
"""
Generic base class for tests. Adds settings freeze and clears MEDIA_ROOT.
"""
longMessage = True
- def setUp(self):
- self._MEDIA_ROOT, settings.MEDIA_ROOT = settings.MEDIA_ROOT, tempfile.mkdtemp(prefix='djangotest_')
- settings.NO_SEARCH_INDEX = settings.NO_BUILD_PDF = settings.NO_BUILD_MOBI = settings.NO_BUILD_EPUB = settings.NO_BUILD_TXT = settings.NO_BUILD_FB2 = True
- settings.CELERY_ALWAYS_EAGER = True
- self._CACHES, settings.CACHES = settings.CACHES, {
- 'default': {
- 'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
- }
- }
-
- def tearDown(self):
- shutil.rmtree(settings.MEDIA_ROOT, True)
- settings.MEDIA_ROOT = self._MEDIA_ROOT
- settings.CACHES = self._CACHES
-
class PersonStub(object):
# -*- coding: utf-8 -*-
-from __future__ import with_statement
+from django.conf import settings
from django.core.files.base import ContentFile, File
from catalogue.test_utils import *
self.book = models.Book.from_xml_file(xml)
def test_gen_pdf(self):
- self.book.build_pdf()
+ self.book.pdf_file.build()
book = models.Book.objects.get(pk=self.book.pk)
self.assertTrue(path.exists(book.pdf_file.path))
"""This book contains a child."""
xml = path.join(path.dirname(__file__), "files/fraszki.xml")
parent = models.Book.from_xml_file(xml)
- parent.build_pdf()
+ parent.pdf_file.build()
parent = models.Book.objects.get(pk=parent.pk)
self.assertTrue(path.exists(parent.pdf_file.path))
# -*- coding: utf-8 -*-
+from django.conf import settings
from os.path import basename, exists, join, dirname
from django.core.files.base import ContentFile, File
"""
from waiter.utils import clear_cache
clear_cache('book/%s' % book.slug)
+
+
+class AppSettings(object):
+ """Allows specyfying custom settings for an app, with default values.
+
+ Just subclass, set some properties and instantiate with a prefix.
+ Getting a SETTING from an instance will check for prefix_SETTING
+ in project settings if set, else take the default. The value will be
+ then filtered through _more_SETTING method, if there is one.
+
+ """
+ def __init__(self, prefix):
+ self._prefix = prefix
+
+ def __getattribute__(self, name):
+ if name.startswith('_'):
+ return object.__getattribute__(self, name)
+ value = getattr(settings,
+ "%s_%s" % (self._prefix, name),
+ object.__getattribute__(self, name))
+ more = "_more_%s" % name
+ if hasattr(self, more):
+ value = getattr(self, more)(value)
+ return value
from __future__ import with_statement
from os import path
-from django.test import TestCase
from picture.models import Picture
+from catalogue.test_utils import WLTestCase
-class PictureTest(TestCase):
+class PictureTest(WLTestCase):
def test_import(self):
picture = Picture.from_xml_file(path.join(path.dirname(__file__), "files/kandinsky-composition-viii.xml"))
# -*- coding: utf-8 -*-
-
-from __future__ import with_statement
-
from django.conf import settings
-from search import Index, Search, IndexStore, JVM, SearchResult
-from catalogue import models
+from django.test.utils import override_settings
from catalogue.test_utils import WLTestCase
from lucene import PolishAnalyzer, Version
-#from nose.tools import raises
from os import path
+import tempfile
+from catalogue import models
+from search import Search, SearchResult
+@override_settings(
+ SEARCH_INDEX = tempfile.mkdtemp(prefix='djangotest_search_'),
+)
class BookSearchTests(WLTestCase):
def setUp(self):
- JVM.attachCurrentThread()
WLTestCase.setUp(self)
- settings.NO_SEARCH_INDEX = False
- settings.SEARCH_INDEX = path.join(settings.MEDIA_ROOT, 'search')
txt = path.join(path.dirname(__file__), 'files/fraszka-do-anusie.xml')
- self.book = models.Book.from_xml_file(txt)
-
- index = Index()
- index.open()
- try:
- index.index_book(self.book)
- except:
- index.close()
-
+ with self.settings(NO_SEARCH_INDEX=False):
+ self.book = models.Book.from_xml_file(txt)
self.search = Search()
def test_search_perfect_book_author(self):
-Subproject commit 17a9ed3b7ef12e0786ddf46bf8a52b1087224762
+Subproject commit cbe81ee35b07783b4f52c3d3dda83db7aaf82d34
MAX_TAG_LIST = 6
NO_SEARCH_INDEX = False
-NO_BUILD_EPUB = False
-NO_BUILD_TXT = False
-NO_BUILD_FB2 = False
-# You'll need XeLaTeX to generate PDF files.
-NO_BUILD_PDF = True
NO_CUSTOM_PDF = True
-# You'll need Calibre installed to generate MOBI files.
-NO_BUILD_MOBI = True
-
-
-ALL_EPUB_ZIP = 'wolnelektury_pl_epub'
-ALL_PDF_ZIP = 'wolnelektury_pl_pdf'
-ALL_MOBI_ZIP = 'wolnelektury_pl_mobi'
-ALL_FB2_ZIP = 'wolnelektury_pl_fb2'
CATALOGUE_DEFAULT_LANGUAGE = 'pol'
PUBLISH_PLAN_FEED = 'http://redakcja.wolnelektury.pl/documents/track/editor-proofreading/?published=false'
# limit rate for ebooks creation
-CATALOGUE_PDF_RATE_LIMIT = '1/m'
-CATALOGUE_EPUB_RATE_LIMIT = '6/m'
-CATALOGUE_FB2_RATE_LIMIT = '5/m'
-CATALOGUE_MOBI_RATE_LIMIT = '5/m'
CATALOGUE_CUSTOMPDF_RATE_LIMIT = '1/m'
# set to 'new' or 'old' to skip time-consuming test
fnp / wolnelektury.git / commitdiff