From 7f302cfc1fac98e08e65061111fd1395bb399209 Mon Sep 17 00:00:00 2001 From: Radek Czajka Date: Mon, 5 Jul 2010 17:46:59 +0200 Subject: [PATCH] epub creation on import, log produced files (Fixed #723) librarian bump --- .../migrations/0010_auto__add_filerecord.py | 156 ++++++++++++++++++ apps/catalogue/models.py | 27 ++- lib/librarian | 2 +- 3 files changed, 183 insertions(+), 2 deletions(-) create mode 100644 apps/catalogue/migrations/0010_auto__add_filerecord.py diff --git a/apps/catalogue/migrations/0010_auto__add_filerecord.py b/apps/catalogue/migrations/0010_auto__add_filerecord.py new file mode 100644 index 000000000..f072365f8 --- /dev/null +++ b/apps/catalogue/migrations/0010_auto__add_filerecord.py @@ -0,0 +1,156 @@ +# encoding: utf-8 +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + +class Migration(SchemaMigration): + + def forwards(self, orm): + + # Adding model 'FileRecord' + db.create_table('catalogue_filerecord', ( + ('time', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)), + ('sha1', self.gf('django.db.models.fields.CharField')(max_length=40)), + ('type', self.gf('django.db.models.fields.CharField')(max_length=20, db_index=True)), + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('slug', self.gf('django.db.models.fields.SlugField')(max_length=120, db_index=True)), + )) + db.send_create_signal('catalogue', ['FileRecord']) + + + def backwards(self, orm): + + # Deleting model 'FileRecord' + db.delete_table('catalogue_filerecord') + + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'blank': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'catalogue.book': { + 'Meta': {'object_name': 'Book'}, + '_short_html': ('django.db.models.fields.TextField', [], {}), + '_short_html_de': ('django.db.models.fields.TextField', [], {'null': True, 'blank': True}), + '_short_html_en': ('django.db.models.fields.TextField', [], {'null': True, 'blank': True}), + '_short_html_es': ('django.db.models.fields.TextField', [], {'null': True, 'blank': True}), + '_short_html_fr': ('django.db.models.fields.TextField', [], {'null': True, 'blank': True}), + '_short_html_lt': ('django.db.models.fields.TextField', [], {'null': True, 'blank': True}), + '_short_html_pl': ('django.db.models.fields.TextField', [], {'null': True, 'blank': True}), + '_short_html_ru': ('django.db.models.fields.TextField', [], {'null': True, 'blank': True}), + '_short_html_uk': ('django.db.models.fields.TextField', [], {'null': True, 'blank': True}), + '_tag_counter': ('catalogue.fields.JSONField', [], {'null': 'True'}), + '_theme_counter': ('catalogue.fields.JSONField', [], {'null': 'True'}), + 'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'epub_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'blank': 'True'}), + 'extra_info': ('catalogue.fields.JSONField', [], {}), + 'gazeta_link': ('django.db.models.fields.CharField', [], {'max_length': '240', 'blank': 'True'}), + 'html_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mp3_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'blank': 'True'}), + 'odt_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'blank': 'True'}), + 'ogg_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'blank': 'True'}), + 'parent': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'children'", 'null': 'True', 'to': "orm['catalogue.Book']"}), + 'parent_number': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'pdf_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'blank': 'True'}), + 'slug': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '120', 'db_index': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '120'}), + 'txt_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'blank': 'True'}), + 'wiki_link': ('django.db.models.fields.CharField', [], {'max_length': '240', 'blank': 'True'}), + 'xml_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'blank': 'True'}) + }, + 'catalogue.bookstub': { + 'Meta': {'object_name': 'BookStub'}, + 'author': ('django.db.models.fields.CharField', [], {'max_length': '120'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'pd': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'slug': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '120', 'db_index': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '120'}), + 'translator': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'translator_death': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'catalogue.filerecord': { + 'Meta': {'object_name': 'FileRecord'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'sha1': ('django.db.models.fields.CharField', [], {'max_length': '40'}), + 'slug': ('django.db.models.fields.SlugField', [], {'max_length': '120', 'db_index': 'True'}), + 'time': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'type': ('django.db.models.fields.CharField', [], {'max_length': '20', 'db_index': 'True'}) + }, + 'catalogue.fragment': { + 'Meta': {'object_name': 'Fragment'}, + '_short_html': ('django.db.models.fields.TextField', [], {}), + '_short_html_de': ('django.db.models.fields.TextField', [], {'null': True, 'blank': True}), + '_short_html_en': ('django.db.models.fields.TextField', [], {'null': True, 'blank': True}), + '_short_html_es': ('django.db.models.fields.TextField', [], {'null': True, 'blank': True}), + '_short_html_fr': ('django.db.models.fields.TextField', [], {'null': True, 'blank': True}), + '_short_html_lt': ('django.db.models.fields.TextField', [], {'null': True, 'blank': True}), + '_short_html_pl': ('django.db.models.fields.TextField', [], {'null': True, 'blank': True}), + '_short_html_ru': ('django.db.models.fields.TextField', [], {'null': True, 'blank': True}), + '_short_html_uk': ('django.db.models.fields.TextField', [], {'null': True, 'blank': True}), + 'anchor': ('django.db.models.fields.CharField', [], {'max_length': '120'}), + 'book': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'fragments'", 'to': "orm['catalogue.Book']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'short_text': ('django.db.models.fields.TextField', [], {}), + 'text': ('django.db.models.fields.TextField', [], {}) + }, + 'catalogue.tag': { + 'Meta': {'unique_together': "(('slug', 'category'),)", 'object_name': 'Tag'}, + 'book_count': ('django.db.models.fields.IntegerField', [], {'null': 'True'}), + 'category': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}), + 'death': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'gazeta_link': ('django.db.models.fields.CharField', [], {'max_length': '240', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'main_page': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}), + 'slug': ('django.db.models.fields.SlugField', [], {'max_length': '120', 'db_index': 'True'}), + 'sort_key': ('django.db.models.fields.SlugField', [], {'max_length': '120', 'db_index': 'True'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True', 'blank': 'True'}), + 'wiki_link': ('django.db.models.fields.CharField', [], {'max_length': '240', 'blank': 'True'}) + }, + 'catalogue.tagrelation': { + 'Meta': {'unique_together': "(('tag', 'content_type', 'object_id'),)", 'object_name': 'TagRelation', 'db_table': "'catalogue_tag_relation'"}, + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'object_id': ('django.db.models.fields.PositiveIntegerField', [], {'db_index': 'True'}), + 'tag': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'items'", 'to': "orm['catalogue.Tag']"}) + }, + 'contenttypes.contenttype': { + 'Meta': {'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + } + } + + complete_apps = ['catalogue'] diff --git a/apps/catalogue/models.py b/apps/catalogue/models.py index 922828440..8c83faa53 100644 --- a/apps/catalogue/models.py +++ b/apps/catalogue/models.py @@ -17,7 +17,7 @@ from newtagging.models import TagBase, tags_updated from newtagging import managers from catalogue.fields import JSONField -from librarian import html, dcparser +from librarian import dcparser, html, epub from mutagen import id3 @@ -332,6 +332,10 @@ class Book(models.Model): from tempfile import NamedTemporaryFile from slughifi import slughifi from markupstring import MarkupString + from hashlib import sha1 + from django.core.files.base import ContentFile + from django.core.files.storage import default_storage + from StringIO import StringIO # Read book metadata book_base, book_slug = book_info.url.rsplit('/', 1) @@ -400,6 +404,12 @@ class Book(models.Model): if html.transform(book.xml_file.path, html_file, parse_dublincore=False): book.html_file.save('%s.html' % book.slug, File(html_file), save=False) + # Create EPUB + epub_file = StringIO() + epub.transform(book.xml_file, epub_file) + book.epub_file.save('%s.epub' % book.slug, ContentFile(epub_file.getvalue()), save=False) + FileRecord(slug=book.slug, type='epub', sha1=sha1(epub_file.getvalue()).hexdigest()).save() + # Extract fragments closed_fragments, open_fragments = html.extract_fragments(book.html_file.path) for fragment in closed_fragments.values(): @@ -543,6 +553,21 @@ class BookStub(models.Model): return self.title +class FileRecord(models.Model): + slug = models.SlugField(_('slug'), max_length=120, db_index=True) + type = models.CharField(_('type'), max_length=20, db_index=True) + sha1 = models.CharField(_('sha-1 hash'), max_length=40) + time = models.DateTimeField(_('time'), auto_now_add=True) + + class Meta: + ordering = ('-time','-slug', '-type') + verbose_name = _('file record') + verbose_name_plural = _('file records') + + def __unicode__(self): + return "%s %s.%s" % (self.sha1, self.slug, self.type) + + def _tags_updated_handler(sender, affected_tags, **kwargs): # reset tag global counter Tag.objects.filter(pk__in=[tag.pk for tag in affected_tags]).update(book_count=None) diff --git a/lib/librarian b/lib/librarian index 5938bc78f..4edcff3a9 160000 --- a/lib/librarian +++ b/lib/librarian @@ -1 +1 @@ -Subproject commit 5938bc78f11ebad067eae86f51b4f5243003f946 +Subproject commit 4edcff3a9e2a4bb2ecb7ab228e5bf37dd28d7e14 -- 2.20.1