epub creation on import, log produced files (Fixed #723)

librarian bump

diff --git a/apps/catalogue/migrations/0010_auto__add_filerecord.py b/apps/catalogue/migrations/0010_auto__add_filerecord.py
new file mode 100644 (file)
index 0000000..f072365
--- /dev/null
+++ b/apps/catalogue/migrations/0010_auto__add_filerecord.py
@@ -0,0 +1,156 @@
+# encoding: utf-8
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+class Migration(SchemaMigration):
+    
+    def forwards(self, orm):
+        
+        # Adding model 'FileRecord'
+        db.create_table('catalogue_filerecord', (
+            ('time', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)),
+            ('sha1', self.gf('django.db.models.fields.CharField')(max_length=40)),
+            ('type', self.gf('django.db.models.fields.CharField')(max_length=20, db_index=True)),
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('slug', self.gf('django.db.models.fields.SlugField')(max_length=120, db_index=True)),
+        ))
+        db.send_create_signal('catalogue', ['FileRecord'])
+    
+    
+    def backwards(self, orm):
+        
+        # Deleting model 'FileRecord'
+        db.delete_table('catalogue_filerecord')
+    
+    
+    models = {
+        'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'blank': 'True'})
+        },
+        'auth.permission': {
+            'Meta': {'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'blank': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'catalogue.book': {
+            'Meta': {'object_name': 'Book'},
+            '_short_html': ('django.db.models.fields.TextField', [], {}),
+            '_short_html_de': ('django.db.models.fields.TextField', [], {'null': True, 'blank': True}),
+            '_short_html_en': ('django.db.models.fields.TextField', [], {'null': True, 'blank': True}),
+            '_short_html_es': ('django.db.models.fields.TextField', [], {'null': True, 'blank': True}),
+            '_short_html_fr': ('django.db.models.fields.TextField', [], {'null': True, 'blank': True}),
+            '_short_html_lt': ('django.db.models.fields.TextField', [], {'null': True, 'blank': True}),
+            '_short_html_pl': ('django.db.models.fields.TextField', [], {'null': True, 'blank': True}),
+            '_short_html_ru': ('django.db.models.fields.TextField', [], {'null': True, 'blank': True}),
+            '_short_html_uk': ('django.db.models.fields.TextField', [], {'null': True, 'blank': True}),
+            '_tag_counter': ('catalogue.fields.JSONField', [], {'null': 'True'}),
+            '_theme_counter': ('catalogue.fields.JSONField', [], {'null': 'True'}),
+            'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'epub_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'blank': 'True'}),
+            'extra_info': ('catalogue.fields.JSONField', [], {}),
+            'gazeta_link': ('django.db.models.fields.CharField', [], {'max_length': '240', 'blank': 'True'}),
+            'html_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'mp3_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'blank': 'True'}),
+            'odt_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'blank': 'True'}),
+            'ogg_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'blank': 'True'}),
+            'parent': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'children'", 'null': 'True', 'to': "orm['catalogue.Book']"}),
+            'parent_number': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'pdf_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'blank': 'True'}),
+            'slug': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '120', 'db_index': 'True'}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '120'}),
+            'txt_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'blank': 'True'}),
+            'wiki_link': ('django.db.models.fields.CharField', [], {'max_length': '240', 'blank': 'True'}),
+            'xml_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'blank': 'True'})
+        },
+        'catalogue.bookstub': {
+            'Meta': {'object_name': 'BookStub'},
+            'author': ('django.db.models.fields.CharField', [], {'max_length': '120'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'pd': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'slug': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '120', 'db_index': 'True'}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '120'}),
+            'translator': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'translator_death': ('django.db.models.fields.TextField', [], {'blank': 'True'})
+        },
+        'catalogue.filerecord': {
+            'Meta': {'object_name': 'FileRecord'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'sha1': ('django.db.models.fields.CharField', [], {'max_length': '40'}),
+            'slug': ('django.db.models.fields.SlugField', [], {'max_length': '120', 'db_index': 'True'}),
+            'time': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'type': ('django.db.models.fields.CharField', [], {'max_length': '20', 'db_index': 'True'})
+        },
+        'catalogue.fragment': {
+            'Meta': {'object_name': 'Fragment'},
+            '_short_html': ('django.db.models.fields.TextField', [], {}),
+            '_short_html_de': ('django.db.models.fields.TextField', [], {'null': True, 'blank': True}),
+            '_short_html_en': ('django.db.models.fields.TextField', [], {'null': True, 'blank': True}),
+            '_short_html_es': ('django.db.models.fields.TextField', [], {'null': True, 'blank': True}),
+            '_short_html_fr': ('django.db.models.fields.TextField', [], {'null': True, 'blank': True}),
+            '_short_html_lt': ('django.db.models.fields.TextField', [], {'null': True, 'blank': True}),
+            '_short_html_pl': ('django.db.models.fields.TextField', [], {'null': True, 'blank': True}),
+            '_short_html_ru': ('django.db.models.fields.TextField', [], {'null': True, 'blank': True}),
+            '_short_html_uk': ('django.db.models.fields.TextField', [], {'null': True, 'blank': True}),
+            'anchor': ('django.db.models.fields.CharField', [], {'max_length': '120'}),
+            'book': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'fragments'", 'to': "orm['catalogue.Book']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'short_text': ('django.db.models.fields.TextField', [], {}),
+            'text': ('django.db.models.fields.TextField', [], {})
+        },
+        'catalogue.tag': {
+            'Meta': {'unique_together': "(('slug', 'category'),)", 'object_name': 'Tag'},
+            'book_count': ('django.db.models.fields.IntegerField', [], {'null': 'True'}),
+            'category': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}),
+            'death': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'gazeta_link': ('django.db.models.fields.CharField', [], {'max_length': '240', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'main_page': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}),
+            'slug': ('django.db.models.fields.SlugField', [], {'max_length': '120', 'db_index': 'True'}),
+            'sort_key': ('django.db.models.fields.SlugField', [], {'max_length': '120', 'db_index': 'True'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True', 'blank': 'True'}),
+            'wiki_link': ('django.db.models.fields.CharField', [], {'max_length': '240', 'blank': 'True'})
+        },
+        'catalogue.tagrelation': {
+            'Meta': {'unique_together': "(('tag', 'content_type', 'object_id'),)", 'object_name': 'TagRelation', 'db_table': "'catalogue_tag_relation'"},
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'object_id': ('django.db.models.fields.PositiveIntegerField', [], {'db_index': 'True'}),
+            'tag': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'items'", 'to': "orm['catalogue.Tag']"})
+        },
+        'contenttypes.contenttype': {
+            'Meta': {'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        }
+    }
+    
+    complete_apps = ['catalogue']

diff --git a/apps/catalogue/models.py b/apps/catalogue/models.py
index 9228284..8c83faa 100644 (file)
--- a/apps/catalogue/models.py
+++ b/apps/catalogue/models.py
@@ -17,7 +17,7 @@ from newtagging.models import TagBase, tags_updated
 from newtagging import managers
 from catalogue.fields import JSONField
 
-from librarian import html, dcparser
+from librarian import dcparser, html, epub
 from mutagen import id3
 
 
@@ -332,6 +332,10 @@ class Book(models.Model):
         from tempfile import NamedTemporaryFile
         from slughifi import slughifi
         from markupstring import MarkupString
+        from hashlib import sha1
+        from django.core.files.base import ContentFile
+        from django.core.files.storage import default_storage
+        from StringIO import StringIO
 
         # Read book metadata
         book_base, book_slug = book_info.url.rsplit('/', 1)
@@ -400,6 +404,12 @@ class Book(models.Model):
         if html.transform(book.xml_file.path, html_file, parse_dublincore=False):
             book.html_file.save('%s.html' % book.slug, File(html_file), save=False)
 
+            # Create EPUB
+            epub_file = StringIO()
+            epub.transform(book.xml_file, epub_file)
+            book.epub_file.save('%s.epub' % book.slug, ContentFile(epub_file.getvalue()), save=False)
+            FileRecord(slug=book.slug, type='epub', sha1=sha1(epub_file.getvalue()).hexdigest()).save()
+
             # Extract fragments
             closed_fragments, open_fragments = html.extract_fragments(book.html_file.path)
             for fragment in closed_fragments.values():
@@ -543,6 +553,21 @@ class BookStub(models.Model):
         return self.title
 
 
+class FileRecord(models.Model):
+    slug = models.SlugField(_('slug'), max_length=120, db_index=True)
+    type = models.CharField(_('type'), max_length=20, db_index=True)
+    sha1 = models.CharField(_('sha-1 hash'), max_length=40)
+    time = models.DateTimeField(_('time'), auto_now_add=True)
+
+    class Meta:
+        ordering = ('-time','-slug', '-type')
+        verbose_name = _('file record')
+        verbose_name_plural = _('file records')
+
+    def __unicode__(self):
+        return "%s %s.%s" % (self.sha1,  self.slug, self.type)
+
+
 def _tags_updated_handler(sender, affected_tags, **kwargs):
     # reset tag global counter
     Tag.objects.filter(pk__in=[tag.pk for tag in affected_tags]).update(book_count=None)

diff --git a/lib/librarian b/lib/librarian
index 5938bc7..4edcff3 160000 (submodule)
--- a/lib/librarian
+++ b/lib/librarian
@@ -1 +1 @@
-Subproject commit 5938bc78f11ebad067eae86f51b4f5243003f946
+Subproject commit 4edcff3a9e2a4bb2ecb7ab228e5bf37dd28d7e14