From: Radek Czajka Date: Fri, 5 Aug 2011 09:06:16 +0000 (+0200) Subject: celery tasks and some other changes X-Git-Url: https://git.mdrn.pl/audio.git/commitdiff_plain/d64f05ea901ccb7e61eacd9a5579bb81138b193d?hp=6a220b0ed7d874876b6a64c9e297af2da0d303de celery tasks and some other changes --- diff --git a/apps/archive/constants.py b/apps/archive/constants.py new file mode 100755 index 0000000..17f4e78 --- /dev/null +++ b/apps/archive/constants.py @@ -0,0 +1,14 @@ +from django.utils.translation import ugettext_lazy as _ + +class status: + WAITING = 1 + ENCODING = 2 + TAGGING = 3 + SENDING = 4 + + choices = [ + (WAITING, _('Waiting')), + (ENCODING, _('Encoding')), + (TAGGING, _('Tagging')), + (SENDING, _('Sending')), + ] diff --git a/apps/archive/cover.png b/apps/archive/cover.png new file mode 100644 index 0000000..1cdf8e1 Binary files /dev/null and b/apps/archive/cover.png differ diff --git a/apps/archive/forms.py b/apps/archive/forms.py index 922bd28..c551cf8 100755 --- a/apps/archive/forms.py +++ b/apps/archive/forms.py @@ -1,6 +1,6 @@ +from datetime import datetime import os import os.path -from datetime import datetime from django import forms from django.utils.translation import ugettext_lazy as _ @@ -8,7 +8,7 @@ import mutagen from archive.models import Audiobook from archive.settings import FILES_PATH -from archive.utils import ExistingFile +from archive.utils import ExistingFile, sha1_file class AudiobookForm(forms.ModelForm): class Meta: @@ -25,9 +25,12 @@ class AudiobookForm(forms.ModelForm): # save the file in model m.source_file.save( - os.path.join(FILES_PATH, os.path.basename(path)), + os.path.basename(path), ExistingFile(path)) + f = open(m.source_file.path) + m.source_sha1 = sha1_file(f) + f.close() + if commit: m.save() - diff --git a/apps/archive/migrations/0002_auto__del_field_audiobook_publishing_tags__del_field_audiobook_publish.py b/apps/archive/migrations/0002_auto__del_field_audiobook_publishing_tags__del_field_audiobook_publish.py new file mode 100644 index 0000000..03cb130 --- /dev/null +++ b/apps/archive/migrations/0002_auto__del_field_audiobook_publishing_tags__del_field_audiobook_publish.py @@ -0,0 +1,139 @@ +# encoding: utf-8 +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + +class Migration(SchemaMigration): + + def forwards(self, orm): + + # Deleting field 'Audiobook.publishing_tags' + db.delete_column('archive_audiobook', 'publishing_tags') + + # Deleting field 'Audiobook.published_tags' + db.delete_column('archive_audiobook', 'published_tags') + + # Deleting field 'Audiobook.publish_wait' + db.delete_column('archive_audiobook', 'publish_wait') + + # Deleting field 'Audiobook.published' + db.delete_column('archive_audiobook', 'published') + + # Deleting field 'Audiobook.publishing' + db.delete_column('archive_audiobook', 'publishing') + + # Adding field 'Audiobook.mp3_status' + db.add_column('archive_audiobook', 'mp3_status', self.gf('django.db.models.fields.SmallIntegerField')(null=True), keep_default=False) + + # Adding field 'Audiobook.mp3_task' + db.add_column('archive_audiobook', 'mp3_task', self.gf('django.db.models.fields.CharField')(max_length=64, null=True), keep_default=False) + + # Adding field 'Audiobook.mp3_tags' + db.add_column('archive_audiobook', 'mp3_tags', self.gf('jsonfield.fields.JSONField')(null=True), keep_default=False) + + # Adding field 'Audiobook.mp3_published_tags' + db.add_column('archive_audiobook', 'mp3_published_tags', self.gf('jsonfield.fields.JSONField')(null=True), keep_default=False) + + # Adding field 'Audiobook.mp3_published' + db.add_column('archive_audiobook', 'mp3_published', self.gf('django.db.models.fields.DateTimeField')(null=True), keep_default=False) + + # Adding field 'Audiobook.ogg_status' + db.add_column('archive_audiobook', 'ogg_status', self.gf('django.db.models.fields.SmallIntegerField')(null=True), keep_default=False) + + # Adding field 'Audiobook.ogg_task' + db.add_column('archive_audiobook', 'ogg_task', self.gf('django.db.models.fields.CharField')(max_length=64, null=True), keep_default=False) + + # Adding field 'Audiobook.ogg_tags' + db.add_column('archive_audiobook', 'ogg_tags', self.gf('jsonfield.fields.JSONField')(null=True), keep_default=False) + + # Adding field 'Audiobook.ogg_published_tags' + db.add_column('archive_audiobook', 'ogg_published_tags', self.gf('jsonfield.fields.JSONField')(null=True), keep_default=False) + + # Adding field 'Audiobook.ogg_published' + db.add_column('archive_audiobook', 'ogg_published', self.gf('django.db.models.fields.DateTimeField')(null=True), keep_default=False) + + + def backwards(self, orm): + + # Adding field 'Audiobook.publishing_tags' + db.add_column('archive_audiobook', 'publishing_tags', self.gf('jsonfield.fields.JSONField')(null=True), keep_default=False) + + # Adding field 'Audiobook.published_tags' + db.add_column('archive_audiobook', 'published_tags', self.gf('jsonfield.fields.JSONField')(null=True), keep_default=False) + + # Adding field 'Audiobook.publish_wait' + db.add_column('archive_audiobook', 'publish_wait', self.gf('django.db.models.fields.DateTimeField')(null=True), keep_default=False) + + # Adding field 'Audiobook.published' + db.add_column('archive_audiobook', 'published', self.gf('django.db.models.fields.DateTimeField')(null=True), keep_default=False) + + # Adding field 'Audiobook.publishing' + db.add_column('archive_audiobook', 'publishing', self.gf('django.db.models.fields.BooleanField')(default=False), keep_default=False) + + # Deleting field 'Audiobook.mp3_status' + db.delete_column('archive_audiobook', 'mp3_status') + + # Deleting field 'Audiobook.mp3_task' + db.delete_column('archive_audiobook', 'mp3_task') + + # Deleting field 'Audiobook.mp3_tags' + db.delete_column('archive_audiobook', 'mp3_tags') + + # Deleting field 'Audiobook.mp3_published_tags' + db.delete_column('archive_audiobook', 'mp3_published_tags') + + # Deleting field 'Audiobook.mp3_published' + db.delete_column('archive_audiobook', 'mp3_published') + + # Deleting field 'Audiobook.ogg_status' + db.delete_column('archive_audiobook', 'ogg_status') + + # Deleting field 'Audiobook.ogg_task' + db.delete_column('archive_audiobook', 'ogg_task') + + # Deleting field 'Audiobook.ogg_tags' + db.delete_column('archive_audiobook', 'ogg_tags') + + # Deleting field 'Audiobook.ogg_published_tags' + db.delete_column('archive_audiobook', 'ogg_published_tags') + + # Deleting field 'Audiobook.ogg_published' + db.delete_column('archive_audiobook', 'ogg_published') + + + models = { + 'archive.audiobook': { + 'Meta': {'ordering': "('title',)", 'object_name': 'Audiobook'}, + 'artist': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'conductor': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'date': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'encoded_by': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'modified': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'mp3_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True'}), + 'mp3_published': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'mp3_published_tags': ('jsonfield.fields.JSONField', [], {'null': 'True'}), + 'mp3_status': ('django.db.models.fields.SmallIntegerField', [], {'null': 'True'}), + 'mp3_tags': ('jsonfield.fields.JSONField', [], {'null': 'True'}), + 'mp3_task': ('django.db.models.fields.CharField', [], {'max_length': '64', 'null': 'True'}), + 'ogg_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True'}), + 'ogg_published': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'ogg_published_tags': ('jsonfield.fields.JSONField', [], {'null': 'True'}), + 'ogg_status': ('django.db.models.fields.SmallIntegerField', [], {'null': 'True'}), + 'ogg_tags': ('jsonfield.fields.JSONField', [], {'null': 'True'}), + 'ogg_task': ('django.db.models.fields.CharField', [], {'max_length': '64', 'null': 'True'}), + 'project': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['archive.Project']"}), + 'source_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'url': ('django.db.models.fields.URLField', [], {'max_length': '255'}) + }, + 'archive.project': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Project'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128', 'db_index': 'True'}), + 'sponsors': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}) + } + } + + complete_apps = ['archive'] diff --git a/apps/archive/migrations/0003_auto__add_field_audiobook_source_sha1__add_field_audiobook_translator.py b/apps/archive/migrations/0003_auto__add_field_audiobook_source_sha1__add_field_audiobook_translator.py new file mode 100644 index 0000000..9711b72 --- /dev/null +++ b/apps/archive/migrations/0003_auto__add_field_audiobook_source_sha1__add_field_audiobook_translator.py @@ -0,0 +1,84 @@ +# encoding: utf-8 +import datetime +from hashlib import sha1 +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +def sha1_file(f): + sha = sha1() + for piece in iter(lambda: f.read(1024*1024), ''): + sha.update(piece) + return sha.hexdigest() + + +class Migration(SchemaMigration): + + def forwards(self, orm): + + # Adding field 'Audiobook.source_sha1' + db.add_column('archive_audiobook', 'source_sha1', self.gf('django.db.models.fields.CharField')(default='', max_length=40), keep_default=False) + + # Adding field 'Audiobook.translator' + db.add_column('archive_audiobook', 'translator', self.gf('django.db.models.fields.CharField')(max_length=255, null=True, blank=True), keep_default=False) + + if not db.dry_run: + for obj in orm.Audiobook.objects.all(): + try: + f = open(obj.source_file.path) + except ValueError, e: + print "Audiobook has no source file" + pass + else: + obj.source_sha1 = sha1_file(f) + f.close() + obj.save() + + + def backwards(self, orm): + + # Deleting field 'Audiobook.source_sha1' + db.delete_column('archive_audiobook', 'source_sha1') + + # Deleting field 'Audiobook.translator' + db.delete_column('archive_audiobook', 'translator') + + + models = { + 'archive.audiobook': { + 'Meta': {'ordering': "('title',)", 'object_name': 'Audiobook'}, + 'artist': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'conductor': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'date': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'encoded_by': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'modified': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'mp3_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True'}), + 'mp3_published': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'mp3_published_tags': ('jsonfield.fields.JSONField', [], {'null': 'True'}), + 'mp3_status': ('django.db.models.fields.SmallIntegerField', [], {'null': 'True'}), + 'mp3_tags': ('jsonfield.fields.JSONField', [], {'null': 'True'}), + 'mp3_task': ('django.db.models.fields.CharField', [], {'max_length': '64', 'null': 'True'}), + 'ogg_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True'}), + 'ogg_published': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'ogg_published_tags': ('jsonfield.fields.JSONField', [], {'null': 'True'}), + 'ogg_status': ('django.db.models.fields.SmallIntegerField', [], {'null': 'True'}), + 'ogg_tags': ('jsonfield.fields.JSONField', [], {'null': 'True'}), + 'ogg_task': ('django.db.models.fields.CharField', [], {'max_length': '64', 'null': 'True'}), + 'project': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['archive.Project']"}), + 'source_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100'}), + 'source_sha1': ('django.db.models.fields.CharField', [], {'max_length': '40'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'translator': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'url': ('django.db.models.fields.URLField', [], {'max_length': '255'}) + }, + 'archive.project': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Project'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128', 'db_index': 'True'}), + 'sponsors': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}) + } + } + + complete_apps = ['archive'] diff --git a/apps/archive/models.py b/apps/archive/models.py index 78c5ab3..cf3829b 100644 --- a/apps/archive/models.py +++ b/apps/archive/models.py @@ -1,7 +1,10 @@ +# -*- coding: utf-8 -*- + from django.db import models from jsonfield.fields import JSONField from django.utils.translation import ugettext_lazy as _ -from archive.settings import FILES_PATH +from archive.constants import status +from archive.settings import FILES_PATH, ADVERT, LICENSE, ORGANIZATION, PROJECT # Create your models here. @@ -22,7 +25,8 @@ class Project(models.Model): class Audiobook(models.Model): - source_file = models.FileField(upload_to=FILES_PATH, verbose_name=_('source file'), editable=False) + source_file = models.FileField(upload_to='archive/files', verbose_name=_('source file'), editable=False) + source_sha1 = models.CharField(max_length=40, editable=False) title = models.CharField(max_length=255, verbose_name=_('title')) artist = models.CharField(max_length=255, verbose_name=_('artist')) @@ -31,16 +35,24 @@ class Audiobook(models.Model): date = models.CharField(max_length=255, verbose_name=_('date')) project = models.ForeignKey(Project, verbose_name=_('project')) url = models.URLField(max_length=255, verbose_name=_('book url')) + translator = models.CharField(max_length=255, null=True, blank=True, verbose_name=_('translator')) modified = models.DateTimeField(null=True, editable=False) - published_tags = JSONField(null=True, editable=False) + # publishing process + mp3_status = models.SmallIntegerField(null=True, editable=False, choices=status.choices) + mp3_task = models.CharField(max_length=64, null=True, editable=False) + mp3_tags = JSONField(null=True, editable=False) mp3_file = models.FileField(null=True, upload_to='archive/final', editable=False) + mp3_published_tags = JSONField(null=True, editable=False) + mp3_published = models.DateTimeField(null=True, editable=False) + + ogg_status = models.SmallIntegerField(null=True, editable=False, choices=status.choices) + ogg_task = models.CharField(max_length=64, null=True, editable=False) + ogg_tags = JSONField(null=True, editable=False) ogg_file = models.FileField(null=True, upload_to='archive/final', editable=False) - publishing_tags = JSONField(null=True, editable=False) + ogg_published_tags = JSONField(null=True, editable=False) + ogg_published = models.DateTimeField(null=True, editable=False) - publish_wait = models.DateTimeField(null=True, editable=False) # somebody hit "publish" - publishing = models.BooleanField(default=False, editable=False) - published = models.DateTimeField(null=True, editable=False) class Meta: verbose_name = _("audiobook") @@ -50,8 +62,35 @@ class Audiobook(models.Model): def __unicode__(self): return self.title + def published(self): + return self.mp3_published and self.ogg_published + def new_publish_tags(self): + title = self.title + if self.translator: + title += u' (tłum. %s)' % self.translator + + copyright = u"%s %s. Licensed to the public under %s verify at %s" % ( + self.date, ORGANIZATION, LICENSE, self.url) + + comment = u"Audiobook nagrany w ramach projektu %s%s.\n%s" % ( + self.project.name, + u" finansowanego przez %s" % self.project.sponsors if self.project.sponsors else "", + ADVERT) + return { - 'title': self.title, - 'copyright': 'Fundacja Nowoczesna Polska', + 'album': PROJECT, + 'albumartist': ORGANIZATION, + 'artist': self.artist, + 'comment': comment, + 'conductor': self.conductor, + 'contact': self.url, + 'copyright': copyright, + 'date': self.date, + 'genre': u'Speech', + 'language': u'pol', + 'license': LICENSE, + 'organization': ORGANIZATION, + 'title': title, + 'flac_sha1': self.source_sha1, } diff --git a/apps/archive/settings.py b/apps/archive/settings.py index a0db355..9c05b70 100755 --- a/apps/archive/settings.py +++ b/apps/archive/settings.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + import os.path from django.conf import settings @@ -30,3 +32,69 @@ try: except AttributeError: FINAL_PATH = os.path.abspath(os.path.join(settings.MEDIA_ROOT, "archive/final")) + + +# here the app keeps temporary build files +try: + BUILD_PATH = settings.ARCHIVE_BUILD_PATH +except AttributeError: + BUILD_PATH = os.path.abspath(os.path.join(settings.MEDIA_ROOT, + "archive/build")) + +# upload conf +try: + UPLOAD_HOST = settings.ARCHIVE_UPLOAD_HOST +except AttributeError: + UPLOAD_HOST = 'wolnelektury.pl' + +try: + UPLOAD_USER = settings.ARCHIVE_UPLOAD_USER +except AttributeError: + UPLOAD_USER = 'username' + +try: + UPLOAD_PATH = settings.ARCHIVE_UPLOAD_PATH +except AttributeError: + UPLOAD_PATH = '' + +try: + UPLOAD_CMD = settings.ARCHIVE_UPLOAD_CMD +except AttributeError: + UPLOAD_CMD = '/path/to/manage.py savemedia' + +try: + UPLOAD_SUDO = settings.ARCHIVE_UPLOAD_SUDO +except AttributeError: + UPLOAD_SUDO = None + + +try: + PROJECT = settings.ARCHIVE_PROJECT +except AttributeError: + PROJECT = u'Wolne Lektury' + +try: + LICENSE = settings.ARCHIVE_LICENSE +except AttributeError: + LICENSE = u'http://creativecommons.org/licenses/by-sa/3.0/deed.pl' + +try: + ORGANIZATION = settings.ARCHIVE_ORGANIZATION +except AttributeError: + ORGANIZATION = u'Fundacja Nowoczesna Polska' + +try: + ADVERT = settings.ARCHIVE_ADVERT +except AttributeError: + ADVERT = u""" +Przekaż 1% podatku na rozwój Wolnych Lektur: +Fundacja Nowoczesna Polska +KRS 0000070056 +http://nowoczesnapolska.org.pl/wesprzyj_nas/ +""" + +try: + COVER_IMAGE = settings.ARCHIVE_COVER_IMAGE +except AttributeError: + COVER_IMAGE = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'cover.png') + diff --git a/apps/archive/static/style.css b/apps/archive/static/style.css index 0d30ad6..2e7dbb3 100755 --- a/apps/archive/static/style.css +++ b/apps/archive/static/style.css @@ -47,3 +47,13 @@ a { .errorlist li { color: red; } + + +.tags, .multiple_tags { + border-collapse: collapse; +} + +.tags td, .tags th, .multiple_tags td, .multiple_tags th { + border: 1px solid #ddd; + padding: 3px; +} diff --git a/apps/archive/tasks.py b/apps/archive/tasks.py new file mode 100755 index 0000000..b90b15e --- /dev/null +++ b/apps/archive/tasks.py @@ -0,0 +1,178 @@ +from datetime import datetime +import errno +import mimetypes +import os +import os.path +import pipes +import subprocess +from tempfile import NamedTemporaryFile +from time import sleep + +#from celery.decorators import task +from celery.task import Task +from fabric import api +from mutagen import File +from mutagen import id3 + +import mutagen + +from archive.constants import status +from archive.models import Audiobook +from archive.settings import (BUILD_PATH, COVER_IMAGE, + UPLOAD_HOST, UPLOAD_USER, UPLOAD_PATH, UPLOAD_CMD, UPLOAD_SUDO) +from archive.utils import ExistingFile + +api.env.host_string = UPLOAD_HOST +api.env.user = UPLOAD_USER + +class AudioFormatTask(Task): + abstract = True + + @classmethod + def set_status(cls, audiobook, status): + setattr(audiobook, '%s_status' % cls.ext, status) + audiobook.save() + + @staticmethod + def encode(in_path, out_path): + pass + + @classmethod + def set_tags(cls, audiobook, file_name): + audio = File(file_name) + for k, v in getattr(audiobook, "%s_tags" % cls.ext)['tags'].items(): + audio[k] = v + audio.save() + + @classmethod + def save(cls, audiobook, file_name): + getattr(audiobook, "%s_file" % cls.ext).save( + "%d.%s" % (audiobook.pk, cls.ext), + ExistingFile(file_name) + ) + + @classmethod + def published(cls, audiobook): + setattr(audiobook, "%s_published_tags" % cls.ext, + getattr(audiobook, "%s_tags" % cls.ext)) + setattr(audiobook, "%s_tags" % cls.ext, None) + setattr(audiobook, "%s_published" % cls.ext, datetime.now()) + cls.set_status(audiobook, None) + + @classmethod + def put(cls, audiobook): + tags = getattr(audiobook, "%s_tags" % cls.ext) + prefix, slug = tags['url'].rstrip('/').rsplit('/', 1) + name = tags['name'] + path = getattr(audiobook, "%s_file" % cls.ext).path + api.put(path, UPLOAD_PATH) + command = UPLOAD_CMD + (u' %s %s %s > output.txt' % ( + pipes.quote(os.path.join(UPLOAD_PATH, os.path.basename(path))), + pipes.quote(slug), + pipes.quote(name) + )).encode('utf-8') + print command + if UPLOAD_SUDO: + api.sudo(command, user=UPLOAD_SUDO, shell=False) + else: + api.run(command) + + def run(self, aid): + audiobook = Audiobook.objects.get(id=aid) + self.set_status(audiobook, status.ENCODING) + + try: + os.makedirs(BUILD_PATH) + except OSError as e: + if e.errno == errno.EEXIST: + pass + else: + raise + + out_file = NamedTemporaryFile(delete=False, prefix='audiobook-', suffix='.%s' % self.ext, dir=BUILD_PATH) + out_file.close() + self.encode(audiobook.source_file.path, out_file.name) + self.set_status(audiobook, status.TAGGING) + self.set_tags(audiobook, out_file.name) + self.save(audiobook, out_file.name) + self.set_status(audiobook, status.SENDING) + + #self.put(audiobook) + + self.published(audiobook) + audiobook.save() + + +class Mp3Task(AudioFormatTask): + ext = 'mp3' + + # these shouldn't be staticmethods + def id3_text(tag, text): + return tag(encoding=1, text=text) + def id3_url(tag, text): + return tag(url=text) + def id3_comment(tag, text, lang=u'pol'): + return tag(encoding=1, lang=lang, desc=u'', text=text) + def id3_sha1(tag, text, what=u''): + return tag(owner='http://wolnelektury.pl?%s' % what, data=text) + + TAG_MAP = { + 'album': (id3_text, id3.TALB), + 'albumartist': (id3_text, id3.TPE2), + 'artist': (id3_text, id3.TPE1), + 'conductor': (id3_text, id3.TPE3), + 'copyright': (id3_text, id3.TCOP), + 'date': (id3_text, id3.TDRC), + 'genre': (id3_text, id3.TCON), + 'language': (id3_text, id3.TLAN), + 'organization': (id3_text, id3.TPUB), + 'title': (id3_text, id3.TIT2), + 'comment': (id3_comment, id3.COMM, 'pol'), + 'contact': (id3_url, id3.WOAF), + 'license': (id3_url, id3.WCOP), + 'flac_sha1': (id3_sha1, id3.PRIV, 'flac_sha1'), + } + + @staticmethod + def encode(in_path, out_path): + # 44.1kHz 64kbps mono MP3 + subprocess.check_call(['ffmpeg', + '-i', in_path, + '-ar', '44100', + '-ab', '64k', + '-ac', '1', + '-y', + out_path + ]) + + @classmethod + def set_tags(cls, audiobook, file_name): + audio = id3.ID3(file_name) + for k, v in audiobook.mp3_tags['tags'].items(): + factory_tuple = cls.TAG_MAP[k] + factory, tagtype = factory_tuple[:2] + audio.add(factory(tagtype, v, *factory_tuple[2:])) + + if COVER_IMAGE: + mime = mimetypes.guess_type(COVER_IMAGE) + f = open(COVER_IMAGE) + audio.add(id3.APIC(encoding=0, mime=mime, type=3, desc=u'', data=f.read())) + f.close() + + audio.save() + + +class OggTask(AudioFormatTask): + ext = 'ogg' + + @staticmethod + def encode(in_path, out_path): + # 44.1kHz 64kbps mono Ogg Vorbis + subprocess.check_call(['oggenc', + in_path, + '--discard-comments', + '--resample', '44100', + '--downmix', + '-b', '64', + '-o', out_path + ]) diff --git a/apps/archive/templates/archive/base.html b/apps/archive/templates/archive/base.html index 807227b..fbc5a18 100644 --- a/apps/archive/templates/archive/base.html +++ b/apps/archive/templates/archive/base.html @@ -1,28 +1,11 @@ +{% extends "base.html" %} {% load i18n %} - - - - - - {% trans "Audiobook repository" %} - - -
+{% block repo-zones-nav %} {% trans "New" %} {% trans "Unpublished" %} {% trans "Published" %} {% trans "Archive" %} + {% trans "Logout" %}
-
- -
-{% block messages %}{% endblock %} -
- -
-{% block content %}{% endblock %} -
- - - +{% endblock %} diff --git a/apps/archive/templates/archive/file_managed.html b/apps/archive/templates/archive/file_managed.html index 897aae2..c67edea 100755 --- a/apps/archive/templates/archive/file_managed.html +++ b/apps/archive/templates/archive/file_managed.html @@ -1,37 +1,46 @@ {% extends "archive/base.html" %} {% load i18n %} +{% load tags %} {% block content %}

{% trans "Publishing" %}

-{% if audiobook.publish_wait %} -

{% trans "Audiobook marked for publishing with tags:" %}

+{% if audiobook.mp3_status or audiobook.ogg_status %} - - {% for t, v in audiobook.publishing_tags.items %} - - {% endfor %} -
{{ t }}{{ v }}
+

{% trans "Publishing pending" %}

- {% if audiobook.publishing %} -

{% trans "Publishing already in progress." %}

- {% else %} -
- {% csrf_token %} - -
- {% endif %} -{% else %} - {% if audiobook.published %} - Here be currently published version, for comparison. - {% endif %} +
+ {% csrf_token %} + +
+ + +{% if audiobook.mp3_status %} +
+

MP3

+ + {% tags_table audiobook.mp3_tags.tags %} + +

Status: {{ audiobook.get_mp3_status_display }}

+{% endif %} + +{% if audiobook.ogg_status %} +
+

Ogg Vorbis

+ + {% tags_table audiobook.ogg_tags.tags %} +

Status: {{ audiobook.get_ogg_status_display }}

+{% endif %} + + + + +{% else %}
- - {% for k, v in audiobook.new_publish_tags.items %} - - {% endfor %} +
{{ k }}{{ v }}
+ {% tags_table audiobook.new_publish_tags 0 %} {% csrf_token %} @@ -39,6 +48,22 @@ {% endif %} +
+{% if audiobook.mp3_published %} +

Published MP3

+ {{ audiobook.mp3_published }} + {% tags_table audiobook.mp3_published_tags.tags %} +{% else %}

MP3 file hasn't been published yet.

+{% endif %} + +
+{% if audiobook.ogg_published %} +

Published Ogg Vorbis

+ {{ audiobook.ogg_published }} + {% tags_table audiobook.ogg_published_tags.tags %} +{% else %}Ogg Vorbis file hasn't been published yet. +{% endif %} + @@ -49,16 +74,9 @@

{% trans "Update tags" %}

+Last modified: {{ audiobook.modified }} -
- {% for t, v in tags.items %} - - {% endfor %} -
{{t}} - {% for x in v %} - {{x}}
- {% endfor %} -
+{% multiple_tags_table tags %} diff --git a/apps/archive/templates/archive/list_published.html b/apps/archive/templates/archive/list_published.html index c51cace..49544b4 100755 --- a/apps/archive/templates/archive/list_published.html +++ b/apps/archive/templates/archive/list_published.html @@ -14,7 +14,7 @@ {% block file-list %} {% for file in objects %}
  • - {{ file }} + {{ file }}
    • {% endfor %} {% endblock %} diff --git a/apps/archive/templates/archive/tags/multiple_tags_table.html b/apps/archive/templates/archive/tags/multiple_tags_table.html index 4a39808..4609278 100755 --- a/apps/archive/templates/archive/tags/multiple_tags_table.html +++ b/apps/archive/templates/archive/tags/multiple_tags_table.html @@ -1,9 +1,9 @@ - +{% if table %}
    {% endif %} {% for t, v in tags.items %} - {% endfor %} -
    {{t}} +
    {{ t }} {% for x in v %} -
    {{x}}
    +
    {{ x|linebreaksbr }}
    {% endfor %}
    +{% if table %}{% endif %} diff --git a/apps/archive/templates/archive/tags/tags_table.html b/apps/archive/templates/archive/tags/tags_table.html new file mode 100755 index 0000000..70b9d48 --- /dev/null +++ b/apps/archive/templates/archive/tags/tags_table.html @@ -0,0 +1,7 @@ +{% if table %}{% endif %} + {% for t, v in tags.items %} + + {% endfor %} +{% if table %}
    {{ t }} + {{ v|linebreaksbr }} +
    {% endif %} diff --git a/apps/archive/templates/base.html b/apps/archive/templates/base.html new file mode 100755 index 0000000..c6b7ee6 --- /dev/null +++ b/apps/archive/templates/base.html @@ -0,0 +1,24 @@ +{% load i18n %} + + + + + + {% trans "Audiobook repository" %} + + + +
    + {% block repo-zones-nav %} {% endblock %} +
    + +
    +{% block messages %}{% endblock %} +
    + +
    +{% block content %}{% endblock %} +
    + + + diff --git a/apps/archive/templates/registration/login.html b/apps/archive/templates/registration/login.html new file mode 100755 index 0000000..115cdf1 --- /dev/null +++ b/apps/archive/templates/registration/login.html @@ -0,0 +1,12 @@ +{% extends "base.html" %} +{% load i18n %} + +{% block content %} + +
    +{% csrf_token %} +{{ form.as_p }} +

    +
    + +{% endblock %} diff --git a/apps/archive/templatetags/tags.py b/apps/archive/templatetags/tags.py index 68a6a9b..25389ef 100755 --- a/apps/archive/templatetags/tags.py +++ b/apps/archive/templatetags/tags.py @@ -2,18 +2,11 @@ from django import template register = template.Library() -@register.simple_tag -def multiple_tags_table(tags): - return template.loader.render_to_string( - "archive/tags/multiple_tags_table.html", - {"tags": tags} - ) +@register.inclusion_tag('archive/tags/multiple_tags_table.html') +def multiple_tags_table(tags, table=True): + return locals() -#@register.simple_tag -#def multiple_tags_table(tags): -# return template.loader.render_to_string( -# "archive/tags/multiple_tags_table.html", -# {"tags": tags} -# ) - +@register.inclusion_tag('archive/tags/tags_table.html') +def tags_table(tags, table=True): + return locals() diff --git a/apps/archive/utils.py b/apps/archive/utils.py index 4710c20..6e0a8d1 100755 --- a/apps/archive/utils.py +++ b/apps/archive/utils.py @@ -1,5 +1,7 @@ +from hashlib import sha1 from django.core.files.uploadedfile import UploadedFile + class ExistingFile(UploadedFile): def __init__(self, path, *args, **kwargs): @@ -11,3 +13,10 @@ class ExistingFile(UploadedFile): def close(self): pass + + +def sha1_file(f): + sha = sha1() + for piece in iter(lambda: f.read(1024*1024), ''): + sha.update(piece) + return sha.hexdigest() diff --git a/apps/archive/views.py b/apps/archive/views.py index 3ee87cb..575f5d9 100644 --- a/apps/archive/views.py +++ b/apps/archive/views.py @@ -5,17 +5,23 @@ import os import os.path from archive import settings +from django.contrib.auth import logout +from django.contrib.auth.decorators import login_required from django.core.urlresolvers import reverse +from django.db.models import Q from django.http import Http404 from django.shortcuts import render, redirect, get_object_or_404 from django.views.decorators.http import require_POST import mutagen +from archive.constants import status from archive import models from archive.forms import AudiobookForm +from archive import tasks +@login_required def list_new(request): division = 'new' @@ -24,6 +30,7 @@ def list_new(request): return render(request, "archive/list_new.html", locals()) +@login_required def file_new(request, filename): division = 'new' @@ -60,6 +67,7 @@ def file_new(request, filename): @require_POST +@login_required def move_to_archive(request, filename): """ move a new file to the unmanaged files dir """ @@ -84,6 +92,7 @@ def move_to_archive(request, filename): @require_POST +@login_required def move_to_new(request, filename): """ move a unmanaged file to new files dir """ @@ -106,37 +115,70 @@ def move_to_new(request, filename): return redirect(list_unmanaged) + @require_POST -def publish(request, id): +@login_required +def publish(request, aid): """ mark file for publishing """ - audiobook = get_object_or_404(models.Audiobook, id=id) - audiobook.publish_wait = datetime.now() - audiobook.publishing_tags = audiobook.new_publish_tags() + audiobook = get_object_or_404(models.Audiobook, id=aid) + tags = { + 'name': audiobook.title, + 'url': audiobook.url, + 'tags': audiobook.new_publish_tags(), + } + audiobook.mp3_tags = tags + audiobook.ogg_tags = tags + audiobook.mp3_status = audiobook.ogg_status = status.WAITING audiobook.save() - return redirect(file_managed, id) + # isn't there a race here? + audiobook.mp3_task = tasks.Mp3Task.delay(aid).task_id + audiobook.ogg_task = tasks.OggTask.delay(aid).task_id + audiobook.save() + + return redirect(file_managed, aid) + @require_POST -def cancel_publishing(request, id): +@login_required +def cancel_publishing(request, aid): """ cancel scheduled publishing """ - audiobook = get_object_or_404(models.Audiobook, id=id) - if not audiobook.publishing: - audiobook.publish_wait = None - audiobook.publishing_tags = None - audiobook.save() - return redirect(file_managed, id) + audiobook = get_object_or_404(models.Audiobook, id=aid) + # TODO: cancel tasks + audiobook.mp3_status = None + audiobook.ogg_status = None + audiobook.save() + return redirect(file_managed, aid) +@login_required def list_unpublished(request): division = 'unpublished' - objects = models.Audiobook.objects.filter(published=None) + objects = models.Audiobook.objects.filter(Q(mp3_published=None) | Q(ogg_published=None)) return render(request, "archive/list_unpublished.html", locals()) +@login_required +def list_published(request): + division = 'published' + + objects = models.Audiobook.objects.exclude(Q(mp3_published=None) | Q(ogg_published=None)) + return render(request, "archive/list_published.html", locals()) + +@login_required def file_managed(request, id): audiobook = get_object_or_404(models.Audiobook, id=id) - division = 'published' if audiobook.published else 'unpublished' + + if request.POST: + form = AudiobookForm(request.POST, instance=audiobook) + if form.is_valid(): + try: + form.save() + except IOError: + raise Http404 + + division = 'published' if audiobook.published() else 'unpublished' # for tags update tags = mutagen.File(audiobook.source_file.path) @@ -145,16 +187,7 @@ def file_managed(request, id): return render(request, "archive/file_managed.html", locals()) - -def list_published(request): - division = 'published' - - objects = models.Audiobook.objects.exclude(published=None) - return render(request, "archive/list_published.html", locals()) - - - - +@login_required def list_unmanaged(request): division = 'unmanaged' @@ -162,6 +195,7 @@ def list_unmanaged(request): return render(request, "archive/list_unmanaged.html", locals()) +@login_required def file_unmanaged(request, filename): division = 'unmanaged' @@ -169,3 +203,8 @@ def file_unmanaged(request, filename): err_exists = request.GET.get('exists') return render(request, "archive/file_unmanaged.html", locals()) + +@login_required +def logout_view(request): + logout(request) + return redirect(list_new) diff --git a/audiobooks/settings.py b/audiobooks/settings.py index 9f0731a..dc55333 100644 --- a/audiobooks/settings.py +++ b/audiobooks/settings.py @@ -58,7 +58,7 @@ MEDIA_ROOT = os.path.join(PROJECT_ROOT, '../media') # URL that handles the media served from MEDIA_ROOT. Make sure to use a # trailing slash. # Examples: "http://media.lawrence.com/media/", "http://example.com/media/" -MEDIA_URL = '' +MEDIA_URL = '/media/' # Absolute path to the directory static files should be collected to. # Don't put anything in this directory yourself; store your static files @@ -128,6 +128,9 @@ INSTALLED_APPS = ( # Uncomment the next line to enable admin documentation: # 'django.contrib.admindocs', + 'djcelery', + 'djkombu', + 'south', 'archive', ) @@ -157,6 +160,18 @@ LOGGING = { + +import djcelery +djcelery.setup_loader() + +BROKER_BACKEND = "djkombu.transport.DatabaseTransport" +BROKER_HOST = "localhost" +BROKER_PORT = 5672 +BROKER_USER = "guest" +BROKER_PASSWORD = "guest" +BROKER_VHOST = "/" + + try: from localsettings import * except: diff --git a/audiobooks/urls.py b/audiobooks/urls.py index 2cb91e0..76efbb3 100644 --- a/audiobooks/urls.py +++ b/audiobooks/urls.py @@ -1,3 +1,4 @@ +from django.conf import settings from django.conf.urls.defaults import patterns, include, url # Uncomment the next two lines to enable the admin: @@ -9,4 +10,11 @@ urlpatterns = patterns('', url(r'^archive/', include('archive.urls')), url(r'^admin/', include(admin.site.urls)), + url(r'^accounts/login/', 'django.contrib.auth.views.login'), + url(r'^accounts/logout/', 'archive.views.logout_view', name='logout'), +) + +if settings.DEBUG: + urlpatterns += patterns('', + (r'^media/(?P.*)$', 'django.views.static.serve', {'document_root': settings.MEDIA_ROOT, 'show_indexes':True}), ) diff --git a/requirements.txt b/requirements.txt index 7b29e99..899e65b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,9 @@ django>=1.3 django-jsonfield South>=0.7 +django-celery +django-kombu + +paramiko>=1.7.7.1 # fabric dependency with http://code.fabfile.org/issues/show/214 fixed +fabric mutagen