From: Radek Czajka Date: Fri, 12 Aug 2011 11:50:36 +0000 (+0200) Subject: add project and funding tags to audiobooks, X-Git-Url: https://git.mdrn.pl/audio.git/commitdiff_plain/c64903445bdf2294335b43d464d8147f2fdf41c4 add project and funding tags to audiobooks, overwrite output files on save: they're id-prefixed and big, error handling: e-mails from celery, audibook state reset, save files in model only after sending them, --- diff --git a/apps/archive/models.py b/apps/archive/models.py index f9e86f6..afa90b5 100644 --- a/apps/archive/models.py +++ b/apps/archive/models.py @@ -5,6 +5,7 @@ from jsonfield.fields import JSONField from django.utils.translation import ugettext_lazy as _ from archive.constants import status from archive.settings import FILES_PATH, ADVERT, LICENSE, ORGANIZATION, PROJECT +from archive.utils import OverwriteStorage # Create your models here. @@ -43,14 +44,14 @@ class Audiobook(models.Model): mp3_status = models.SmallIntegerField(null=True, editable=False, choices=status.choices) mp3_task = models.CharField(max_length=64, null=True, editable=False) mp3_tags = JSONField(null=True, editable=False) - mp3_file = models.FileField(null=True, upload_to='archive/final', editable=False) + mp3_file = models.FileField(null=True, upload_to='archive/final', storage=OverwriteStorage(), editable=False) mp3_published_tags = JSONField(null=True, editable=False) mp3_published = models.DateTimeField(null=True, editable=False) ogg_status = models.SmallIntegerField(null=True, editable=False, choices=status.choices) ogg_task = models.CharField(max_length=64, null=True, editable=False) ogg_tags = JSONField(null=True, editable=False) - ogg_file = models.FileField(null=True, upload_to='archive/final', editable=False) + ogg_file = models.FileField(null=True, upload_to='archive/final', storage=OverwriteStorage(), editable=False) ogg_published_tags = JSONField(null=True, editable=False) ogg_published = models.DateTimeField(null=True, editable=False) @@ -94,4 +95,6 @@ class Audiobook(models.Model): 'organization': ORGANIZATION, 'title': title, 'flac_sha1': self.source_sha1, + 'project': self.project.name, + 'funded_by': self.project.sponsors, } diff --git a/apps/archive/tasks.py b/apps/archive/tasks.py index 37169d2..e8ce4fb 100755 --- a/apps/archive/tasks.py +++ b/apps/archive/tasks.py @@ -10,6 +10,7 @@ from time import sleep #from celery.decorators import task from celery.task import Task +from django.db.models import F from fabric import api from fabric.network import disconnect_all from mutagen import File @@ -30,14 +31,17 @@ api.env.password = UPLOAD_PASSWORD class AudioFormatTask(Task): abstract = True + class RemoteOperationError(BaseException): + pass + @classmethod - def set_status(cls, audiobook, status): - Audiobook.objects.filter(pk=audiobook.pk).update( + def set_status(cls, aid, status): + Audiobook.objects.filter(pk=aid).update( **{'%s_status' % cls.ext: status}) @staticmethod def encode(in_path, out_path): - pass + raise NotImplemented @classmethod def set_tags(cls, audiobook, file_name): @@ -58,37 +62,39 @@ class AudioFormatTask(Task): **{field: getattr(audiobook, field)}) @classmethod - def published(cls, audiobook): + def published(cls, aid): kwargs = { - "%s_published_tags" % cls.ext: - getattr(audiobook, "%s_tags" % cls.ext), + "%s_published_tags" % cls.ext: F("%s_tags" % cls.ext), "%s_tags" % cls.ext: None, "%s_published" % cls.ext: datetime.now(), '%s_status' % cls.ext: None, } - Audiobook.objects.filter(pk=audiobook.pk).update(**kwargs) + Audiobook.objects.filter(pk=aid).update(**kwargs) @classmethod - def put(cls, audiobook): + def put(cls, audiobook, path): tags = getattr(audiobook, "%s_tags" % cls.ext) prefix, slug = tags['url'].rstrip('/').rsplit('/', 1) name = tags['name'] - path = getattr(audiobook, "%s_file" % cls.ext).path - api.put(path, UPLOAD_PATH) command = UPLOAD_CMD + (u' %s %s %s > output.txt' % ( pipes.quote(os.path.join(UPLOAD_PATH, os.path.basename(path))), pipes.quote(slug), pipes.quote(name) )).encode('utf-8') - if UPLOAD_SUDO: - api.sudo(command, user=UPLOAD_SUDO, shell=False) - else: - api.run(command) - disconnect_all() + try: + api.put(path, UPLOAD_PATH) + if UPLOAD_SUDO: + api.sudo(command, user=UPLOAD_SUDO, shell=False) + else: + api.run(command) + disconnect_all() + except SystemExit, e: + raise cls.RemoteOperationError def run(self, aid): + aid = int(aid) audiobook = Audiobook.objects.get(id=aid) - self.set_status(audiobook, status.ENCODING) + self.set_status(aid, status.ENCODING) try: os.makedirs(BUILD_PATH) @@ -98,17 +104,21 @@ class AudioFormatTask(Task): else: raise - out_file = NamedTemporaryFile(delete=False, prefix='audiobook-', suffix='.%s' % self.ext, dir=BUILD_PATH) + out_file = NamedTemporaryFile(delete=False, prefix='%d-' % aid, suffix='.%s' % self.ext, dir=BUILD_PATH) out_file.close() self.encode(audiobook.source_file.path, out_file.name) - self.set_status(audiobook, status.TAGGING) + self.set_status(aid, status.TAGGING) self.set_tags(audiobook, out_file.name) - self.save(audiobook, out_file.name) - self.set_status(audiobook, status.SENDING) + self.set_status(aid, status.SENDING) - self.put(audiobook) + self.put(audiobook, out_file.name) + + self.save(audiobook, out_file.name) + self.published(aid) - self.published(audiobook) + def on_failure(self, exc, task_id, args, kwargs, einfo): + aid = (args[0], kwargs.get('aid'))[0] + self.set_status(aid, None) class Mp3Task(AudioFormatTask): @@ -121,8 +131,8 @@ class Mp3Task(AudioFormatTask): return tag(url=text) def id3_comment(tag, text, lang=u'pol'): return tag(encoding=1, lang=lang, desc=u'', text=text) - def id3_sha1(tag, text, what=u''): - return tag(owner='http://wolnelektury.pl?%s' % what, data=text) + def id3_priv(tag, text, what=u''): + return tag(owner='wolnelektury.pl?%s' % what, data=text.encode('utf-8')) TAG_MAP = { 'album': (id3_text, id3.TALB), @@ -138,7 +148,9 @@ class Mp3Task(AudioFormatTask): 'comment': (id3_comment, id3.COMM, 'pol'), 'contact': (id3_url, id3.WOAF), 'license': (id3_url, id3.WCOP), - 'flac_sha1': (id3_sha1, id3.PRIV, 'flac_sha1'), + 'flac_sha1': (id3_priv, id3.PRIV, 'flac_sha1'), + 'project': (id3_priv, id3.PRIV, 'project'), + 'funded_by': (id3_priv, id3.PRIV, 'funded_by'), } @staticmethod diff --git a/apps/archive/utils.py b/apps/archive/utils.py index 6e0a8d1..c69fe1c 100755 --- a/apps/archive/utils.py +++ b/apps/archive/utils.py @@ -1,4 +1,5 @@ from hashlib import sha1 +from django.core.files.storage import FileSystemStorage from django.core.files.uploadedfile import UploadedFile @@ -15,6 +16,17 @@ class ExistingFile(UploadedFile): pass +class OverwriteStorage(FileSystemStorage): + + def _save(self, name, content): + if self.exists(name): + self.delete(name) + return super(OverwriteStorage, self)._save(name, content) + + def get_available_name(self, name): + return name + + def sha1_file(f): sha = sha1() for piece in iter(lambda: f.read(1024*1024), ''): diff --git a/audiobooks/settings.py b/audiobooks/settings.py index dc55333..1df2c37 100644 --- a/audiobooks/settings.py +++ b/audiobooks/settings.py @@ -159,11 +159,15 @@ LOGGING = { } - +EMAIL_SUBJECT_PREFIX = '[Audio] ' +SERVER_EMAIL = 'no-reply@audio.wolnelektury.pl' +EMAIL_HOST = 'localhost' +EMAIL_PORT = 25 import djcelery djcelery.setup_loader() +CELERY_SEND_TASK_ERROR_EMAILS = True BROKER_BACKEND = "djkombu.transport.DatabaseTransport" BROKER_HOST = "localhost" BROKER_PORT = 5672