add project and funding tags to audiobooks,

overwrite output files on save: they're id-prefixed and big,
error handling: e-mails from celery, audibook state reset, save files in model only after sending them,

diff --git a/apps/archive/models.py b/apps/archive/models.py
index f9e86f6..afa90b5 100644 (file)
--- a/apps/archive/models.py
+++ b/apps/archive/models.py
@@ -5,6 +5,7 @@ from jsonfield.fields import JSONField
 from django.utils.translation import ugettext_lazy as _
 from archive.constants import status
 from archive.settings import FILES_PATH, ADVERT, LICENSE, ORGANIZATION, PROJECT
+from archive.utils import OverwriteStorage
 
 # Create your models here.
 
@@ -43,14 +44,14 @@ class Audiobook(models.Model):
     mp3_status = models.SmallIntegerField(null=True, editable=False, choices=status.choices)
     mp3_task = models.CharField(max_length=64, null=True, editable=False)
     mp3_tags = JSONField(null=True, editable=False)
-    mp3_file = models.FileField(null=True, upload_to='archive/final', editable=False)
+    mp3_file = models.FileField(null=True, upload_to='archive/final', storage=OverwriteStorage(), editable=False)
     mp3_published_tags = JSONField(null=True, editable=False)
     mp3_published = models.DateTimeField(null=True, editable=False)
 
     ogg_status = models.SmallIntegerField(null=True, editable=False, choices=status.choices)
     ogg_task = models.CharField(max_length=64, null=True, editable=False)
     ogg_tags = JSONField(null=True, editable=False)
-    ogg_file = models.FileField(null=True, upload_to='archive/final', editable=False)
+    ogg_file = models.FileField(null=True, upload_to='archive/final', storage=OverwriteStorage(), editable=False)
     ogg_published_tags = JSONField(null=True, editable=False)
     ogg_published = models.DateTimeField(null=True, editable=False)
 
@@ -94,4 +95,6 @@ class Audiobook(models.Model):
             'organization': ORGANIZATION,
             'title': title,
             'flac_sha1': self.source_sha1,
+            'project': self.project.name,
+            'funded_by': self.project.sponsors,
         }

diff --git a/apps/archive/tasks.py b/apps/archive/tasks.py
index 37169d2..e8ce4fb 100755 (executable)
--- a/apps/archive/tasks.py
+++ b/apps/archive/tasks.py
@@ -10,6 +10,7 @@ from time import sleep
 
 #from celery.decorators import task
 from celery.task import Task
+from django.db.models import F
 from fabric import api
 from fabric.network import disconnect_all
 from mutagen import File
@@ -30,14 +31,17 @@ api.env.password = UPLOAD_PASSWORD
 class AudioFormatTask(Task):
     abstract = True
 
+    class RemoteOperationError(BaseException):
+        pass
+
     @classmethod
-    def set_status(cls, audiobook, status):
-        Audiobook.objects.filter(pk=audiobook.pk).update(
+    def set_status(cls, aid, status):
+        Audiobook.objects.filter(pk=aid).update(
             **{'%s_status' % cls.ext: status})
 
     @staticmethod
     def encode(in_path, out_path):
-        pass
+        raise NotImplemented
 
     @classmethod
     def set_tags(cls, audiobook, file_name):
@@ -58,37 +62,39 @@ class AudioFormatTask(Task):
             **{field: getattr(audiobook, field)})
 
     @classmethod
-    def published(cls, audiobook):
+    def published(cls, aid):
         kwargs = {
-            "%s_published_tags" % cls.ext: 
-                    getattr(audiobook, "%s_tags" % cls.ext),
+            "%s_published_tags" % cls.ext: F("%s_tags" % cls.ext),
             "%s_tags" % cls.ext: None,
             "%s_published" % cls.ext: datetime.now(),
             '%s_status' % cls.ext: None,
         }
-        Audiobook.objects.filter(pk=audiobook.pk).update(**kwargs)
+        Audiobook.objects.filter(pk=aid).update(**kwargs)
 
     @classmethod
-    def put(cls, audiobook):
+    def put(cls, audiobook, path):
         tags = getattr(audiobook, "%s_tags" % cls.ext)
         prefix, slug = tags['url'].rstrip('/').rsplit('/', 1)
         name = tags['name']
-        path = getattr(audiobook, "%s_file" % cls.ext).path
-        api.put(path, UPLOAD_PATH)
         command = UPLOAD_CMD + (u' %s %s %s > output.txt' % (
             pipes.quote(os.path.join(UPLOAD_PATH, os.path.basename(path))),
             pipes.quote(slug),
             pipes.quote(name)
             )).encode('utf-8')
-        if UPLOAD_SUDO:
-            api.sudo(command, user=UPLOAD_SUDO, shell=False)
-        else:
-            api.run(command)
-        disconnect_all()
+        try:
+            api.put(path, UPLOAD_PATH)
+            if UPLOAD_SUDO:
+                api.sudo(command, user=UPLOAD_SUDO, shell=False)
+            else:
+                api.run(command)
+            disconnect_all()
+        except SystemExit, e:
+            raise cls.RemoteOperationError
 
     def run(self, aid):
+        aid = int(aid)
         audiobook = Audiobook.objects.get(id=aid)
-        self.set_status(audiobook, status.ENCODING)
+        self.set_status(aid, status.ENCODING)
 
         try:
             os.makedirs(BUILD_PATH)
@@ -98,17 +104,21 @@ class AudioFormatTask(Task):
             else:
                 raise
 
-        out_file = NamedTemporaryFile(delete=False, prefix='audiobook-', suffix='.%s' % self.ext, dir=BUILD_PATH)
+        out_file = NamedTemporaryFile(delete=False, prefix='%d-' % aid, suffix='.%s' % self.ext, dir=BUILD_PATH)
         out_file.close()
         self.encode(audiobook.source_file.path, out_file.name)
-        self.set_status(audiobook, status.TAGGING)
+        self.set_status(aid, status.TAGGING)
         self.set_tags(audiobook, out_file.name)
-        self.save(audiobook, out_file.name)
-        self.set_status(audiobook, status.SENDING)
+        self.set_status(aid, status.SENDING)
 
-        self.put(audiobook)
+        self.put(audiobook, out_file.name)
+
+        self.save(audiobook, out_file.name)
+        self.published(aid)
 
-        self.published(audiobook)
+    def on_failure(self, exc, task_id, args, kwargs, einfo):
+        aid = (args[0], kwargs.get('aid'))[0]
+        self.set_status(aid, None)
 
 
 class Mp3Task(AudioFormatTask):
@@ -121,8 +131,8 @@ class Mp3Task(AudioFormatTask):
         return tag(url=text)
     def id3_comment(tag, text, lang=u'pol'):
         return tag(encoding=1, lang=lang, desc=u'', text=text)
-    def id3_sha1(tag, text, what=u''):
-        return tag(owner='http://wolnelektury.pl?%s' % what, data=text)
+    def id3_priv(tag, text, what=u''):
+        return tag(owner='wolnelektury.pl?%s' % what, data=text.encode('utf-8'))
 
     TAG_MAP = {
         'album': (id3_text, id3.TALB),
@@ -138,7 +148,9 @@ class Mp3Task(AudioFormatTask):
         'comment': (id3_comment, id3.COMM, 'pol'),
         'contact': (id3_url, id3.WOAF),
         'license': (id3_url, id3.WCOP),
-        'flac_sha1': (id3_sha1, id3.PRIV, 'flac_sha1'),
+        'flac_sha1': (id3_priv, id3.PRIV, 'flac_sha1'),
+        'project': (id3_priv, id3.PRIV, 'project'),
+        'funded_by': (id3_priv, id3.PRIV, 'funded_by'),
     }
 
     @staticmethod

diff --git a/apps/archive/utils.py b/apps/archive/utils.py
index 6e0a8d1..c69fe1c 100755 (executable)
--- a/apps/archive/utils.py
+++ b/apps/archive/utils.py
@@ -1,4 +1,5 @@
 from hashlib import sha1
+from django.core.files.storage import FileSystemStorage
 from django.core.files.uploadedfile import UploadedFile
 
 
@@ -15,6 +16,17 @@ class ExistingFile(UploadedFile):
         pass
 
 
+class OverwriteStorage(FileSystemStorage):
+
+    def _save(self, name, content):
+        if self.exists(name):
+            self.delete(name)
+        return super(OverwriteStorage, self)._save(name, content)
+
+    def get_available_name(self, name):
+        return name
+
+
 def sha1_file(f):
     sha = sha1()
     for piece in iter(lambda: f.read(1024*1024), ''):

diff --git a/audiobooks/settings.py b/audiobooks/settings.py
index dc55333..1df2c37 100644 (file)
--- a/audiobooks/settings.py
+++ b/audiobooks/settings.py
@@ -159,11 +159,15 @@ LOGGING = {
 }
 
 
-
+EMAIL_SUBJECT_PREFIX = '[Audio] '
+SERVER_EMAIL = 'no-reply@audio.wolnelektury.pl'
+EMAIL_HOST = 'localhost'
+EMAIL_PORT = 25
 
 import djcelery
 djcelery.setup_loader()
 
+CELERY_SEND_TASK_ERROR_EMAILS = True
 BROKER_BACKEND = "djkombu.transport.DatabaseTransport"
 BROKER_HOST = "localhost"
 BROKER_PORT = 5672