Use proper locking, and other minor fixes.

diff --git a/requirements.txt b/requirements.txt
index 12bcbf9..76f851d 100644 (file)
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,13 +2,12 @@
 
 django>=1.11,<2.0
 django-cas-ng==3.6.0
 
 django>=1.11,<2.0
 django-cas-ng==3.6.0

-django-bootstrap4==0.0.7
+django-bootstrap4==0.0.8
+django-pglocks==1.0.4

 
 

-celery[redis]==4.2
-kombu
-redis
-vine
+celery[redis]==4.4.0

 
 

+psycopg2-binary==2.8.4

 mutagen==1.42.0
 mutagen==1.42.0

-requests
-requests-oauthlib
+requests==2.23.0
+requests-oauthlib==1.3.0

diff --git a/src/archive/models.py b/src/archive/models.py
index 4a1d763..9e92fb8 100644 (file)
--- a/src/archive/models.py
+++ b/src/archive/models.py
@@ -1,9 +1,10 @@
+import json

 import os.path
 
 from django.db import models
 from time import sleep
 import os.path
 
 from django.db import models
 from time import sleep

-from django.utils.encoding import force_bytes

 from django.utils.translation import ugettext_lazy as _
 from django.utils.translation import ugettext_lazy as _

+from django_pglocks import advisory_lock

 from archive.constants import status
 from archive.settings import FILES_SAVE_PATH, ADVERT, LICENSE, ORGANIZATION, PROJECT
 from archive.utils import OverwriteStorage, sha1_file
 from archive.constants import status
 from archive.settings import FILES_SAVE_PATH, ADVERT, LICENSE, ORGANIZATION, PROJECT
 from archive.utils import OverwriteStorage, sha1_file


@@ -81,28 +82,13 @@ class Audiobook(models.Model):
         return self.mp3_published and self.ogg_published
 
     def get_source_sha1(self):
         return self.mp3_published and self.ogg_published
 
     def get_source_sha1(self):

-        source_sha1 = self.source_sha1
-        if self.pk:
-            source_sha1 = type(self).objects.get(pk=self.pk).source_sha1
-            while source_sha1 == 'wait':
-                sleep(10)
-        if not source_sha1:
-            self.source_sha1 = 'wait'
-            if self.pk:
-                type(self).objects.filter(pk=self.pk).update(source_sha1='wait')
-            try:
-                f = open(force_bytes(self.source_file.path))
-                source_sha1 = sha1_file(f)
-                self.source_sha1 = source_sha1
-                if self.pk:
-                    type(self).objects.filter(pk=self.pk).update(source_sha1=source_sha1)
-            except:
-                self.source_sha1 = ''
-                if self.pk:
-                    type(self).objects.filter(pk=self.pk).update(source_sha1='')
-                return None
-        return source_sha1
-
+        assert self.pk or self.source_sha1
+        if not self.source_sha1:
+            with advisory_lock(f'get_source_sha1_{self.pk}'):
+                with open(self.source_file.path, 'rb') as f:
+                    self.source_sha1 = sha1_file(f)
+                self.save(update_fields=['source_sha1'])
+        return self.source_sha1

 
     def new_publish_tags(self):
         title = self.title
 
     def new_publish_tags(self):
         title = self.title


@@ -131,11 +117,10 @@ class Audiobook(models.Model):
             'license': LICENSE,
             'organization': ORGANIZATION,
             'title': title,
             'license': LICENSE,
             'organization': ORGANIZATION,
             'title': title,

-            #'flac_sha1': self.get_source_sha1(),

             'project': self.project.name,
             'funded_by': self.project.sponsors,
         }
             'project': self.project.name,
             'funded_by': self.project.sponsors,
         }

-        if self.source_sha1 and self.source_sha1 != 'wait':
+        if self.source_sha1:

             tags['flac_sha1'] = self.source_sha1
         return tags
 
             tags['flac_sha1'] = self.source_sha1
         return tags
 

diff --git a/src/archive/tasks.py b/src/archive/tasks.py
index 13bf5f2..76f640b 100644 (file)
--- a/src/archive/tasks.py
+++ b/src/archive/tasks.py
@@ -81,9 +81,10 @@ class AudioFormatTask(Task):
             'parts_count': audiobook.parts_count,
             'source_sha1': audiobook.source_sha1,
         }
             'parts_count': audiobook.parts_count,
             'source_sha1': audiobook.source_sha1,
         }

-        api_call(user, UPLOAD_URL, data=data, files={
-            "file": open(path, 'rb'),
-        })
+        with open(path, 'rb') as f:
+            api_call(user, UPLOAD_URL, data=data, files={
+                "file": f,
+            })

 
     def run(self, uid, aid, publish=True):
         aid = int(aid)
 
     def run(self, uid, aid, publish=True):
         aid = int(aid)


@@ -125,11 +126,11 @@ class Mp3Task(AudioFormatTask):
 
     # these shouldn't be staticmethods
     def id3_text(tag, text):
 
     # these shouldn't be staticmethods
     def id3_text(tag, text):

-        return tag(encoding=1, text=text)
+        return tag(encoding=3, text=text)

     def id3_url(tag, text):
         return tag(url=text)
     def id3_comment(tag, text, lang='pol'):
     def id3_url(tag, text):
         return tag(url=text)
     def id3_comment(tag, text, lang='pol'):

-        return tag(encoding=1, lang=lang, desc='', text=text)
+        return tag(encoding=3, lang=lang, desc='', text=text)

     def id3_priv(tag, text, what=''):
         return tag(owner='wolnelektury.pl?%s' % what, data=text.encode('utf-8'))
 
     def id3_priv(tag, text, what=''):
         return tag(owner='wolnelektury.pl?%s' % what, data=text.encode('utf-8'))
 


@@ -178,9 +179,8 @@ class Mp3Task(AudioFormatTask):
 
         if COVER_IMAGE:
             mime = mimetypes.guess_type(COVER_IMAGE)
 
         if COVER_IMAGE:
             mime = mimetypes.guess_type(COVER_IMAGE)

-            f = open(COVER_IMAGE)
-            audio.add(id3.APIC(encoding=0, mime=mime, type=3, desc='', data=f.read()))
-            f.close()
+            with open(COVER_IMAGE, 'rb') as f:
+                audio.add(id3.APIC(encoding=0, mime=mime, type=3, desc='', data=f.read()))

 
         audio.save()
 
 
         audio.save()
 

diff --git a/src/archive/utils.py b/src/archive/utils.py
index f169847..476d506 100644 (file)
--- a/src/archive/utils.py
+++ b/src/archive/utils.py
@@ -31,7 +31,7 @@ class OverwriteStorage(FileSystemStorage):
 
 def sha1_file(f):
     sha = sha1()
 
 def sha1_file(f):
     sha = sha1()

-    for piece in iter(lambda: f.read(1024*1024), ''):
+    for piece in iter(lambda: f.read(1024*1024), b''):

         sha.update(piece)
     return sha.hexdigest()
 
         sha.update(piece)
     return sha.hexdigest()
 

diff --git a/src/archive/views.py b/src/archive/views.py
index f237fab..8621f64 100644 (file)
--- a/src/archive/views.py
+++ b/src/archive/views.py
@@ -198,7 +198,7 @@ def download(request, aid, which="source"):
     if not file_:
         raise Http404
     ext = file_.path.rsplit('.', 1)[-1]
     if not file_:
         raise Http404
     ext = file_.path.rsplit('.', 1)[-1]

-    response = HttpResponse(mimetype='application/force-download')
+    response = HttpResponse(content_type='application/force-download')

     
     response['Content-Disposition'] = "attachment; filename*=UTF-8''%s.%s" % (
         quote(audiobook.title.encode('utf-8'), safe=''), ext)
     
     response['Content-Disposition'] = "attachment; filename*=UTF-8''%s.%s" % (
         quote(audiobook.title.encode('utf-8'), safe=''), ext)

diff --git a/src/audiobooks/settings.py b/src/audiobooks/settings.py
index 46e73d3..7b6250d 100644 (file)
--- a/src/audiobooks/settings.py
+++ b/src/audiobooks/settings.py
@@ -15,12 +15,8 @@ MANAGERS = ADMINS
 
 DATABASES = {
     'default': {
 
 DATABASES = {
     'default': {

-        'ENGINE': 'django.db.backends.sqlite3', # Add 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'.
-        'NAME': os.path.join(PROJECT_ROOT, 'dev.sqlite'),                      # Or path to database file if using sqlite3.
-        'USER': '',                      # Not used with sqlite3.
-        'PASSWORD': '',                  # Not used with sqlite3.
-        'HOST': '',                      # Set to empty string for localhost. Not used with sqlite3.
-        'PORT': '',                      # Set to empty string for default. Not used with sqlite3.
+        'ENGINE': 'django.db.backends.postgresql',
+        'NAME': 'audio',

     }
 }
 
     }
 }