From: Radek Czajka Date: Mon, 30 Mar 2020 12:41:14 +0000 (+0200) Subject: Use proper locking, and other minor fixes. X-Git-Url: https://git.mdrn.pl/audio.git/commitdiff_plain/583f4d519b134720024e4fa387b5a00561bd00cf Use proper locking, and other minor fixes. --- diff --git a/requirements.txt b/requirements.txt index 12bcbf9..76f851d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,13 +2,12 @@ django>=1.11,<2.0 django-cas-ng==3.6.0 -django-bootstrap4==0.0.7 +django-bootstrap4==0.0.8 +django-pglocks==1.0.4 -celery[redis]==4.2 -kombu -redis -vine +celery[redis]==4.4.0 +psycopg2-binary==2.8.4 mutagen==1.42.0 -requests -requests-oauthlib +requests==2.23.0 +requests-oauthlib==1.3.0 diff --git a/src/archive/models.py b/src/archive/models.py index 4a1d763..9e92fb8 100644 --- a/src/archive/models.py +++ b/src/archive/models.py @@ -1,9 +1,10 @@ +import json import os.path from django.db import models from time import sleep -from django.utils.encoding import force_bytes from django.utils.translation import ugettext_lazy as _ +from django_pglocks import advisory_lock from archive.constants import status from archive.settings import FILES_SAVE_PATH, ADVERT, LICENSE, ORGANIZATION, PROJECT from archive.utils import OverwriteStorage, sha1_file @@ -81,28 +82,13 @@ class Audiobook(models.Model): return self.mp3_published and self.ogg_published def get_source_sha1(self): - source_sha1 = self.source_sha1 - if self.pk: - source_sha1 = type(self).objects.get(pk=self.pk).source_sha1 - while source_sha1 == 'wait': - sleep(10) - if not source_sha1: - self.source_sha1 = 'wait' - if self.pk: - type(self).objects.filter(pk=self.pk).update(source_sha1='wait') - try: - f = open(force_bytes(self.source_file.path)) - source_sha1 = sha1_file(f) - self.source_sha1 = source_sha1 - if self.pk: - type(self).objects.filter(pk=self.pk).update(source_sha1=source_sha1) - except: - self.source_sha1 = '' - if self.pk: - type(self).objects.filter(pk=self.pk).update(source_sha1='') - return None - return source_sha1 - + assert self.pk or self.source_sha1 + if not self.source_sha1: + with advisory_lock(f'get_source_sha1_{self.pk}'): + with open(self.source_file.path, 'rb') as f: + self.source_sha1 = sha1_file(f) + self.save(update_fields=['source_sha1']) + return self.source_sha1 def new_publish_tags(self): title = self.title @@ -131,11 +117,10 @@ class Audiobook(models.Model): 'license': LICENSE, 'organization': ORGANIZATION, 'title': title, - #'flac_sha1': self.get_source_sha1(), 'project': self.project.name, 'funded_by': self.project.sponsors, } - if self.source_sha1 and self.source_sha1 != 'wait': + if self.source_sha1: tags['flac_sha1'] = self.source_sha1 return tags diff --git a/src/archive/tasks.py b/src/archive/tasks.py index 13bf5f2..76f640b 100644 --- a/src/archive/tasks.py +++ b/src/archive/tasks.py @@ -81,9 +81,10 @@ class AudioFormatTask(Task): 'parts_count': audiobook.parts_count, 'source_sha1': audiobook.source_sha1, } - api_call(user, UPLOAD_URL, data=data, files={ - "file": open(path, 'rb'), - }) + with open(path, 'rb') as f: + api_call(user, UPLOAD_URL, data=data, files={ + "file": f, + }) def run(self, uid, aid, publish=True): aid = int(aid) @@ -125,11 +126,11 @@ class Mp3Task(AudioFormatTask): # these shouldn't be staticmethods def id3_text(tag, text): - return tag(encoding=1, text=text) + return tag(encoding=3, text=text) def id3_url(tag, text): return tag(url=text) def id3_comment(tag, text, lang='pol'): - return tag(encoding=1, lang=lang, desc='', text=text) + return tag(encoding=3, lang=lang, desc='', text=text) def id3_priv(tag, text, what=''): return tag(owner='wolnelektury.pl?%s' % what, data=text.encode('utf-8')) @@ -178,9 +179,8 @@ class Mp3Task(AudioFormatTask): if COVER_IMAGE: mime = mimetypes.guess_type(COVER_IMAGE) - f = open(COVER_IMAGE) - audio.add(id3.APIC(encoding=0, mime=mime, type=3, desc='', data=f.read())) - f.close() + with open(COVER_IMAGE, 'rb') as f: + audio.add(id3.APIC(encoding=0, mime=mime, type=3, desc='', data=f.read())) audio.save() diff --git a/src/archive/utils.py b/src/archive/utils.py index f169847..476d506 100644 --- a/src/archive/utils.py +++ b/src/archive/utils.py @@ -31,7 +31,7 @@ class OverwriteStorage(FileSystemStorage): def sha1_file(f): sha = sha1() - for piece in iter(lambda: f.read(1024*1024), ''): + for piece in iter(lambda: f.read(1024*1024), b''): sha.update(piece) return sha.hexdigest() diff --git a/src/archive/views.py b/src/archive/views.py index f237fab..8621f64 100644 --- a/src/archive/views.py +++ b/src/archive/views.py @@ -198,7 +198,7 @@ def download(request, aid, which="source"): if not file_: raise Http404 ext = file_.path.rsplit('.', 1)[-1] - response = HttpResponse(mimetype='application/force-download') + response = HttpResponse(content_type='application/force-download') response['Content-Disposition'] = "attachment; filename*=UTF-8''%s.%s" % ( quote(audiobook.title.encode('utf-8'), safe=''), ext) diff --git a/src/audiobooks/settings.py b/src/audiobooks/settings.py index 46e73d3..7b6250d 100644 --- a/src/audiobooks/settings.py +++ b/src/audiobooks/settings.py @@ -15,12 +15,8 @@ MANAGERS = ADMINS DATABASES = { 'default': { - 'ENGINE': 'django.db.backends.sqlite3', # Add 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'. - 'NAME': os.path.join(PROJECT_ROOT, 'dev.sqlite'), # Or path to database file if using sqlite3. - 'USER': '', # Not used with sqlite3. - 'PASSWORD': '', # Not used with sqlite3. - 'HOST': '', # Set to empty string for localhost. Not used with sqlite3. - 'PORT': '', # Set to empty string for default. Not used with sqlite3. + 'ENGINE': 'django.db.backends.postgresql', + 'NAME': 'audio', } }