From: Radek Czajka Date: Fri, 29 May 2020 13:32:26 +0000 (+0200) Subject: More automation for YouTube: volume management (WiP). X-Git-Url: https://git.mdrn.pl/audio.git/commitdiff_plain/cf1c5c7118e320bdd5b9fa6dcef8585ba117ebb6 More automation for YouTube: volume management (WiP). --- diff --git a/src/archive/constants.py b/src/archive/constants.py index 6cb92a7..3493e00 100644 --- a/src/archive/constants.py +++ b/src/archive/constants.py @@ -1,14 +1,28 @@ from django.utils.translation import gettext_lazy as _ class status: - WAITING = 1 - ENCODING = 2 - TAGGING = 3 - SENDING = 4 + QUEUED = 1 + WAITING = 10 + ENCODING = 20 + TAGGING = 30 + CONVERTING_AUDIO = 40 + CONVERTING_VIDEO = 50 + ASSEMBLING_AUDIO = 60 + ASSEMBLING_VIDEO = 70 + JOINING_AUDIO_VIDEO = 80 + SENDING = 100 + SETTING_THUMBNAIL = 110 choices = [ + (QUEUED, _('Queued')), (WAITING, _('Waiting')), (ENCODING, _('Encoding')), (TAGGING, _('Tagging')), + (CONVERTING_AUDIO, _('Converting audio')), + (CONVERTING_VIDEO, _('Converting video')), + (ASSEMBLING_AUDIO, _('Assembling audio')), + (ASSEMBLING_VIDEO, _('Assembling video')), + (JOINING_AUDIO_VIDEO, _('Joining audio and video')), (SENDING, _('Sending')), + (SETTING_THUMBNAIL, _('Setting thumbnail')), ] diff --git a/src/archive/migrations/0011_audiobook_slug.py b/src/archive/migrations/0011_audiobook_slug.py new file mode 100644 index 0000000..520a0d8 --- /dev/null +++ b/src/archive/migrations/0011_audiobook_slug.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.6 on 2020-05-29 11:57 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('archive', '0010_populate_license'), + ] + + operations = [ + migrations.AddField( + model_name='audiobook', + name='slug', + field=models.SlugField(default='', max_length=120), + preserve_default=False, + ), + ] diff --git a/src/archive/migrations/0012_url_to_slug.py b/src/archive/migrations/0012_url_to_slug.py new file mode 100644 index 0000000..d880c95 --- /dev/null +++ b/src/archive/migrations/0012_url_to_slug.py @@ -0,0 +1,30 @@ +# Generated by Django 3.0.6 on 2020-05-29 11:57 + +from django.db import migrations + + +def url_to_slug(apps, schema_editor): + Audiobook = apps.get_model('archive', 'Audiobook') + for a in Audiobook.objects.all(): + a.slug = a.url.rstrip().rstrip('/').rsplit('/', 1)[-1] + a.save() + +def slug_to_url(apps, schema_editor): + Audiobook = apps.get_model('archive', 'Audiobook') + for a in Audiobook.objects.all(): + a.url = 'https://wolnelektury.pl/katalog/lektura/{}/'.format(a.slug) + a.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ('archive', '0011_audiobook_slug'), + ] + + operations = [ + migrations.RunPython( + url_to_slug, + slug_to_url + ) + ] diff --git a/src/archive/migrations/0013_remove_audiobook_url.py b/src/archive/migrations/0013_remove_audiobook_url.py new file mode 100644 index 0000000..84a6741 --- /dev/null +++ b/src/archive/migrations/0013_remove_audiobook_url.py @@ -0,0 +1,17 @@ +# Generated by Django 3.0.6 on 2020-05-29 12:10 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('archive', '0012_url_to_slug'), + ] + + operations = [ + migrations.RemoveField( + model_name='audiobook', + name='url', + ), + ] diff --git a/src/archive/migrations/0014_remove_audiobook_parts_count.py b/src/archive/migrations/0014_remove_audiobook_parts_count.py new file mode 100644 index 0000000..14be63f --- /dev/null +++ b/src/archive/migrations/0014_remove_audiobook_parts_count.py @@ -0,0 +1,17 @@ +# Generated by Django 3.0.6 on 2020-05-29 14:23 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('archive', '0013_remove_audiobook_url'), + ] + + operations = [ + migrations.RemoveField( + model_name='audiobook', + name='parts_count', + ), + ] diff --git a/src/archive/migrations/0015_auto_20200529_1430.py b/src/archive/migrations/0015_auto_20200529_1430.py new file mode 100644 index 0000000..2702f97 --- /dev/null +++ b/src/archive/migrations/0015_auto_20200529_1430.py @@ -0,0 +1,23 @@ +# Generated by Django 3.0.6 on 2020-05-29 14:30 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('archive', '0014_remove_audiobook_parts_count'), + ] + + operations = [ + migrations.AddField( + model_name='audiobook', + name='youtube_queued', + field=models.DateTimeField(blank=True, null=True), + ), + migrations.AddField( + model_name='audiobook', + name='youtube_volume', + field=models.CharField(blank=True, help_text='If set, audiobooks with the save value will be published as single YouTube video.', max_length=1000, verbose_name='Volume name for YouTube'), + ), + ] diff --git a/src/archive/migrations/0016_auto_20200529_1528.py b/src/archive/migrations/0016_auto_20200529_1528.py new file mode 100644 index 0000000..65c4cb3 --- /dev/null +++ b/src/archive/migrations/0016_auto_20200529_1528.py @@ -0,0 +1,38 @@ +# Generated by Django 3.0.6 on 2020-05-29 15:28 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('archive', '0015_auto_20200529_1430'), + ] + + operations = [ + migrations.AlterField( + model_name='audiobook', + name='index', + field=models.IntegerField(default=0, help_text='Ordering of parts of a book.', verbose_name='index'), + ), + migrations.AlterField( + model_name='audiobook', + name='mp3_status', + field=models.SmallIntegerField(choices=[(1, 'Queued'), (10, 'Waiting'), (20, 'Encoding'), (30, 'Tagging'), (40, 'Converting audio'), (50, 'Converting video'), (60, 'Assembling audio'), (70, 'Assembling video'), (80, 'Joining audio and video'), (100, 'Sending'), (110, 'Setting thumbnail')], editable=False, null=True), + ), + migrations.AlterField( + model_name='audiobook', + name='ogg_status', + field=models.SmallIntegerField(choices=[(1, 'Queued'), (10, 'Waiting'), (20, 'Encoding'), (30, 'Tagging'), (40, 'Converting audio'), (50, 'Converting video'), (60, 'Assembling audio'), (70, 'Assembling video'), (80, 'Joining audio and video'), (100, 'Sending'), (110, 'Setting thumbnail')], editable=False, null=True), + ), + migrations.AlterField( + model_name='audiobook', + name='slug', + field=models.SlugField(help_text='WL catalogue slug of the book.', max_length=120), + ), + migrations.AlterField( + model_name='audiobook', + name='youtube_status', + field=models.SmallIntegerField(choices=[(1, 'Queued'), (10, 'Waiting'), (20, 'Encoding'), (30, 'Tagging'), (40, 'Converting audio'), (50, 'Converting video'), (60, 'Assembling audio'), (70, 'Assembling video'), (80, 'Joining audio and video'), (100, 'Sending'), (110, 'Setting thumbnail')], editable=False, null=True), + ), + ] diff --git a/src/archive/models.py b/src/archive/models.py index a4420a9..ad1ef78 100644 --- a/src/archive/models.py +++ b/src/archive/models.py @@ -66,14 +66,14 @@ class Audiobook(models.Model): title = models.CharField(max_length=255, verbose_name=_('title')) part_name = models.CharField(max_length=255, verbose_name=_('part name'), help_text=_('eg. chapter in a novel'), default='', blank=True) - index = models.IntegerField(verbose_name=_('index'), default=0) - parts_count = models.IntegerField(verbose_name=_('parts count'), default=1) + index = models.IntegerField(verbose_name=_('index'), default=0, help_text=_('Ordering of parts of a book.')) + youtube_volume = models.CharField(_('Volume name for YouTube'), max_length=1000, blank=True, help_text=_('If set, audiobooks with the save value will be published as single YouTube video.')) artist = models.CharField(max_length=255, verbose_name=_('artist')) conductor = models.CharField(max_length=255, verbose_name=_('conductor')) encoded_by = models.CharField(max_length=255, verbose_name=_('encoded by')) date = models.CharField(max_length=255, verbose_name=_('date')) project = models.ForeignKey(Project, models.PROTECT, verbose_name=_('project')) - url = models.URLField(max_length=255, verbose_name=_('book url')) + slug = models.SlugField(max_length=120, help_text=_('WL catalogue slug of the book.')) translator = models.CharField(max_length=255, null=True, blank=True, verbose_name=_('translator')) modified = models.DateTimeField(null=True, editable=False) license = models.ForeignKey(License, models.PROTECT, null=True, blank=True, verbose_name=_('license')) @@ -99,6 +99,7 @@ class Audiobook(models.Model): youtube_published_tags = models.TextField(null=True, editable=False) youtube_published = models.DateTimeField(null=True, editable=False) youtube_id = models.CharField(max_length=255, blank=True, default='') + youtube_queued = models.DateTimeField(null=True, blank=True) class Meta: verbose_name = _("audiobook") @@ -108,6 +109,34 @@ class Audiobook(models.Model): def __str__(self): return self.title + @property + def url(self): + return f'https://wolnelektury.pl/katalog/lektura/{self.slug}/' + + @property + def parts_count(self): + return type(self).objects.filter(slug=self.slug).count() + + @property + def youtube_volume_count(self): + total = 0 + prev_volume = None + for a in type(self).objects.filter(slug=self.slug).order_by("index"): + if not a.youtube_volume or a.youtube_volume != prev_volume: + total += 1 + prev_volume = a.youtube_volume + return total + + @property + def youtube_volume_index(self): + index = 0 + prev_volume = None + for a in type(self).objects.filter(slug=self.slug, index__lte=self.index).order_by("index"): + if not a.youtube_volume or a.youtube_volume != prev_volume: + index += 1 + prev_volume = a.youtube_volume + return index + def get_mp3_tags(self): return json.loads(self.mp3_tags) if self.mp3_tags else None def get_ogg_tags(self): return json.loads(self.ogg_tags) if self.ogg_tags else None def get_mp3_published_tags(self): return json.loads(self.mp3_published_tags) if self.mp3_published_tags else None @@ -169,7 +198,6 @@ class Audiobook(models.Model): @cached_property def book(self): - slug = self.url.rstrip('/').rsplit('/', 1)[-1] - apidata = requests.get(f'https://wolnelektury.pl/api/books/{slug}/').json() + apidata = requests.get(f'https://wolnelektury.pl/api/books/{self.slug}/').json() return apidata diff --git a/src/archive/tasks.py b/src/archive/tasks.py index 4b8f5fe..157824a 100644 --- a/src/archive/tasks.py +++ b/src/archive/tasks.py @@ -91,7 +91,10 @@ class AudioFormatTask(Task): audiobook = Audiobook.objects.get(id=aid) self.set_status(aid, status.ENCODING) - user = User.objects.get(id=uid) + if uid: + user = User.objects.get(id=uid) + else: + user = None out_file = NamedTemporaryFile(delete=False, prefix='%d-' % aid, suffix='.%s' % self.ext) out_file.close() diff --git a/src/archive/templates/archive/book.html b/src/archive/templates/archive/book.html new file mode 100644 index 0000000..fac3700 --- /dev/null +++ b/src/archive/templates/archive/book.html @@ -0,0 +1,29 @@ +{% extends "archive/base.html" %} + +{% block content %} +
+
+

Audiobooki

+
+
+ + + + + + + + {% for audiobook in object_list %} + + + + + {% endfor %} + +
?x
{{ audiobook.index }} + + {{ audiobook.title }} + +
+
+{% endblock %} diff --git a/src/archive/urls.py b/src/archive/urls.py index 32a6f73..da1873c 100644 --- a/src/archive/urls.py +++ b/src/archive/urls.py @@ -1,4 +1,5 @@ from django.conf.urls import url +from django.urls import path from django.views.generic import RedirectView from . import views @@ -12,6 +13,7 @@ urlpatterns = [ url(r'^unpublished/$', views.list_unpublished, name="list_unpublished"), url(r'^publishing/$', views.list_publishing, name="list_publishing"), url(r'^published/$', views.list_published, name="list_published"), + path('book//', views.BookView.as_view(), name="book"), url(r'^file/(\d+)/$', views.file_managed, name="file"), url(r'^publish/(\d+)/$', views.publish, name="publish"), url(r'^convert/(\d+)/$', views.publish, {'publish': False}, name="convert"), diff --git a/src/archive/views.py b/src/archive/views.py index aee88e6..10a998f 100644 --- a/src/archive/views.py +++ b/src/archive/views.py @@ -11,6 +11,7 @@ from django.http import Http404, HttpResponse from django.shortcuts import render, redirect, get_object_or_404 from django.utils.translation import gettext as _ from django.views.decorators.http import require_POST +from django.views.generic import ListView import mutagen @@ -270,15 +271,13 @@ def file_managed(request, id): request.user.oauthconnection_set.filter(access=True).exists()) alerts = [] - series = models.Audiobook.objects.filter(url=audiobook.url) - real = series.count() - if real != audiobook.parts_count: - alerts.append(_('Parts number inconsitent. Declared number: %(declared)d. Real number: %(real)d') % {"declared": audiobook.parts_count, "real": real}) - if audiobook.parts_count > 1: + parts_count = audiobook.parts_count + if parts_count > 1: + series = models.Audiobook.objects.filter(slug=audiobook.slug) if not audiobook.index: alerts.append(_('There is more than one part, but index is not set.')) - if set(series.values_list('index', flat=True)) != set(range(1, audiobook.parts_count + 1)): - alerts.append(_('Part indexes are not 1..%(parts_count)d.') % {"parts_count": audiobook.parts_count}) + if set(series.values_list('index', flat=True)) != set(range(1, parts_count + 1)): + alerts.append(_('Part indexes are not 1..%(parts_count)d.') % {"parts_count": parts_count}) return render(request, "archive/file_managed.html", locals()) @@ -299,3 +298,10 @@ def file_unmanaged(request, filename): err_exists = request.GET.get('exists') return render(request, "archive/file_unmanaged.html", locals()) + + +class BookView(ListView): + template_name = 'archive/book.html' + + def get_queryset(self): + return models.Audiobook.objects.filter(slug=self.kwargs['slug']) diff --git a/src/youtube/urls.py b/src/youtube/urls.py index 58071e5..fb9ddc7 100644 --- a/src/youtube/urls.py +++ b/src/youtube/urls.py @@ -4,7 +4,6 @@ from . import views urlpatterns = [ url(r'^publish/(\d+)/$', views.publish, name="youtube_publish"), - url(r'^convert/(\d+)/$', views.publish, {'publish': False}, name="youtube_convert"), path('thumbnail//', views.thumbnail, name='youtube_thumbnail'), path('preview//', views.Preview.as_view(), name="youtube_preview"), path('update//', views.Update.as_view(), name="youtube_update"), diff --git a/src/youtube/views.py b/src/youtube/views.py index 73eecd3..cbe1520 100644 --- a/src/youtube/views.py +++ b/src/youtube/views.py @@ -3,6 +3,7 @@ from django.http import HttpResponse from django.shortcuts import redirect, get_object_or_404 from django.urls import reverse from django.utils.decorators import method_decorator +from django.utils.timezone import now from django.views import View from django.views.decorators.http import require_POST from django.views.generic import DetailView @@ -17,12 +18,9 @@ from . import models, tasks @permission_required('archive.change_audiobook') def publish(request, aid, publish=True): audiobook = get_object_or_404(Audiobook, id=aid) - tags = {} - #audiobook.set_youtube_tags(tags) - audiobook.youtube_status = status.WAITING - audiobook.save(update_fields=['youtube_status']) - audiobook.youtube_task = tasks.YouTubeTask.delay(request.user.id, aid, publish).task_id - audiobook.save(update_fields=['youtube_task']) + audiobook.youtube_status = status.QUEUED + audiobook.youtube_queued = now() + audiobook.save(update_fields=['youtube_status', 'youtube_queued']) return redirect(reverse('file', args=[aid]))