From 7a7629297f382c47a84ec5786c8cdd52224887ae Mon Sep 17 00:00:00 2001 From: Radek Czajka Date: Tue, 19 May 2020 10:57:06 +0200 Subject: [PATCH] YouTube bells'n'whistles. --- requirements.txt | 2 + src/apiclient/__init__.py | 4 +- src/apiclient/models.py | 25 ++++---- src/youtube/admin.py | 3 + src/youtube/migrations/0008_font.py | 21 +++++++ .../migrations/0009_auto_20200518_1656.py | 23 ++++++++ src/youtube/models.py | 57 ++++++++++++++----- src/youtube/urls.py | 2 + src/youtube/utils.py | 41 +++++++++++-- src/youtube/views.py | 10 +++- 10 files changed, 157 insertions(+), 31 deletions(-) create mode 100644 src/youtube/migrations/0008_font.py create mode 100644 src/youtube/migrations/0009_auto_20200518_1656.py diff --git a/requirements.txt b/requirements.txt index d374c51..3cb1899 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,3 +11,5 @@ psycopg2-binary==2.8.4 mutagen==1.42.0 requests==2.23.0 requests-oauthlib==1.3.0 +PyYAML==5.3.1 +Pillow diff --git a/src/apiclient/__init__.py b/src/apiclient/__init__.py index f4da3a5..07b8123 100644 --- a/src/apiclient/__init__.py +++ b/src/apiclient/__init__.py @@ -33,7 +33,7 @@ def api_call(user, path, method='POST', data=None, files=None): raise ApiError("WL API call error %s, path: %s" % (r.status_code, path)) -def youtube_call(method, url, params=None, data=None, media_data=None): +def youtube_call(method, url, params=None, json=None, data=None, resumable_data=None): from .models import YouTubeToken yt = YouTubeToken.objects.first() - return yt.call(method, url, params=params, data=data, media_data=media_data) + return yt.call(method, url, params=params, json=json, data=data, resumable_data=resumable_data) diff --git a/src/apiclient/models.py b/src/apiclient/models.py index ae8c74a..07f804f 100644 --- a/src/apiclient/models.py +++ b/src/apiclient/models.py @@ -37,24 +37,27 @@ class YouTubeToken(models.Model): token_updater=self.token_updater ) - def call(self, method, url, params=None, data=None, media_data=None): + def call(self, method, url, params=None, json=None, data=None, resumable_data=None): params = params or {} - params['uploadType'] = 'resumable' + if resumable_data: + params['uploadType'] = 'resumable' session = self.get_session() response = session.request( method=method, url=url, - json=data, + json=json, + data=data, params=params, headers={ - 'X-Upload-Content-Length': str(len(media_data)), + 'X-Upload-Content-Length': str(len(resumable_data)), 'x-upload-content-type': 'application/octet-stream', - } - ) - location = response.headers['Location'] - return session.put( - url=location, - data=media_data, - headers={"Content-Type": "application/octet-stream"}, + } if resumable_data else {} ) + if resumable_data: + location = response.headers['Location'] + return session.put( + url=location, + data=resumable_data, + headers={"Content-Type": "application/octet-stream"}, + ) diff --git a/src/youtube/admin.py b/src/youtube/admin.py index 4069b61..2edc745 100644 --- a/src/youtube/admin.py +++ b/src/youtube/admin.py @@ -11,3 +11,6 @@ class YouTubeAdmin(admin.ModelAdmin): admin.site.register(models.YouTube, YouTubeAdmin) + + +admin.site.register(models.Font) diff --git a/src/youtube/migrations/0008_font.py b/src/youtube/migrations/0008_font.py new file mode 100644 index 0000000..d7b78d0 --- /dev/null +++ b/src/youtube/migrations/0008_font.py @@ -0,0 +1,21 @@ +# Generated by Django 3.0.4 on 2020-05-18 16:07 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('youtube', '0007_auto_20200515_1645'), + ] + + operations = [ + migrations.CreateModel( + name='Font', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=255, unique=True)), + ('truetype', models.FileField(upload_to='youtube/font')), + ], + ), + ] diff --git a/src/youtube/migrations/0009_auto_20200518_1656.py b/src/youtube/migrations/0009_auto_20200518_1656.py new file mode 100644 index 0000000..ba6c69c --- /dev/null +++ b/src/youtube/migrations/0009_auto_20200518_1656.py @@ -0,0 +1,23 @@ +# Generated by Django 3.0.4 on 2020-05-18 16:56 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('youtube', '0008_font'), + ] + + operations = [ + migrations.AddField( + model_name='youtube', + name='intro_flac', + field=models.FileField(blank=True, upload_to='youtube/intro_flac'), + ), + migrations.AddField( + model_name='youtube', + name='outro_flac', + field=models.FileField(blank=True, upload_to='youtube/outro_flac'), + ), + ] diff --git a/src/youtube/models.py b/src/youtube/models.py index 08b2d18..6306a49 100644 --- a/src/youtube/models.py +++ b/src/youtube/models.py @@ -4,14 +4,17 @@ from tempfile import NamedTemporaryFile from django.db import models from django.utils.translation import gettext_lazy as _ from django.template import Template, Context +import requests from apiclient import youtube_call from .utils import ( - video_from_image, - cut_video, + concat_audio, concat_videos, + cut_video, get_duration, get_framerate, mux, + standardize_video, + video_from_image, ) from .thumbnail import create_thumbnail @@ -20,6 +23,8 @@ class YouTube(models.Model): title_template = models.CharField(max_length=1024, blank=True) description_template = models.TextField(blank=True) category = models.IntegerField(null=True, blank=True) # get categories + intro_flac = models.FileField(upload_to='youtube/intro_flac', blank=True) + outro_flac = models.FileField(upload_to='youtube/outro_flac', blank=True) loop_card = models.FileField(upload_to='youtube/card', blank=True) loop_video = models.FileField(upload_to='youtube/loop_video', blank=True) thumbnail_template = models.FileField(upload_to='youtube/thumbnail', blank=True) @@ -60,8 +65,8 @@ class YouTube(models.Model): "POST", "https://www.googleapis.com/upload/youtube/v3/videos", params={'part': part}, - data=data, - media_data=f.read(), + json=data, + resumable_data=f.read(), ) data = response.json() audiobook.youtube_id = data['id'] @@ -71,12 +76,31 @@ class YouTube(models.Model): return response def prepare_file(self, input_path, output_path=None): - duration = get_duration(input_path) + audio = self.prepare_audio(input_path) + duration = self.get_duration(input_path) video = self.prepare_video(duration) - output = mux([video, input_path], output_path=output_path) + output = mux([video, audio], output_path=output_path) + unlink(audio) unlink(video) return output + def get_duration(self, input_path): + d = get_duration(input_path) + if self.intro_flac: + d += get_duration(self.intro_flac.path) + if self.outro_flac: + d += get_duration(self.outro_flac.path) + return d + + def prepare_audio(self, input_path): + files = [] + if self.intro_flac: + files.append(self.intro_flac.path) + files.append(input_path) + if self.outro_flac: + files.append(self.outro_flac.path) + return concat_audio(files) + def prepare_video(self, duration): concat = [] outro = [] @@ -84,25 +108,26 @@ class YouTube(models.Model): if self.loop_video: fps = get_framerate(self.loop_video.path) + loop_video = standardize_video(self.loop_video.path) else: fps = 25 loop_duration = duration - for card in self.card_set.filter(order__lt=0, duration__gt=0): + for card in self.card_set.filter(duration__gt=0): loop_duration -= card.duration card_video = video_from_image( card.image.path, card.duration, fps=fps ) (concat if card.order < 0 else outro).append(card_video) - delete.append(intro) + delete.append(card_video) if self.loop_video: - loop_video_duration = get_duration(self.loop_video.path) + loop_video_duration = get_duration(loop_video) times_loop = int(loop_duration // loop_video_duration) leftover_duration = loop_duration % loop_video_duration - leftover = cut_video(self.loop_video.path, leftover_duration) - concat.extend([self.loop_video.path] * times_loop + [leftover]) + leftover = cut_video(loop_video, leftover_duration) + concat.extend([loop_video] * times_loop + [leftover]) delete.append(leftover) else: leftover = video_from_image(self.loop_card.path, loop_duration) @@ -113,6 +138,7 @@ class YouTube(models.Model): output = concat_videos(concat) for p in delete: unlink(p) + unlink(loop_video) return output # tags @@ -125,14 +151,19 @@ class YouTube(models.Model): "POST", "https://www.googleapis.com/upload/youtube/v3/thumbnails/set", params={'videoId': audiobook.youtube_id}, - media_data=buf.read(), # Or just data? + data=thumbnail.getvalue(), ) def prepare_thumbnail(self, audiobook): + slug = audiobook.url.rstrip('/').rsplit('/', 1)[-1] + apidata = requests.get(f'https://wolnelektury.pl/api/books/{slug}/').json() img = create_thumbnail( self.thumbnail_template.path, self.thumbnail_definition, - {}, # TODO proper context + { + "author": ', '.join((a['name'] for a in apidata['authors'])), + "title": apidata['title'], + }, lambda name: Font.objects.get(name=name).truetype.path ) buf = io.BytesIO() diff --git a/src/youtube/urls.py b/src/youtube/urls.py index f170e2b..8dbd3d8 100644 --- a/src/youtube/urls.py +++ b/src/youtube/urls.py @@ -1,7 +1,9 @@ from django.conf.urls import url +from django.urls import path from . import views urlpatterns = [ url(r'^publish/(\d+)/$', views.publish, name="youtube_publish"), url(r'^convert/(\d+)/$', views.publish, {'publish': False}, name="youtube_convert"), + path('thumbnail//', views.thumbnail, name='youtube_thumbnail'), ] diff --git a/src/youtube/utils.py b/src/youtube/utils.py index 59d1033..d5b320e 100644 --- a/src/youtube/utils.py +++ b/src/youtube/utils.py @@ -1,3 +1,4 @@ +from os import unlink import subprocess from tempfile import NamedTemporaryFile @@ -6,7 +7,7 @@ def video_from_image(img_path, duration, fps=25): tmp = NamedTemporaryFile(prefix='image', suffix='.mkv', delete=False) tmp.close() subprocess.run( - ['ffmpeg', '-y', '-framerate', f'1/{duration}', '-i', img_path, '-c:v', 'libx264', '-vf', f'fps={fps},format=yuv420p', tmp.name], check=True) + ['ffmpeg', '-y', '-loop', '1', '-t', str(duration), '-i', img_path, '-c:v', 'libx264', '-vf', f'fps={fps},format=yuv420p', tmp.name], check=True) return tmp.name @@ -18,13 +19,13 @@ def cut_video(video_path, duration): return tmp.name -def concat_videos(paths): +def ffmpeg_concat(paths, suffix): filelist = NamedTemporaryFile(prefix='concat', suffix='.txt') for path in paths: filelist.write(f"file '{path}'\n".encode('utf-8')) filelist.flush() - output = NamedTemporaryFile(prefix='concat', suffix='.mkv', delete=False) + output = NamedTemporaryFile(prefix='concat', suffix=suffix, delete=False) output.close() subprocess.run( @@ -35,6 +36,37 @@ def concat_videos(paths): return output.name +def concat_videos(paths): + return ffmpeg_concat(paths, '.mkv') + +def concat_audio(paths): + std_paths = [ + standardize_audio(p) + for p in paths + ] + output = ffmpeg_concat(std_paths, '.flac') + for p in std_paths: + unlink(p) + return output + +def standardize_audio(p): + output = NamedTemporaryFile(prefix='standarize', suffix='.flac', delete=False) + output.close() + subprocess.run( + ['ffmpeg', '-y', '-i', p, '-sample_fmt', 's16', '-acodec', 'flac', '-ac', '2', '-ar', '44100', output.name], + check=True) + return output.name + +def standardize_video(p): + output = NamedTemporaryFile(prefix='standarize', suffix='.mkv', delete=False) + output.close() + subprocess.run( + ['ffmpeg', '-y', '-i', p, output.name], + check=True) + return output.name + + + def mux(channels, output_path=None): if not output_path: output = NamedTemporaryFile(prefix='concat', suffix='.mkv', delete=False) @@ -43,7 +75,8 @@ def mux(channels, output_path=None): args = ['ffmpeg'] for c in channels: args.extend(['-i', c]) - args.extend(['-shortest', '-y', output_path]) + args.extend([ + '-y', output_path]) subprocess.run(args, check=True) return output_path diff --git a/src/youtube/views.py b/src/youtube/views.py index 1247c50..4e59f5f 100644 --- a/src/youtube/views.py +++ b/src/youtube/views.py @@ -1,10 +1,11 @@ from django.contrib.auth.decorators import permission_required +from django.http import HttpResponse from django.shortcuts import redirect, get_object_or_404 from django.urls import reverse from django.views.decorators.http import require_POST from archive.constants import status from archive.models import Audiobook -from . import tasks +from . import models, tasks @@ -19,3 +20,10 @@ def publish(request, aid, publish=True): audiobook.youtube_task = tasks.YouTubeTask.delay(request.user.id, aid, publish).task_id audiobook.save(update_fields=['youtube_task']) return redirect(reverse('file', args=[aid])) + + +def thumbnail(request, aid): + audiobook = get_object_or_404(Audiobook, id=aid) + yt = models.YouTube.objects.first() + buf = yt.prepare_thumbnail(audiobook) + return HttpResponse(buf.getvalue(), content_type='image/png') -- 2.20.1