From: Radek Czajka Date: Mon, 6 Apr 2020 11:52:53 +0000 (+0200) Subject: YouTube support. X-Git-Url: https://git.mdrn.pl/audio.git/commitdiff_plain/d7186e88fcc0cf5d5449a16d3907e01aaafc484e?ds=sidebyside;hp=c84a8c9ad465608d565821914169f19f30b3b2c3 YouTube support. --- diff --git a/src/apiclient/__init__.py b/src/apiclient/__init__.py index e489807..f4da3a5 100644 --- a/src/apiclient/__init__.py +++ b/src/apiclient/__init__.py @@ -32,3 +32,8 @@ def api_call(user, path, method='POST', data=None, files=None): else: raise ApiError("WL API call error %s, path: %s" % (r.status_code, path)) + +def youtube_call(method, url, params=None, data=None, media_data=None): + from .models import YouTubeToken + yt = YouTubeToken.objects.first() + return yt.call(method, url, params=params, data=data, media_data=media_data) diff --git a/src/apiclient/migrations/0002_youtubetoken.py b/src/apiclient/migrations/0002_youtubetoken.py new file mode 100644 index 0000000..c6772d4 --- /dev/null +++ b/src/apiclient/migrations/0002_youtubetoken.py @@ -0,0 +1,20 @@ +# Generated by Django 3.0.4 on 2020-04-06 10:41 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('apiclient', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='YouTubeToken', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('token', models.TextField()), + ], + ), + ] diff --git a/src/apiclient/models.py b/src/apiclient/models.py index 83979e8..ae8c74a 100644 --- a/src/apiclient/models.py +++ b/src/apiclient/models.py @@ -1,5 +1,8 @@ +import json from django.db import models from django.contrib.auth.models import User +from requests_oauthlib import OAuth2Session +from .settings import YOUTUBE_CLIENT_ID, YOUTUBE_CLIENT_SECRET, YOUTUBE_TOKEN_URL class OAuthConnection(models.Model): @@ -18,3 +21,40 @@ class OAuthConnection(models.Model): return o +class YouTubeToken(models.Model): + token = models.TextField() + + def token_updater(self, token): + self.token = json.dumps(token) + self.save() + + def get_session(self): + return OAuth2Session( + client_id=YOUTUBE_CLIENT_ID, + auto_refresh_url=YOUTUBE_TOKEN_URL, + token=json.loads(self.token), + auto_refresh_kwargs={'client_id':YOUTUBE_CLIENT_ID,'client_secret':YOUTUBE_CLIENT_SECRET}, + token_updater=self.token_updater + ) + + def call(self, method, url, params=None, data=None, media_data=None): + params = params or {} + params['uploadType'] = 'resumable' + + session = self.get_session() + response = session.request( + method=method, + url=url, + json=data, + params=params, + headers={ + 'X-Upload-Content-Length': str(len(media_data)), + 'x-upload-content-type': 'application/octet-stream', + } + ) + location = response.headers['Location'] + return session.put( + url=location, + data=media_data, + headers={"Content-Type": "application/octet-stream"}, + ) diff --git a/src/apiclient/settings.py b/src/apiclient/settings.py index 49037fe..f347229 100755 --- a/src/apiclient/settings.py +++ b/src/apiclient/settings.py @@ -12,3 +12,13 @@ WL_ACCESS_TOKEN_URL = getattr(settings, 'APICLIENT_WL_ACCESS_TOKEN_URL', WL_API_URL + 'oauth/access_token/') WL_AUTHORIZE_URL = getattr(settings, 'APICLIENT_WL_AUTHORIZE_URL', WL_API_URL + 'oauth/authorize/') + + +YOUTUBE_SCOPE = [ + 'https://www.googleapis.com/auth/youtube', +] +YOUTUBE_AUTH_URL = 'https://accounts.google.com/o/oauth2/auth' +YOUTUBE_TOKEN_URL = 'https://oauth2.googleapis.com/token' +YOUTUBE_CLIENT_ID = getattr(settings, 'YOUTUBE_CLIENT_ID', None) +YOUTUBE_CLIENT_SECRET = getattr(settings, 'YOUTUBE_CLIENT_SECRET', None) + diff --git a/src/apiclient/urls.py b/src/apiclient/urls.py index 09d4712..ad3f166 100755 --- a/src/apiclient/urls.py +++ b/src/apiclient/urls.py @@ -5,4 +5,6 @@ from . import views urlpatterns = [ url(r'^oauth/$', views.oauth, name='apiclient_oauth'), url(r'^oauth_callback/$', views.oauth_callback, name='apiclient_oauth_callback'), + url(r'^oauth2/$', views.oauth2, name='apiclient_oauth2'), + url(r'^oauth2_redirect/$', views.oauth2_redirect, name='apiclient_oauth2_redirect'), ] diff --git a/src/apiclient/views.py b/src/apiclient/views.py index f41f8f3..25fd5c6 100644 --- a/src/apiclient/views.py +++ b/src/apiclient/views.py @@ -1,12 +1,13 @@ +import json from django.contrib.auth.decorators import login_required from django.urls import reverse from django.http import HttpResponseRedirect, HttpResponse import requests -from requests_oauthlib import OAuth1Session +from requests_oauthlib import OAuth1Session, OAuth2Session -from apiclient.models import OAuthConnection +from apiclient.models import OAuthConnection, YouTubeToken from apiclient.settings import WL_CONSUMER_KEY, WL_CONSUMER_SECRET -from apiclient.settings import WL_REQUEST_TOKEN_URL, WL_ACCESS_TOKEN_URL, WL_AUTHORIZE_URL +from apiclient.settings import WL_REQUEST_TOKEN_URL, WL_ACCESS_TOKEN_URL, WL_AUTHORIZE_URL, YOUTUBE_AUTH_URL, YOUTUBE_TOKEN_URL, YOUTUBE_CLIENT_ID, YOUTUBE_CLIENT_SECRET, YOUTUBE_SCOPE @login_required @@ -42,3 +43,25 @@ def oauth_callback(request): conn.save() return HttpResponseRedirect('/') + + +@login_required +def oauth2(request): + redirect_uri = request.build_absolute_uri(reverse("apiclient_oauth2_redirect")) + oauth = OAuth2Session(YOUTUBE_CLIENT_ID, redirect_uri=redirect_uri, scope=YOUTUBE_SCOPE) + authorization_url, state = oauth.authorization_url(YOUTUBE_AUTH_URL) + return HttpResponseRedirect(authorization_url) + + +@login_required +def oauth2_redirect(request): + redirect_uri = request.build_absolute_uri(reverse("apiclient_oauth2_redirect")) + oauth = OAuth2Session(YOUTUBE_CLIENT_ID, redirect_uri=redirect_uri, scope=YOUTUBE_SCOPE) + token = oauth.fetch_token( + YOUTUBE_TOKEN_URL, + authorization_response=request.get_full_path(), + client_secret=YOUTUBE_CLIENT_SECRET, + ) + YouTubeToken.objects.all().delete() + YouTubeToken.objects.create(token=json.dumps(token)) + return HttpResponseRedirect('/') diff --git a/src/archive/locale/pl/LC_MESSAGES/django.mo b/src/archive/locale/pl/LC_MESSAGES/django.mo index e1135bf..3156f35 100644 Binary files a/src/archive/locale/pl/LC_MESSAGES/django.mo and b/src/archive/locale/pl/LC_MESSAGES/django.mo differ diff --git a/src/archive/locale/pl/LC_MESSAGES/django.po b/src/archive/locale/pl/LC_MESSAGES/django.po index 317ab76..7820153 100644 --- a/src/archive/locale/pl/LC_MESSAGES/django.po +++ b/src/archive/locale/pl/LC_MESSAGES/django.po @@ -5,18 +5,19 @@ # msgid "" msgstr "" -"Project-Id-Version: PACKAGE VERSION\n" +"Project-Id-Version: \n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2018-03-02 09:36+0100\n" -"PO-Revision-Date: 2012-07-11 15:52+0100\n" +"POT-Creation-Date: 2020-04-01 14:18+0200\n" +"PO-Revision-Date: 2020-04-01 14:19+0200\n" "Last-Translator: Radek Czajka \n" -"Language-Team: LANGUAGE \n" -"Language: \n" +"Language: pl\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" "Plural-Forms: nplurals=3; plural=(n==1 ? 0 : n%10>=2 && n%10<=4 && (n%100<10 " -"|| n%100>=20) ? 1 : 2)\n" +"|| n%100>=20) ? 1 : 2);\n" +"Language-Team: \n" +"X-Generator: Poedit 2.2.4\n" #: constants.py:10 msgid "Waiting" @@ -34,111 +35,107 @@ msgstr "Opisywanie" msgid "Sending" msgstr "Wysyłanie" -#: models.py:23 models.py:49 +#: models.py:20 models.py:46 msgid "project" msgstr "projekt" -#: models.py:24 +#: models.py:21 msgid "projects" msgstr "projekty" -#: models.py:37 +#: models.py:34 msgid "source file" msgstr "plik źródłowy" -#: models.py:40 +#: models.py:37 msgid "title" msgstr "tytuł" -#: models.py:41 +#: models.py:38 msgid "part name" msgstr "nazwa części" -#: models.py:41 +#: models.py:38 msgid "eg. chapter in a novel" msgstr "np. rozdział w powieści" -#: models.py:43 +#: models.py:40 msgid "index" msgstr "numer" -#: models.py:44 +#: models.py:41 msgid "parts count" msgstr "liczba części" -#: models.py:45 +#: models.py:42 msgid "artist" msgstr "lektor" -#: models.py:46 +#: models.py:43 msgid "conductor" msgstr "reżyser" -#: models.py:47 +#: models.py:44 msgid "encoded by" msgstr "przyg. techn." -#: models.py:48 +#: models.py:45 msgid "date" msgstr "data" -#: models.py:50 +#: models.py:47 msgid "book url" msgstr "URL książki" -#: models.py:51 +#: models.py:48 msgid "translator" msgstr "tłumacz" -#: models.py:71 +#: models.py:74 msgid "audiobook" msgstr "audiobook" -#: models.py:72 +#: models.py:75 msgid "audiobooks" msgstr "audiobooki" -#: templates/base.html:7 -msgid "Audiobook repository" -msgstr "Repozytorium audiobooków" - -#: templates/archive/base.html:5 +#: templates/archive/base.html:6 msgid "New" msgstr "Nowe" -#: templates/archive/base.html:6 +#: templates/archive/base.html:7 msgid "Unpublished" msgstr "Nie opublikowane" -#: templates/archive/base.html:7 templates/archive/file_managed.html:11 +#: templates/archive/base.html:8 templates/archive/file_managed.html:12 msgid "Publishing" msgstr "Publikacja" -#: templates/archive/base.html:8 +#: templates/archive/base.html:9 msgid "Published" msgstr "Opublikowane" -#: templates/archive/base.html:9 +#: templates/archive/base.html:10 msgid "Archive" msgstr "Archiwum" -#: templates/archive/base.html:11 +#: templates/archive/base.html:15 msgid "Projects" msgstr "Projekty" -#: templates/archive/base.html:14 +#: templates/archive/base.html:18 msgid "Logout" msgstr "Wyloguj" -#: templates/archive/base.html:16 templates/registration/login.html:9 +#: templates/archive/base.html:20 templates/registration/login.html:9 msgid "Login" msgstr "Zaloguj" -#: templates/archive/base.html:19 +#: templates/archive/base.html:23 msgid "Administration" msgstr "Administracja" -#: templates/archive/file_managed.html:15 +#: templates/archive/file_managed.html:16 msgid "Publishing pending" msgstr "Czeka na publikację" @@ -146,73 +143,92 @@ msgstr "Czeka na publikację" msgid "Cancel publishing" msgstr "Anuluj publikację" -#: templates/archive/file_managed.html:51 +#: templates/archive/file_managed.html:58 msgid "Publish" msgstr "Opublikuj" -#: templates/archive/file_managed.html:57 +#: templates/archive/file_managed.html:67 +#: templates/archive/file_managed.html:79 msgid "Convert without publishing" msgstr "Konwertuj bez publikacji" -#: templates/archive/file_managed.html:67 +#: templates/archive/file_managed.html:74 +msgid "Publish on YouTube" +msgstr "Opublikuj na YouTube" + +#: templates/archive/file_managed.html:95 msgid "MP3 file" msgstr "Plik MP3" -#: templates/archive/file_managed.html:68 +#: templates/archive/file_managed.html:99 msgid "Download MP3 file." msgstr "Pobierz plik MP3." -#: templates/archive/file_managed.html:70 -#: templates/archive/file_managed.html:86 +#: templates/archive/file_managed.html:101 +#: templates/archive/file_managed.html:124 +#: templates/archive/file_managed.html:147 msgid "Published:" msgstr "Opublikowano:" -#: templates/archive/file_managed.html:75 -#: templates/archive/file_managed.html:91 +#: templates/archive/file_managed.html:106 +#: templates/archive/file_managed.html:129 +#: templates/archive/file_managed.html:152 msgid "Not published yet." msgstr "Nie opublikowane." -#: templates/archive/file_managed.html:78 +#: templates/archive/file_managed.html:109 msgid "MP3 file hasn't been generated yet." msgstr "Plik MP3 nie został jeszcze wygenerowany." -#: templates/archive/file_managed.html:83 +#: templates/archive/file_managed.html:118 msgid "Ogg Vorbis file" msgstr "Plik Ogg Vorbis" -#: templates/archive/file_managed.html:84 +#: templates/archive/file_managed.html:122 msgid "Download Ogg Vorbis file." msgstr "Pobierz plik Ogg Vorbis." -#: templates/archive/file_managed.html:94 +#: templates/archive/file_managed.html:132 msgid "Ogg Vorbis file hasn't been generated yet." msgstr "Plik Ogg Vorbis nie został jeszcze wygenerowany." -#: templates/archive/file_managed.html:105 -msgid "Update tags" -msgstr "Uaktualnij tagi" +#: templates/archive/file_managed.html:141 +msgid "YouTube" +msgstr "YouTube" -#: templates/archive/file_managed.html:117 templates/archive/file_new.html:19 -msgid "Commit" -msgstr "Zatwierdź" +#: templates/archive/file_managed.html:145 +msgid "Download YouTube file." +msgstr "Pobierz plik dla YouTube." -#: templates/archive/file_managed.html:128 +#: templates/archive/file_managed.html:155 +msgid "YouTube file hasn't been generated yet." +msgstr "Plik dla YouTube nie został jeszcze wygenerowany." + +#: templates/archive/file_managed.html:174 msgid "Are you sure you want to move this audiobook to archive?" msgstr "Czy na pewno chcesz przenieść ten plik to archiwum?" -#: templates/archive/file_managed.html:130 +#: templates/archive/file_managed.html:176 msgid "Remove to archive" msgstr "Usuń do archiwum" -#: templates/archive/file_new.html:8 +#: templates/archive/file_managed.html:186 +msgid "Update tags" +msgstr "Uaktualnij tagi" + +#: templates/archive/file_managed.html:193 templates/archive/file_new.html:25 +msgid "Commit" +msgstr "Zatwierdź" + +#: templates/archive/file_new.html:13 msgid "Move to archive" msgstr "Przenieś do archiwum" -#: templates/archive/file_unmanaged.html:6 +#: templates/archive/file_unmanaged.html:7 msgid "File with same name already exists!" msgstr "Plik o tej nazwie już istnieje!" -#: templates/archive/file_unmanaged.html:28 +#: templates/archive/file_unmanaged.html:23 msgid "Move to new files" msgstr "Przenieś do nowych plików" @@ -240,6 +256,10 @@ msgstr "Audiobooki archiwalne" msgid "Unpublished audiobooks" msgstr "Nie opublikowane audiobooki" +#: templates/base.html:12 +msgid "Audiobook repository" +msgstr "Repozytorium audiobooków" + #~ msgid "arranger" #~ msgstr "aranżer" diff --git a/src/archive/migrations/0003_auto_20200401_1154.py b/src/archive/migrations/0003_auto_20200401_1154.py new file mode 100644 index 0000000..6556f0d --- /dev/null +++ b/src/archive/migrations/0003_auto_20200401_1154.py @@ -0,0 +1,38 @@ +# Generated by Django 3.0.4 on 2020-04-01 11:54 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('archive', '0002_auto_20200330_1446'), + ] + + operations = [ + migrations.AddField( + model_name='audiobook', + name='youtube_published', + field=models.DateTimeField(editable=False, null=True), + ), + migrations.AddField( + model_name='audiobook', + name='youtube_published_tags', + field=models.TextField(editable=False, null=True), + ), + migrations.AddField( + model_name='audiobook', + name='youtube_status', + field=models.SmallIntegerField(choices=[(1, 'Waiting'), (2, 'Encoding'), (3, 'Tagging'), (4, 'Sending')], editable=False, null=True), + ), + migrations.AddField( + model_name='audiobook', + name='youtube_tags', + field=models.TextField(editable=False, null=True), + ), + migrations.AddField( + model_name='audiobook', + name='youtube_task', + field=models.CharField(editable=False, max_length=64, null=True), + ), + ] diff --git a/src/archive/migrations/0004_audiobook_youtube_file.py b/src/archive/migrations/0004_audiobook_youtube_file.py new file mode 100644 index 0000000..990d968 --- /dev/null +++ b/src/archive/migrations/0004_audiobook_youtube_file.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.4 on 2020-04-01 13:17 + +import archive.utils +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('archive', '0003_auto_20200401_1154'), + ] + + operations = [ + migrations.AddField( + model_name='audiobook', + name='youtube_file', + field=models.FileField(editable=False, null=True, storage=archive.utils.OverwriteStorage(), upload_to='archive/final'), + ), + ] diff --git a/src/archive/migrations/0005_audiobook_youtube_id.py b/src/archive/migrations/0005_audiobook_youtube_id.py new file mode 100644 index 0000000..683a75c --- /dev/null +++ b/src/archive/migrations/0005_audiobook_youtube_id.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.4 on 2020-04-06 11:35 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('archive', '0004_audiobook_youtube_file'), + ] + + operations = [ + migrations.AddField( + model_name='audiobook', + name='youtube_id', + field=models.CharField(blank=True, default='', max_length=255), + ), + ] diff --git a/src/archive/models.py b/src/archive/models.py index f0b751c..746f46b 100644 --- a/src/archive/models.py +++ b/src/archive/models.py @@ -63,6 +63,14 @@ class Audiobook(models.Model): ogg_published_tags = models.TextField(null=True, editable=False) ogg_published = models.DateTimeField(null=True, editable=False) + youtube_status = models.SmallIntegerField(null=True, editable=False, choices=status.choices) + youtube_task = models.CharField(max_length=64, null=True, editable=False) + youtube_tags = models.TextField(null=True, editable=False) + youtube_file = models.FileField(null=True, upload_to='archive/final', storage=OverwriteStorage(), editable=False) + youtube_published_tags = models.TextField(null=True, editable=False) + youtube_published = models.DateTimeField(null=True, editable=False) + youtube_id = models.CharField(max_length=255, blank=True, default='') + class Meta: verbose_name = _("audiobook") verbose_name_plural = _("audiobooks") diff --git a/src/archive/tasks.py b/src/archive/tasks.py index 76f640b..b5675ce 100644 --- a/src/archive/tasks.py +++ b/src/archive/tasks.py @@ -31,7 +31,7 @@ class AudioFormatTask(Task): @classmethod def set_status(cls, aid, status): Audiobook.objects.filter(pk=aid).update( - **{'%s_status' % cls.ext: status}) + **{'%s_status' % cls.prefix: status}) @staticmethod def encode(in_path, out_path): @@ -39,7 +39,7 @@ class AudioFormatTask(Task): @classmethod def set_tags(cls, audiobook, file_name): - tags = getattr(audiobook, "get_%s_tags" % cls.ext)()['tags'] + tags = getattr(audiobook, "get_%s_tags" % cls.prefix)()['tags'] if not tags.get('flac_sha1'): tags['flac_sha1'] = audiobook.get_source_sha1() audio = File(file_name) @@ -49,7 +49,7 @@ class AudioFormatTask(Task): @classmethod def save(cls, audiobook, file_name): - field = "%s_file" % cls.ext + field = "%s_file" % cls.prefix getattr(audiobook, field).save( "%d.%s" % (audiobook.pk, cls.ext), ExistingFile(file_name), @@ -62,16 +62,16 @@ class AudioFormatTask(Task): @classmethod def published(cls, aid): kwargs = { - "%s_published_tags" % cls.ext: F("%s_tags" % cls.ext), - "%s_tags" % cls.ext: None, - "%s_published" % cls.ext: datetime.now(), - '%s_status' % cls.ext: None, + "%s_published_tags" % cls.prefix: F("%s_tags" % cls.prefix), + "%s_tags" % cls.prefix: None, + "%s_published" % cls.prefix: datetime.now(), + '%s_status' % cls.prefix: None, } Audiobook.objects.filter(pk=aid).update(**kwargs) @classmethod def put(cls, user, audiobook, path): - tags = getattr(audiobook, "get_%s_tags" % cls.ext)() + tags = getattr(audiobook, "get_%s_tags" % cls.prefix)() data = { 'book': tags['url'], 'type': cls.ext, @@ -122,7 +122,7 @@ class AudioFormatTask(Task): class Mp3Task(AudioFormatTask): - ext = 'mp3' + prefix = ext = 'mp3' # these shouldn't be staticmethods def id3_text(tag, text): @@ -186,7 +186,7 @@ class Mp3Task(AudioFormatTask): class OggTask(AudioFormatTask): - ext = 'ogg' + prefix = ext = 'ogg' @staticmethod def encode(in_path, out_path): diff --git a/src/archive/templates/archive/file_managed.html b/src/archive/templates/archive/file_managed.html index 25a6b81..f7992e4 100644 --- a/src/archive/templates/archive/file_managed.html +++ b/src/archive/templates/archive/file_managed.html @@ -12,7 +12,7 @@

{% trans "Publishing" %}

- {% if audiobook.mp3_status or audiobook.ogg_status %} + {% if audiobook.mp3_status or audiobook.ogg_status or audiobook.youtube_status %}

{% trans "Publishing pending" %}

{% csrf_token %} @@ -37,30 +37,50 @@

Status: {{ audiobook.get_ogg_status_display }}

{% endif %} + {% if audiobook.youtube_status %} +
+

YouTube

+ +

Status: {{ audiobook.get_youtube_status_display }}

+ {% endif %} + {% else %} {% tags_table audiobook.new_publish_tags 0 %}
- {% if user_can_publish %} - - {% csrf_token %} - - - {% else %} - Podłącz się - {% endif %} - - {% if not audiobook.mp3_published or not audiobook.ogg_published %} -
- {% csrf_token %} - -
- {% endif %} - - - +
+
+ {% if user_can_publish %} +
+ {% csrf_token %} + +
+ {% else %} + Podłącz się + {% endif %} + + {% if not audiobook.mp3_published or not audiobook.ogg_published %} +
+ {% csrf_token %} + +
+ {% endif %} +
+
+
+ {% csrf_token %} + +
+ +
+ {% csrf_token %} + +
+ +
+
{% endif %} @@ -69,7 +89,7 @@
-
+

{% trans "MP3 file" %}

@@ -92,7 +112,7 @@
-
+

{% trans "Ogg Vorbis file" %}

@@ -115,6 +135,29 @@
+
+
+
+

{% trans "YouTube" %}

+
+
+ {% if audiobook.youtube_file %} +

{% trans "Download YouTube file." %}

+ {% if audiobook.youtube_published %} +

{% trans "Published:" %} {{ audiobook.youtube_published }}

+ {% if audiobook.get_youtube_published_tags.tags %} + {% tags_table audiobook.get_youtube_published_tags.tags %} + {% endif %} + {% else %} +

{% trans "Not published yet." %}

+ {% endif %} + {% else %} +

{% trans "YouTube file hasn't been generated yet." %}

+ {% endif %} +
+
+
+
diff --git a/src/archive/urls.py b/src/archive/urls.py index 0deb995..32a6f73 100644 --- a/src/archive/urls.py +++ b/src/archive/urls.py @@ -16,7 +16,7 @@ urlpatterns = [ url(r'^publish/(\d+)/$', views.publish, name="publish"), url(r'^convert/(\d+)/$', views.publish, {'publish': False}, name="convert"), url(r'^download/(\d+)/$', views.download, name="download"), - url(r'^download/(\d+)\.(mp3|ogg)$', views.download, name="download"), + url(r'^download/(\d+)\.(mp3|ogg|mkv)$', views.download, name="download"), url(r'^cancel/(\d+)/$', views.cancel_publishing, name="cancel_publishing"), url(r'^remove_to_archive/(\d+)/$', views.remove_to_archive, name="remove_to_archive"), diff --git a/src/archive/utils.py b/src/archive/utils.py index 476d506..b8598df 100644 --- a/src/archive/utils.py +++ b/src/archive/utils.py @@ -1,12 +1,12 @@ from hashlib import sha1 import os import os.path +import subprocess from django.core.files.storage import FileSystemStorage from django.core.files.uploadedfile import UploadedFile class ExistingFile(UploadedFile): - def __init__(self, path, *args, **kwargs): self.path = path return super(ExistingFile, self).__init__(*args, **kwargs) @@ -19,7 +19,6 @@ class ExistingFile(UploadedFile): class OverwriteStorage(FileSystemStorage): - def _save(self, name, content): if self.exists(name): self.delete(name) @@ -31,7 +30,7 @@ class OverwriteStorage(FileSystemStorage): def sha1_file(f): sha = sha1() - for piece in iter(lambda: f.read(1024*1024), b''): + for piece in iter(lambda: f.read(1024 * 1024), b""): sha.update(piece) return sha.hexdigest() @@ -40,5 +39,4 @@ def all_files(root_path): root_len = len(root_path) for path, dirs, files in os.walk(root_path): for fname in files: - yield os.path.join(path, fname)[root_len:].lstrip('/') - + yield os.path.join(path, fname)[root_len:].lstrip("/") diff --git a/src/archive/views.py b/src/archive/views.py index 8fbcd7e..b39ac72 100644 --- a/src/archive/views.py +++ b/src/archive/views.py @@ -184,15 +184,19 @@ def cancel_publishing(request, aid): # TODO: cancel tasks audiobook.mp3_status = None audiobook.ogg_status = None + audiobook.youtube_status = None audiobook.save() return redirect(file_managed, aid) def download(request, aid, which="source"): - if which not in ("source", "mp3", "ogg"): + if which not in ("source", "mp3", "ogg", 'mkv'): raise Http404 audiobook = get_object_or_404(models.Audiobook, id=aid) - file_ = getattr(audiobook, "%s_file" % which) + field = which + if which == 'mkv': + field = 'youtube' + file_ = getattr(audiobook, "%s_file" % field) if not file_: raise Http404 ext = file_.path.rsplit('.', 1)[-1] @@ -200,7 +204,9 @@ def download(request, aid, which="source"): response['Content-Disposition'] = "attachment; filename*=UTF-8''%s.%s" % ( quote(audiobook.title.encode('utf-8'), safe=''), ext) - response['X-Sendfile'] = file_.path.encode('utf-8') + with open(file_.path, 'rb') as f: + response.write(f.read()) + #response['X-Sendfile'] = file_.path.encode('utf-8') return response diff --git a/src/audiobooks/settings.py b/src/audiobooks/settings.py index cfb9114..ca3d16e 100644 --- a/src/audiobooks/settings.py +++ b/src/audiobooks/settings.py @@ -131,6 +131,7 @@ INSTALLED_APPS = ( 'apiclient', 'archive', + 'youtube', ) # A sample logging configuration. The only tangible logging diff --git a/src/audiobooks/urls.py b/src/audiobooks/urls.py index 5bde7e4..792a50f 100644 --- a/src/audiobooks/urls.py +++ b/src/audiobooks/urls.py @@ -10,6 +10,7 @@ admin.autodiscover() urlpatterns = [ url(r'^$', RedirectView.as_view(url='archive/', permanent=False)), url(r'^archive/', include('archive.urls')), + url(r'^youtube/', include('youtube.urls')), url(r'^publish/', include('apiclient.urls')), url(r'^admin/', admin.site.urls), diff --git a/src/youtube/__init__.py b/src/youtube/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/youtube/admin.py b/src/youtube/admin.py new file mode 100644 index 0000000..e6fe370 --- /dev/null +++ b/src/youtube/admin.py @@ -0,0 +1,5 @@ +from django.contrib import admin +from . import models + + +admin.site.register(models.YouTube) diff --git a/src/youtube/apps.py b/src/youtube/apps.py new file mode 100644 index 0000000..b8743a0 --- /dev/null +++ b/src/youtube/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class YoutubeConfig(AppConfig): + name = 'youtube' diff --git a/src/youtube/locale/pl/LC_MESSAGES/django.mo b/src/youtube/locale/pl/LC_MESSAGES/django.mo new file mode 100644 index 0000000..1e86547 Binary files /dev/null and b/src/youtube/locale/pl/LC_MESSAGES/django.mo differ diff --git a/src/youtube/locale/pl/LC_MESSAGES/django.po b/src/youtube/locale/pl/LC_MESSAGES/django.po new file mode 100644 index 0000000..6446cb2 --- /dev/null +++ b/src/youtube/locale/pl/LC_MESSAGES/django.po @@ -0,0 +1,29 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# FIRST AUTHOR , YEAR. +# +msgid "" +msgstr "" +"Project-Id-Version: \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2020-04-01 14:18+0200\n" +"PO-Revision-Date: 2020-04-01 14:18+0200\n" +"Language: pl\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=4; plural=(n==1 ? 0 : (n%10>=2 && n%10<=4) && (n" +"%100<12 || n%100>14) ? 1 : n!=1 && (n%10>=0 && n%10<=1) || (n%10>=5 && n" +"%10<=9) || (n%100>=12 && n%100<=14) ? 2 : 3);\n" +"Last-Translator: \n" +"Language-Team: \n" +"X-Generator: Poedit 2.2.4\n" + +#: models.py:23 +msgid "YouTube configuration" +msgstr "konfiguracja YouTube" + +#: models.py:24 +msgid "YouTube configurations" +msgstr "konfiguracje YouTube" diff --git a/src/youtube/migrations/0001_initial.py b/src/youtube/migrations/0001_initial.py new file mode 100644 index 0000000..e45ce04 --- /dev/null +++ b/src/youtube/migrations/0001_initial.py @@ -0,0 +1,35 @@ +# Generated by Django 3.0.4 on 2020-03-30 17:58 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='YouTube', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('client_secrets', models.TextField(blank=True)), + ('oauth2_storage', models.TextField(blank=True, editable=False)), + ('title_template', models.CharField(blank=True, max_length=1024)), + ('description_template', models.TextField(blank=True)), + ('category', models.IntegerField(null=True)), + ('intro_card', models.FileField(blank=True, upload_to='')), + ('intro_card_length', models.FloatField(null=True)), + ('card', models.FileField(blank=True, upload_to='')), + ('loop_video', models.FileField(blank=True, upload_to='')), + ('outro_card', models.FileField(blank=True, upload_to='')), + ('outro_card_length', models.FloatField(null=True)), + ], + options={ + 'verbose_name': 'YouTube configuration', + 'verbose_name_plural': 'YouTube configurations', + }, + ), + ] diff --git a/src/youtube/migrations/0002_auto_20200330_1800.py b/src/youtube/migrations/0002_auto_20200330_1800.py new file mode 100644 index 0000000..a79fc82 --- /dev/null +++ b/src/youtube/migrations/0002_auto_20200330_1800.py @@ -0,0 +1,28 @@ +# Generated by Django 3.0.4 on 2020-03-30 18:00 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('youtube', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='youtube', + name='category', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AlterField( + model_name='youtube', + name='intro_card_length', + field=models.FloatField(blank=True, null=True), + ), + migrations.AlterField( + model_name='youtube', + name='outro_card_length', + field=models.FloatField(blank=True, null=True), + ), + ] diff --git a/src/youtube/migrations/0003_auto_20200331_1404.py b/src/youtube/migrations/0003_auto_20200331_1404.py new file mode 100644 index 0000000..adc0032 --- /dev/null +++ b/src/youtube/migrations/0003_auto_20200331_1404.py @@ -0,0 +1,23 @@ +# Generated by Django 3.0.4 on 2020-03-31 14:04 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('youtube', '0002_auto_20200330_1800'), + ] + + operations = [ + migrations.RenameField( + model_name='youtube', + old_name='intro_card_length', + new_name='intro_card_duration', + ), + migrations.RenameField( + model_name='youtube', + old_name='outro_card_length', + new_name='outro_card_duration', + ), + ] diff --git a/src/youtube/migrations/__init__.py b/src/youtube/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/youtube/models.py b/src/youtube/models.py new file mode 100644 index 0000000..423f338 --- /dev/null +++ b/src/youtube/models.py @@ -0,0 +1,123 @@ +from os import unlink +from tempfile import NamedTemporaryFile +from django.db import models +from django.utils.translation import gettext_lazy as _ +from django.template import Template, Context +from apiclient import youtube_call +from .utils import ( + video_from_image, + cut_video, + concat_videos, + get_duration, + get_framerate, + mux, +) + + +class YouTube(models.Model): + title_template = models.CharField(max_length=1024, blank=True) + description_template = models.TextField(blank=True) + category = models.IntegerField(null=True, blank=True) # get categories + intro_card = models.FileField(blank=True) + intro_card_duration = models.FloatField(null=True, blank=True) + card = models.FileField(blank=True) + loop_video = models.FileField(blank=True) + outro_card = models.FileField(blank=True) + outro_card_duration = models.FloatField(null=True, blank=True) + + class Meta: + verbose_name = _("YouTube configuration") + verbose_name_plural = _("YouTube configurations") + + def publish(self, audiobook, path): + ctx = Context(dict(audiobook=audiobook)) + description = Template(self.description_template).render(ctx) + title = Template(self.title_template).render(ctx) + privacy = 'private' + + data = dict( + snippet=dict( + title=title, + description=description, + # tags=tags, + # categoryId=category, + # defaultLanguage + ), + status=dict( + privacyStatus=privacy, + # license + # selfDeclaredMadeForKids + ), + # recordingDetails=dict( + # recordingDate + # ), + ) + part = ",".join(data.keys()) + + with open(path, "rb") as f: + response = youtube_call( + "POST", + "https://www.googleapis.com/upload/youtube/v3/videos", + params={'part': part}, + data=data, + media_data=f.read(), + ) + data = response.json() + audiobook.youtube_id = data['id'] + audiobook.save(update_fields=['youtube_id']) + return response + + def prepare_file(self, input_path, output_path=None): + duration = get_duration(input_path) + video = self.prepare_video(duration) + output = mux([video, input_path], output_path=output_path) + unlink(video) + return output + + def prepare_video(self, duration): + concat = [] + delete = [] + + if self.loop_video: + fps = get_framerate(self.loop_video.path) + else: + fps = 25 + + loop_duration = duration + if self.intro_card and self.intro_card_duration: + loop_duration -= self.intro_card_duration + intro = video_from_image( + self.intro_card.path, self.intro_card_duration, fps=fps + ) + concat.append(intro) + delete.append(intro) + + if self.outro_card and self.outro_card_duration: + loop_duration -= self.outro_card_duration + outro = video_from_image( + self.outro_card.path, self.outro_card_duration, fps=fps + ) + concat.append(outro) + delete.append(outro) + + if self.loop_video: + loop_video_duration = get_duration(self.loop_video.path) + times_loop = int(loop_duration // loop_video_duration) + + leftover_duration = loop_duration % loop_video_duration + leftover = cut_video(self.loop_video.path, leftover_duration) + concat[1:1] = [self.loop_video.path] * times_loop + [leftover] + delete.append(leftover) + else: + leftover = video_from_image(self.card.path, loop_duration) + concat.insert(1, video_from_image(self.card.path, loop_duration, fps=fps)) + delete.append(leftover) + + output = concat_videos(concat) + for p in delete: + unlink(p) + return output + + # tags + # license + # selfDeclaredMadeForKids diff --git a/src/youtube/tasks.py b/src/youtube/tasks.py new file mode 100644 index 0000000..f58df0a --- /dev/null +++ b/src/youtube/tasks.py @@ -0,0 +1,16 @@ +from archive.tasks import AudioFormatTask +from .models import YouTube + + +class YouTubeTask(AudioFormatTask): + ext = 'mkv' + prefix = 'youtube' + + def encode(self, in_path, out_path): + YouTube.objects.first().prepare_file(in_path, out_path) + + def set_tags(self, audiobook, filename): + pass + + def put(self, user, audiobook, filename): + YouTube.objects.first().publish(audiobook, filename) diff --git a/src/youtube/tests.py b/src/youtube/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/src/youtube/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/src/youtube/urls.py b/src/youtube/urls.py new file mode 100644 index 0000000..f170e2b --- /dev/null +++ b/src/youtube/urls.py @@ -0,0 +1,7 @@ +from django.conf.urls import url +from . import views + +urlpatterns = [ + url(r'^publish/(\d+)/$', views.publish, name="youtube_publish"), + url(r'^convert/(\d+)/$', views.publish, {'publish': False}, name="youtube_convert"), +] diff --git a/src/youtube/utils.py b/src/youtube/utils.py new file mode 100644 index 0000000..59d1033 --- /dev/null +++ b/src/youtube/utils.py @@ -0,0 +1,92 @@ +import subprocess +from tempfile import NamedTemporaryFile + + +def video_from_image(img_path, duration, fps=25): + tmp = NamedTemporaryFile(prefix='image', suffix='.mkv', delete=False) + tmp.close() + subprocess.run( + ['ffmpeg', '-y', '-framerate', f'1/{duration}', '-i', img_path, '-c:v', 'libx264', '-vf', f'fps={fps},format=yuv420p', tmp.name], check=True) + return tmp.name + + +def cut_video(video_path, duration): + tmp = NamedTemporaryFile(prefix='cut', suffix='.mkv', delete=False) + tmp.close() + subprocess.run( + ['ffmpeg', '-y', '-i', video_path, '-t', str(duration), tmp.name], check=True) + return tmp.name + + +def concat_videos(paths): + filelist = NamedTemporaryFile(prefix='concat', suffix='.txt') + for path in paths: + filelist.write(f"file '{path}'\n".encode('utf-8')) + filelist.flush() + + output = NamedTemporaryFile(prefix='concat', suffix='.mkv', delete=False) + output.close() + + subprocess.run( + ['ffmpeg', '-y', '-safe', '0', '-f', 'concat', '-i', filelist.name, '-c', 'copy', output.name], + check=True) + + filelist.close() + return output.name + + +def mux(channels, output_path=None): + if not output_path: + output = NamedTemporaryFile(prefix='concat', suffix='.mkv', delete=False) + output.close() + output_path = output.name + args = ['ffmpeg'] + for c in channels: + args.extend(['-i', c]) + args.extend(['-shortest', '-y', output_path]) + subprocess.run(args, check=True) + return output_path + + +def get_duration(path): + return float( + subprocess.run( + [ + "ffprobe", + "-i", + path, + "-show_entries", + "format=duration", + "-v", + "quiet", + "-of", + "csv=p=0", + ], + capture_output=True, + text=True, + check=True, + ).stdout + ) + + +def get_framerate(path): + rates = subprocess.run( + [ + "ffprobe", + "-i", + path, + "-show_entries", + "stream=r_frame_rate", + "-v", + "quiet", + "-of", + "csv=p=0", + ], + capture_output=True, + text=True, + check=True, + ).stdout.strip().split('\n') + for rate in rates: + a, b = rate.split('/') + if b == '1': + return int(a) diff --git a/src/youtube/views.py b/src/youtube/views.py new file mode 100644 index 0000000..1247c50 --- /dev/null +++ b/src/youtube/views.py @@ -0,0 +1,21 @@ +from django.contrib.auth.decorators import permission_required +from django.shortcuts import redirect, get_object_or_404 +from django.urls import reverse +from django.views.decorators.http import require_POST +from archive.constants import status +from archive.models import Audiobook +from . import tasks + + + +@require_POST +@permission_required('archive.change_audiobook') +def publish(request, aid, publish=True): + audiobook = get_object_or_404(Audiobook, id=aid) + tags = {} + #audiobook.set_youtube_tags(tags) + audiobook.youtube_status = status.WAITING + audiobook.save(update_fields=['youtube_status']) + audiobook.youtube_task = tasks.YouTubeTask.delay(request.user.id, aid, publish).task_id + audiobook.save(update_fields=['youtube_task']) + return redirect(reverse('file', args=[aid]))