From 96f739b685941d100677a374cad55b84c96f180c Mon Sep 17 00:00:00 2001 From: Radek Czajka Date: Fri, 22 May 2020 14:01:34 +0200 Subject: [PATCH 1/1] Use StreamingIterator to avoid loading the whole movie file. --- requirements.txt | 3 ++- src/apiclient/__init__.py | 4 ++-- src/apiclient/models.py | 24 ++++++++++++++---------- src/youtube/models.py | 15 +++++++-------- 4 files changed, 25 insertions(+), 21 deletions(-) diff --git a/requirements.txt b/requirements.txt index 3cb1899..bbb84c3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,6 +10,7 @@ celery[redis]==4.4.0 psycopg2-binary==2.8.4 mutagen==1.42.0 requests==2.23.0 +requests-toolbelt==0.9.1 requests-oauthlib==1.3.0 PyYAML==5.3.1 -Pillow +Pillow==7.1.2 diff --git a/src/apiclient/__init__.py b/src/apiclient/__init__.py index 07b8123..264f030 100644 --- a/src/apiclient/__init__.py +++ b/src/apiclient/__init__.py @@ -33,7 +33,7 @@ def api_call(user, path, method='POST', data=None, files=None): raise ApiError("WL API call error %s, path: %s" % (r.status_code, path)) -def youtube_call(method, url, params=None, json=None, data=None, resumable_data=None): +def youtube_call(method, url, params=None, json=None, data=None, resumable_file_path=None): from .models import YouTubeToken yt = YouTubeToken.objects.first() - return yt.call(method, url, params=params, json=json, data=data, resumable_data=resumable_data) + return yt.call(method, url, params=params, json=json, data=data, resumable_file_path=resumable_file_path) diff --git a/src/apiclient/models.py b/src/apiclient/models.py index 07f804f..cc40143 100644 --- a/src/apiclient/models.py +++ b/src/apiclient/models.py @@ -1,7 +1,9 @@ import json +import os from django.db import models from django.contrib.auth.models import User from requests_oauthlib import OAuth2Session +from requests_toolbelt.streaming_iterator import StreamingIterator from .settings import YOUTUBE_CLIENT_ID, YOUTUBE_CLIENT_SECRET, YOUTUBE_TOKEN_URL @@ -37,10 +39,11 @@ class YouTubeToken(models.Model): token_updater=self.token_updater ) - def call(self, method, url, params=None, json=None, data=None, resumable_data=None): + def call(self, method, url, params=None, json=None, data=None, resumable_file_path=None): params = params or {} - if resumable_data: + if resumable_file_path: params['uploadType'] = 'resumable' + file_size = os.stat(resumable_file_path).st_size session = self.get_session() response = session.request( @@ -50,14 +53,15 @@ class YouTubeToken(models.Model): data=data, params=params, headers={ - 'X-Upload-Content-Length': str(len(resumable_data)), + 'X-Upload-Content-Length': str(file_size), 'x-upload-content-type': 'application/octet-stream', - } if resumable_data else {} + } if resumable_file_path else {} ) - if resumable_data: + if resumable_file_path: location = response.headers['Location'] - return session.put( - url=location, - data=resumable_data, - headers={"Content-Type": "application/octet-stream"}, - ) + with open(resumable_file_path, 'rb') as f: + return session.put( + url=location, + data=StreamingIterator(file_size, f), + headers={"Content-Type": "application/octet-stream"}, + ) diff --git a/src/youtube/models.py b/src/youtube/models.py index eb669e0..027befd 100644 --- a/src/youtube/models.py +++ b/src/youtube/models.py @@ -74,14 +74,13 @@ class YouTube(models.Model): data = self.get_data(audiobook) part = ",".join(data.keys()) - with open(path, "rb") as f: - response = youtube_call( - "POST", - "https://www.googleapis.com/upload/youtube/v3/videos", - params={'part': part}, - json=data, - resumable_data=f.read(), - ) + response = youtube_call( + "POST", + "https://www.googleapis.com/upload/youtube/v3/videos", + params={'part': part}, + json=data, + resumable_file_path=path, + ) data = response.json() audiobook.youtube_id = data['id'] audiobook.save(update_fields=['youtube_id']) -- 2.20.1