From: Radek Czajka <rczajka@rczajka.pl>
Date: Fri, 22 May 2020 12:01:34 +0000 (+0200)
Subject: Use StreamingIterator to avoid loading the whole movie file.
X-Git-Url: https://git.mdrn.pl/audio.git/commitdiff_plain/96f739b685941d100677a374cad55b84c96f180c

Use StreamingIterator to avoid loading the whole movie file.
---

diff --git a/requirements.txt b/requirements.txt
index 3cb1899..bbb84c3 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -10,6 +10,7 @@ celery[redis]==4.4.0
 psycopg2-binary==2.8.4
 mutagen==1.42.0
 requests==2.23.0
+requests-toolbelt==0.9.1
 requests-oauthlib==1.3.0
 PyYAML==5.3.1
-Pillow
+Pillow==7.1.2
diff --git a/src/apiclient/__init__.py b/src/apiclient/__init__.py
index 07b8123..264f030 100644
--- a/src/apiclient/__init__.py
+++ b/src/apiclient/__init__.py
@@ -33,7 +33,7 @@ def api_call(user, path, method='POST', data=None, files=None):
         raise ApiError("WL API call error %s, path: %s" % (r.status_code, path))
 
 
-def youtube_call(method, url, params=None, json=None, data=None, resumable_data=None):
+def youtube_call(method, url, params=None, json=None, data=None, resumable_file_path=None):
     from .models import YouTubeToken
     yt = YouTubeToken.objects.first()
-    return yt.call(method, url, params=params, json=json, data=data, resumable_data=resumable_data)
+    return yt.call(method, url, params=params, json=json, data=data, resumable_file_path=resumable_file_path)
diff --git a/src/apiclient/models.py b/src/apiclient/models.py
index 07f804f..cc40143 100644
--- a/src/apiclient/models.py
+++ b/src/apiclient/models.py
@@ -1,7 +1,9 @@
 import json
+import os
 from django.db import models
 from django.contrib.auth.models import User
 from requests_oauthlib import OAuth2Session
+from requests_toolbelt.streaming_iterator import StreamingIterator
 from .settings import YOUTUBE_CLIENT_ID, YOUTUBE_CLIENT_SECRET, YOUTUBE_TOKEN_URL
 
 
@@ -37,10 +39,11 @@ class YouTubeToken(models.Model):
             token_updater=self.token_updater
         )
 
-    def call(self, method, url, params=None, json=None, data=None, resumable_data=None):
+    def call(self, method, url, params=None, json=None, data=None, resumable_file_path=None):
         params = params or {}
-        if resumable_data:
+        if resumable_file_path:
             params['uploadType'] = 'resumable'
+            file_size = os.stat(resumable_file_path).st_size
 
         session = self.get_session()
         response = session.request(
@@ -50,14 +53,15 @@ class YouTubeToken(models.Model):
             data=data,
             params=params,
             headers={
-                'X-Upload-Content-Length': str(len(resumable_data)),
+                'X-Upload-Content-Length': str(file_size),
                 'x-upload-content-type': 'application/octet-stream',
-            } if resumable_data else {}
+            } if resumable_file_path else {}
         )
-        if resumable_data:
+        if resumable_file_path:
             location = response.headers['Location']
-            return session.put(
-                url=location,
-                data=resumable_data,
-                headers={"Content-Type": "application/octet-stream"},
-            )
+            with open(resumable_file_path, 'rb') as f:
+                return session.put(
+                    url=location,
+                    data=StreamingIterator(file_size, f),
+                    headers={"Content-Type": "application/octet-stream"},
+                )
diff --git a/src/youtube/models.py b/src/youtube/models.py
index eb669e0..027befd 100644
--- a/src/youtube/models.py
+++ b/src/youtube/models.py
@@ -74,14 +74,13 @@ class YouTube(models.Model):
         data = self.get_data(audiobook)
         part = ",".join(data.keys())
 
-        with open(path, "rb") as f:
-            response = youtube_call(
-                "POST",
-                "https://www.googleapis.com/upload/youtube/v3/videos",
-                params={'part': part},
-                json=data,
-                resumable_data=f.read(),
-            )
+        response = youtube_call(
+            "POST",
+            "https://www.googleapis.com/upload/youtube/v3/videos",
+            params={'part': part},
+            json=data,
+            resumable_file_path=path,
+        )
         data = response.json()
         audiobook.youtube_id = data['id']
         audiobook.save(update_fields=['youtube_id'])