Add file cache.

diff --git a/src/archive/migrations/0008_remove_audiobook_youtube_file.py b/src/archive/migrations/0008_remove_audiobook_youtube_file.py
new file mode 100644 (file)
index 0000000..46e7c9c
--- /dev/null
+++ b/src/archive/migrations/0008_remove_audiobook_youtube_file.py
@@ -0,0 +1,17 @@
+# Generated by Django 3.0.4 on 2020-05-21 14:58
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('archive', '0007_project_description'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='audiobook',
+            name='youtube_file',
+        ),
+    ]

diff --git a/src/archive/models.py b/src/archive/models.py
index 4a805f2..d76ed99 100644 (file)
--- a/src/archive/models.py
+++ b/src/archive/models.py
@@ -87,7 +87,6 @@ class Audiobook(models.Model):
     youtube_status = models.SmallIntegerField(null=True, editable=False, choices=status.choices)
     youtube_task = models.CharField(max_length=64, null=True, editable=False)
     youtube_tags = models.TextField(null=True, editable=False)
     youtube_status = models.SmallIntegerField(null=True, editable=False, choices=status.choices)
     youtube_task = models.CharField(max_length=64, null=True, editable=False)
     youtube_tags = models.TextField(null=True, editable=False)

-    youtube_file = models.FileField(null=True, upload_to='archive/final', storage=OverwriteStorage(), editable=False)

     youtube_published_tags = models.TextField(null=True, editable=False)
     youtube_published = models.DateTimeField(null=True, editable=False)
     youtube_id = models.CharField(max_length=255, blank=True, default='')
     youtube_published_tags = models.TextField(null=True, editable=False)
     youtube_published = models.DateTimeField(null=True, editable=False)
     youtube_id = models.CharField(max_length=255, blank=True, default='')

diff --git a/src/archive/templates/archive/file_managed.html b/src/archive/templates/archive/file_managed.html
index 33a6646..adb42c7 100644 (file)
--- a/src/archive/templates/archive/file_managed.html
+++ b/src/archive/templates/archive/file_managed.html
@@ -78,11 +78,14 @@
                 {% csrf_token %}
                 <input class="btn btn-primary" type="submit" value="{% trans "Publish on YouTube" %}" />
               </form>
                 {% csrf_token %}
                 <input class="btn btn-primary" type="submit" value="{% trans "Publish on YouTube" %}" />
               </form>

+              <br>
+              {% if audiobook.youtube_id %}
+                <form method="post" action="{% url 'youtube_update' audiobook.id %}">
+                  {% csrf_token %}
+                  <input class="btn btn-secondary" type="submit" value="{% trans "Update YouTube metadata" %}">
+                </form>
+              {% endif %}

 
 

-              <form class="mt-3" method="post" action="{% url 'youtube_convert' audiobook.id %}">
-                {% csrf_token %}
-                <input class="btn btn-secondary" type="submit" value="{% trans "Convert without publishing" %}" />
-              </form>

             </div>
           </div>
         </td></tr>
             </div>
           </div>
         </td></tr>


@@ -150,10 +153,6 @@
             <p>
               <a href="https://youtu.be/{{ audiobook.youtube_id }}" target="_blank">{% trans "See on YouTube" %}</a>
             </p>
             <p>
               <a href="https://youtu.be/{{ audiobook.youtube_id }}" target="_blank">{% trans "See on YouTube" %}</a>
             </p>

-            <form method="post" action="{% url 'youtube_update' audiobook.id %}">
-              {% csrf_token %}
-              <input class="btn btn-secondary" type="submit" value="{% trans "Update YouTube metadata" %}">
-            </form>

           {% endif %}
           {% if audiobook.youtube_published %}
             <p>{% trans "Published:" %} {{ audiobook.youtube_published }}</a></p>
           {% endif %}
           {% if audiobook.youtube_published %}
             <p>{% trans "Published:" %} {{ audiobook.youtube_published }}</a></p>

diff --git a/src/youtube/models.py b/src/youtube/models.py
index e6a24ce..f0ecc6c 100644 (file)
--- a/src/youtube/models.py
+++ b/src/youtube/models.py
@@ -13,6 +13,7 @@ from .utils import (
     get_duration,
     get_framerate,
     mux,
     get_duration,
     get_framerate,
     mux,

+    standardize_audio,

     standardize_video,
     video_from_image,
 )
     standardize_video,
     video_from_image,
 )


@@ -118,12 +119,19 @@ class YouTube(models.Model):
     
     def prepare_audio(self, input_path):
         files = []
     
     def prepare_audio(self, input_path):
         files = []

+        delete = []

         if self.intro_flac:
         if self.intro_flac:

-            files.append(self.intro_flac.path)
+            files.append(standardize_audio(self.intro_flac.path))
+            delete.append(files[-1])

         files.append(input_path)
         if self.outro_flac:
         files.append(input_path)
         if self.outro_flac:

-            files.append(self.outro_flac.path)
-        return concat_audio(files)
+            files.append(standardize_audio(self.outro_flac.path))
+            delete.append(files[-1])
+        output = concat_audio(files)
+        for d in delete:
+            unlink(d)
+        return output
+

     
     def prepare_video(self, duration):
         concat = []
     
     def prepare_video(self, duration):
         concat = []

diff --git a/src/youtube/tasks.py b/src/youtube/tasks.py
index f58df0a..0c3298a 100644 (file)
--- a/src/youtube/tasks.py
+++ b/src/youtube/tasks.py
@@ -12,5 +12,10 @@ class YouTubeTask(AudioFormatTask):
     def set_tags(self, audiobook, filename):
         pass
 
     def set_tags(self, audiobook, filename):
         pass
 

+    @classmethod
+    def save(cls, audiobook, file_name):
+        """We do not save the video files."""
+        pass
+

     def put(self, user, audiobook, filename):
         YouTube.objects.first().publish(audiobook, filename)
     def put(self, user, audiobook, filename):
         YouTube.objects.first().publish(audiobook, filename)

diff --git a/src/youtube/utils.py b/src/youtube/utils.py
index 324609d..5074c53 100644 (file)
--- a/src/youtube/utils.py
+++ b/src/youtube/utils.py
@@ -1,84 +1,107 @@
-from os import unlink
+import os
+import shutil

 import subprocess
 from tempfile import NamedTemporaryFile
 import subprocess
 from tempfile import NamedTemporaryFile

+from django.conf import settings

 
 
 
 

-def video_from_image(img_path, duration, fps=25):
-    tmp = NamedTemporaryFile(prefix='image', suffix='.mkv', delete=False)
-    tmp.close()
-    subprocess.run(
-        ['ffmpeg', '-y', '-loop', '1', '-t', str(duration), '-i', img_path, '-c:v', 'libx264', '-vf', f'fps={fps},format=yuv420p', tmp.name], check=True)
-    return tmp.name
+FILE_CACHE = getattr(settings, 'FILE_CACHE', 'file_cache/')
+
+
+def link_or_copy(src, dst):
+    dstdir = os.path.dirname(dst)
+    if not os.path.exists(dstdir):
+        os.makedirs(dstdir)
+    if os.path.exists(dst):
+        os.unlink(dst)
+        # FIXME: tiny window here when the temp path is not taken.
+    try:
+        os.link(src, dst)
+    except OSError:
+        shutil.copyfile(src, dst)
+
+
+def process_to_file(cmdline, prefix='', suffix='', cache_key=None, output_path=None):
+    if not output_path:
+        tmp = NamedTemporaryFile(prefix=prefix, suffix=suffix, delete=False)
+        tmp.close()
+        output_path = tmp.name
+
+    if cache_key:
+        cache_path = FILE_CACHE + cache_key.replace('/', '__')
+
+    if cache_key and os.path.exists(cache_path):
+        link_or_copy(cache_path, output_path)
+    else:
+        # Actually run the processing.
+        subprocess.run(cmdline + [output_path], check=True)
+        if cache_key:
+            link_or_copy(output_path, cache_path)
+
+    return output_path
+
+
+def video_from_image(img_path, duration, fps=25, cache=True):
+    return process_to_file(
+        ['ffmpeg', '-y', '-loop', '1', '-t', str(duration), '-i', img_path, '-c:v', 'libx264', '-vf', f'fps={fps},format=yuv420p'],
+        'image-',
+        '.mkv',
+        f'video_from_image:{img_path}:{duration}:{fps}.mkv' if cache else None
+    )

 
 
 def cut_video(video_path, duration):
 
 
 def cut_video(video_path, duration):

-    tmp = NamedTemporaryFile(prefix='cut', suffix='.mkv', delete=False)
-    tmp.close()
-    subprocess.run(
-        ['ffmpeg', '-y', '-i', video_path, '-t', str(duration), tmp.name], check=True)
-    return tmp.name
+    return process_to_file(
+        ['ffmpeg', '-y', '-i', video_path, '-t', str(duration)],
+        'cut-',
+        '.mkv'
+    )

 
 
 def ffmpeg_concat(paths, suffix):
 
 
 def ffmpeg_concat(paths, suffix):

-    filelist = NamedTemporaryFile(prefix='concat', suffix='.txt')
+    filelist = NamedTemporaryFile(prefix='concat-', suffix='.txt')

     for path in paths:
         filelist.write(f"file '{path}'\n".encode('utf-8'))
     filelist.flush()
 
     for path in paths:
         filelist.write(f"file '{path}'\n".encode('utf-8'))
     filelist.flush()
 

-    output = NamedTemporaryFile(prefix='concat', suffix=suffix, delete=False)
-    output.close()
-        
-    subprocess.run(
-        ['ffmpeg', '-y', '-safe', '0', '-f', 'concat', '-i', filelist.name, output.name],
-        check=True)
+    outname = process_to_file(
+        ['ffmpeg', '-y', '-safe', '0', '-f', 'concat', '-i', filelist.name],
+        'concat-', suffix
+    )

 
     filelist.close()
 
     filelist.close()

-    return output.name
+    return outname

 
 
 def concat_videos(paths):
     return ffmpeg_concat(paths, '.mkv')
 
 
 
 def concat_videos(paths):
     return ffmpeg_concat(paths, '.mkv')
 

+

 def concat_audio(paths):
 def concat_audio(paths):

-    std_paths = [
-        standardize_audio(p)
-        for p in paths
-    ]
-    output = ffmpeg_concat(std_paths, '.flac')
-    for p in std_paths:
-        unlink(p)
-    return output
-
-def standardize_audio(p):
-    output = NamedTemporaryFile(prefix='standarize', suffix='.flac', delete=False)
-    output.close()
-    subprocess.run(
-        ['ffmpeg', '-y', '-i', p, '-sample_fmt', 's16', '-acodec', 'flac', '-ac', '2', '-ar', '44100', output.name],
-        check=True)
-    return output.name
-
-def standardize_video(p):
-    output = NamedTemporaryFile(prefix='standarize', suffix='.mkv', delete=False)
-    output.close()
-    subprocess.run(
-        ['ffmpeg', '-y', '-i', p, output.name],
-        check=True)
-    return output.name
+    return ffmpeg_concat(paths, '.flac')
+

 
 

+def standardize_audio(p, cache=True):
+    return process_to_file(
+        ['ffmpeg', '-y', '-i', p, '-sample_fmt', 's16', '-acodec', 'flac', '-ac', '2', '-ar', '44100'],
+        'standardize-', '.flac',
+        f'standardize_audio:{p}.flac' if cache else None
+    )
+
+
+def standardize_video(p, cache=True):
+    return process_to_file(
+        ['ffmpeg', '-y', '-i', p],
+        'standardize-', '.mkv',
+        f'standardize_video:{p}.mkv' if cache else None
+    )

 
 
 def mux(channels, output_path=None):
 
 
 def mux(channels, output_path=None):

-    if not output_path:
-        output = NamedTemporaryFile(prefix='concat', suffix='.mkv', delete=False)
-        output.close()
-        output_path = output.name
-    args = ['ffmpeg']
+    args = ['ffmpeg', '-y']

     for c in channels:
         args.extend(['-i', c])
     for c in channels:
         args.extend(['-i', c])

-    args.extend([
-        '-y', output_path])
-    subprocess.run(args, check=True)
-    return output_path
+    return process_to_file(args, 'mux-', '.mkv', output_path=output_path)

 
 
 def get_duration(path):
 
 
 def get_duration(path):