From: Radek Czajka Date: Thu, 21 May 2020 14:05:39 +0000 (+0200) Subject: Add file cache. X-Git-Url: https://git.mdrn.pl/audio.git/commitdiff_plain/651c3fcf0e96681e5dfeb5afbbc4900a9342beb3 Add file cache. --- diff --git a/src/archive/migrations/0008_remove_audiobook_youtube_file.py b/src/archive/migrations/0008_remove_audiobook_youtube_file.py new file mode 100644 index 0000000..46e7c9c --- /dev/null +++ b/src/archive/migrations/0008_remove_audiobook_youtube_file.py @@ -0,0 +1,17 @@ +# Generated by Django 3.0.4 on 2020-05-21 14:58 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('archive', '0007_project_description'), + ] + + operations = [ + migrations.RemoveField( + model_name='audiobook', + name='youtube_file', + ), + ] diff --git a/src/archive/models.py b/src/archive/models.py index 4a805f2..d76ed99 100644 --- a/src/archive/models.py +++ b/src/archive/models.py @@ -87,7 +87,6 @@ class Audiobook(models.Model): youtube_status = models.SmallIntegerField(null=True, editable=False, choices=status.choices) youtube_task = models.CharField(max_length=64, null=True, editable=False) youtube_tags = models.TextField(null=True, editable=False) - youtube_file = models.FileField(null=True, upload_to='archive/final', storage=OverwriteStorage(), editable=False) youtube_published_tags = models.TextField(null=True, editable=False) youtube_published = models.DateTimeField(null=True, editable=False) youtube_id = models.CharField(max_length=255, blank=True, default='') diff --git a/src/archive/templates/archive/file_managed.html b/src/archive/templates/archive/file_managed.html index 33a6646..adb42c7 100644 --- a/src/archive/templates/archive/file_managed.html +++ b/src/archive/templates/archive/file_managed.html @@ -78,11 +78,14 @@ {% csrf_token %} +
+ {% if audiobook.youtube_id %} +
+ {% csrf_token %} + +
+ {% endif %} -
- {% csrf_token %} - -
@@ -150,10 +153,6 @@

{% trans "See on YouTube" %}

-
- {% csrf_token %} - -
{% endif %} {% if audiobook.youtube_published %}

{% trans "Published:" %} {{ audiobook.youtube_published }}

diff --git a/src/youtube/models.py b/src/youtube/models.py index e6a24ce..f0ecc6c 100644 --- a/src/youtube/models.py +++ b/src/youtube/models.py @@ -13,6 +13,7 @@ from .utils import ( get_duration, get_framerate, mux, + standardize_audio, standardize_video, video_from_image, ) @@ -118,12 +119,19 @@ class YouTube(models.Model): def prepare_audio(self, input_path): files = [] + delete = [] if self.intro_flac: - files.append(self.intro_flac.path) + files.append(standardize_audio(self.intro_flac.path)) + delete.append(files[-1]) files.append(input_path) if self.outro_flac: - files.append(self.outro_flac.path) - return concat_audio(files) + files.append(standardize_audio(self.outro_flac.path)) + delete.append(files[-1]) + output = concat_audio(files) + for d in delete: + unlink(d) + return output + def prepare_video(self, duration): concat = [] diff --git a/src/youtube/tasks.py b/src/youtube/tasks.py index f58df0a..0c3298a 100644 --- a/src/youtube/tasks.py +++ b/src/youtube/tasks.py @@ -12,5 +12,10 @@ class YouTubeTask(AudioFormatTask): def set_tags(self, audiobook, filename): pass + @classmethod + def save(cls, audiobook, file_name): + """We do not save the video files.""" + pass + def put(self, user, audiobook, filename): YouTube.objects.first().publish(audiobook, filename) diff --git a/src/youtube/utils.py b/src/youtube/utils.py index 324609d..5074c53 100644 --- a/src/youtube/utils.py +++ b/src/youtube/utils.py @@ -1,84 +1,107 @@ -from os import unlink +import os +import shutil import subprocess from tempfile import NamedTemporaryFile +from django.conf import settings -def video_from_image(img_path, duration, fps=25): - tmp = NamedTemporaryFile(prefix='image', suffix='.mkv', delete=False) - tmp.close() - subprocess.run( - ['ffmpeg', '-y', '-loop', '1', '-t', str(duration), '-i', img_path, '-c:v', 'libx264', '-vf', f'fps={fps},format=yuv420p', tmp.name], check=True) - return tmp.name +FILE_CACHE = getattr(settings, 'FILE_CACHE', 'file_cache/') + + +def link_or_copy(src, dst): + dstdir = os.path.dirname(dst) + if not os.path.exists(dstdir): + os.makedirs(dstdir) + if os.path.exists(dst): + os.unlink(dst) + # FIXME: tiny window here when the temp path is not taken. + try: + os.link(src, dst) + except OSError: + shutil.copyfile(src, dst) + + +def process_to_file(cmdline, prefix='', suffix='', cache_key=None, output_path=None): + if not output_path: + tmp = NamedTemporaryFile(prefix=prefix, suffix=suffix, delete=False) + tmp.close() + output_path = tmp.name + + if cache_key: + cache_path = FILE_CACHE + cache_key.replace('/', '__') + + if cache_key and os.path.exists(cache_path): + link_or_copy(cache_path, output_path) + else: + # Actually run the processing. + subprocess.run(cmdline + [output_path], check=True) + if cache_key: + link_or_copy(output_path, cache_path) + + return output_path + + +def video_from_image(img_path, duration, fps=25, cache=True): + return process_to_file( + ['ffmpeg', '-y', '-loop', '1', '-t', str(duration), '-i', img_path, '-c:v', 'libx264', '-vf', f'fps={fps},format=yuv420p'], + 'image-', + '.mkv', + f'video_from_image:{img_path}:{duration}:{fps}.mkv' if cache else None + ) def cut_video(video_path, duration): - tmp = NamedTemporaryFile(prefix='cut', suffix='.mkv', delete=False) - tmp.close() - subprocess.run( - ['ffmpeg', '-y', '-i', video_path, '-t', str(duration), tmp.name], check=True) - return tmp.name + return process_to_file( + ['ffmpeg', '-y', '-i', video_path, '-t', str(duration)], + 'cut-', + '.mkv' + ) def ffmpeg_concat(paths, suffix): - filelist = NamedTemporaryFile(prefix='concat', suffix='.txt') + filelist = NamedTemporaryFile(prefix='concat-', suffix='.txt') for path in paths: filelist.write(f"file '{path}'\n".encode('utf-8')) filelist.flush() - output = NamedTemporaryFile(prefix='concat', suffix=suffix, delete=False) - output.close() - - subprocess.run( - ['ffmpeg', '-y', '-safe', '0', '-f', 'concat', '-i', filelist.name, output.name], - check=True) + outname = process_to_file( + ['ffmpeg', '-y', '-safe', '0', '-f', 'concat', '-i', filelist.name], + 'concat-', suffix + ) filelist.close() - return output.name + return outname def concat_videos(paths): return ffmpeg_concat(paths, '.mkv') + def concat_audio(paths): - std_paths = [ - standardize_audio(p) - for p in paths - ] - output = ffmpeg_concat(std_paths, '.flac') - for p in std_paths: - unlink(p) - return output - -def standardize_audio(p): - output = NamedTemporaryFile(prefix='standarize', suffix='.flac', delete=False) - output.close() - subprocess.run( - ['ffmpeg', '-y', '-i', p, '-sample_fmt', 's16', '-acodec', 'flac', '-ac', '2', '-ar', '44100', output.name], - check=True) - return output.name - -def standardize_video(p): - output = NamedTemporaryFile(prefix='standarize', suffix='.mkv', delete=False) - output.close() - subprocess.run( - ['ffmpeg', '-y', '-i', p, output.name], - check=True) - return output.name + return ffmpeg_concat(paths, '.flac') + +def standardize_audio(p, cache=True): + return process_to_file( + ['ffmpeg', '-y', '-i', p, '-sample_fmt', 's16', '-acodec', 'flac', '-ac', '2', '-ar', '44100'], + 'standardize-', '.flac', + f'standardize_audio:{p}.flac' if cache else None + ) + + +def standardize_video(p, cache=True): + return process_to_file( + ['ffmpeg', '-y', '-i', p], + 'standardize-', '.mkv', + f'standardize_video:{p}.mkv' if cache else None + ) def mux(channels, output_path=None): - if not output_path: - output = NamedTemporaryFile(prefix='concat', suffix='.mkv', delete=False) - output.close() - output_path = output.name - args = ['ffmpeg'] + args = ['ffmpeg', '-y'] for c in channels: args.extend(['-i', c]) - args.extend([ - '-y', output_path]) - subprocess.run(args, check=True) - return output_path + return process_to_file(args, 'mux-', '.mkv', output_path=output_path) def get_duration(path):