Upgrades

[audio.git] / src / archive / models.py

diff --git a/src/archive/models.py b/src/archive/models.py
index c75a874..1dbb701 100644 (file)
--- a/src/archive/models.py
+++ b/src/archive/models.py
@@ -1,16 +1,29 @@
-# -*- coding: utf-8 -*-
+import io
+import json
+from os import unlink

 import os.path
 import os.path

+from urllib.parse import urljoin

 
 from django.db import models
 from time import sleep
 
 from django.db import models
 from time import sleep

-from jsonfield.fields import JSONField
-from django.utils.encoding import force_bytes
-from django.utils.translation import ugettext_lazy as _
+from django.contrib.sites.models import Site
+from django.utils.functional import cached_property
+from django.utils.timezone import now
+from django.utils.translation import gettext_lazy as _
+from django_pglocks import advisory_lock
+import requests

 from archive.constants import status
 from archive.constants import status

-from archive.settings import FILES_SAVE_PATH, ADVERT, LICENSE, ORGANIZATION, PROJECT
+from archive.settings import FILES_SAVE_PATH, ADVERT, ORGANIZATION, PROJECT

 from archive.utils import OverwriteStorage, sha1_file
 from archive.utils import OverwriteStorage, sha1_file

+from youtube.utils import concat_audio, standardize_audio

 
 

-# Create your models here.
+
+class License(models.Model):
+    uri = models.CharField(max_length=255, unique=True)
+    name = models.CharField(max_length=255)
+
+    def __str__(self):
+        return self.name

 
 
 class Project(models.Model):
 
 
 class Project(models.Model):


@@ -18,15 +31,67 @@ class Project(models.Model):
 
     name = models.CharField(max_length=128, unique=True, db_index=True, verbose_name="Nazwa")
     sponsors = models.TextField(blank=True, null=True, verbose_name="Sponsorzy")
 
     name = models.CharField(max_length=128, unique=True, db_index=True, verbose_name="Nazwa")
     sponsors = models.TextField(blank=True, null=True, verbose_name="Sponsorzy")

+    description = models.TextField(blank=True, verbose_name="Opis")
+    private_notes = models.TextField(blank=True, verbose_name="Prywatne notatki")
+    config = models.ForeignKey('Config', models.PROTECT)
+    can_sell = models.BooleanField(default=True, verbose_name="Do sprzedaży")
+    required_license = models.ForeignKey('License', models.PROTECT, blank=True, null=True, verbose_name='Wymagana licencja')
+    youtube = models.ForeignKey('youtube.YouTube', models.PROTECT)
+    icon = models.FileField(upload_to='archive/project', blank=True, null=True)
+    info_flac = models.FileField(upload_to='archive/info_flac', blank=True)

 
     class Meta:
         verbose_name = _("project")
         verbose_name_plural = _("projects")
         ordering = ("name",)
 
 
     class Meta:
         verbose_name = _("project")
         verbose_name_plural = _("projects")
         ordering = ("name",)
 

-    def __unicode__(self):
+    def __str__(self):
+        return self.name
+
+    def get_description(self):
+        if self.description:
+            return self.description
+        return "Audiobook nagrany w ramach projektu %s%s." % (
+            self.name,
+            " finansowanego przez %s" % self.sponsors if self.sponsors else "",
+        )
+
+    def get_icon_url(self):
+        if not self.icon:
+            return ''
+        return urljoin(
+            'https://' + Site.objects.get_current().domain,
+            self.icon.url
+        )
+
+
+class Config(models.Model):
+    name = models.CharField(max_length=255)
+    intro_flac = models.FileField(upload_to='config/intro_flac', blank=True)
+    intro_min_seconds = models.IntegerField()
+    outro_flac = models.FileField(upload_to='config/outro_flac', blank=True)
+    outro_min_seconds = models.IntegerField()
+
+    class Meta:
+        verbose_name = _("Configuration")
+        verbose_name_plural = _("Configurations")
+
+    def __str__(self):

         return self.name
 
         return self.name
 

+    def prepare_audio(self, audiobook):
+        total_duration = audiobook.total_duration
+        files = []
+        if self.intro_flac and total_duration > self.intro_min_seconds and audiobook.is_first:
+            files.append(standardize_audio(self.intro_flac.path))
+        files.append(standardize_audio(audiobook.source_file.path))
+        if self.outro_flac and total_duration > self.outro_min_seconds and audiobook.is_last:
+            files.append(standardize_audio(self.outro_flac.path))
+        output = concat_audio(files)
+        for d in files:
+            unlink(d)
+        return output
+

 
 def source_upload_to(intance, filename):
     return os.path.join(FILES_SAVE_PATH, filename) # FIXME: what about really long file names?
 
 def source_upload_to(intance, filename):
     return os.path.join(FILES_SAVE_PATH, filename) # FIXME: what about really long file names?


@@ -36,84 +101,143 @@ class Audiobook(models.Model):
     source_file = models.FileField(upload_to=source_upload_to, max_length=255, 
             verbose_name=_('source file'), editable=False)
     source_sha1 = models.CharField(max_length=40, editable=False)
     source_file = models.FileField(upload_to=source_upload_to, max_length=255, 
             verbose_name=_('source file'), editable=False)
     source_sha1 = models.CharField(max_length=40, editable=False)

+    duration = models.FloatField(null=True, editable=False)

 
     title = models.CharField(max_length=255, verbose_name=_('title'))
     part_name = models.CharField(max_length=255, verbose_name=_('part name'), help_text=_('eg. chapter in a novel'),
                                  default='', blank=True)
 
     title = models.CharField(max_length=255, verbose_name=_('title'))
     part_name = models.CharField(max_length=255, verbose_name=_('part name'), help_text=_('eg. chapter in a novel'),
                                  default='', blank=True)

-    index = models.IntegerField(verbose_name=_('index'), default=0)
-    parts_count = models.IntegerField(verbose_name=_('parts count'), default=1)
+    index = models.IntegerField(verbose_name=_('index'), default=0, help_text=_('Ordering of parts of a book.'))
+    youtube_volume = models.CharField(
+        _("Volume name for YouTube"),
+        max_length=100,
+        blank=True,
+        help_text=_(
+            "If set, audiobooks with the save value will be published as single YouTube video."
+        ),
+    )

     artist = models.CharField(max_length=255, verbose_name=_('artist'))
     conductor = models.CharField(max_length=255, verbose_name=_('conductor'))
     encoded_by = models.CharField(max_length=255, verbose_name=_('encoded by'))
     date = models.CharField(max_length=255, verbose_name=_('date'))
     artist = models.CharField(max_length=255, verbose_name=_('artist'))
     conductor = models.CharField(max_length=255, verbose_name=_('conductor'))
     encoded_by = models.CharField(max_length=255, verbose_name=_('encoded by'))
     date = models.CharField(max_length=255, verbose_name=_('date'))

-    project = models.ForeignKey(Project, verbose_name=_('project'))
-    url = models.URLField(max_length=255, verbose_name=_('book url'))
+    project = models.ForeignKey(Project, models.PROTECT, verbose_name=_('project'))
+    slug = models.SlugField(max_length=120, blank=True, help_text=_('WL catalogue slug of the book.'))

     translator = models.CharField(max_length=255, null=True, blank=True, verbose_name=_('translator'))
     modified = models.DateTimeField(null=True, editable=False)
     translator = models.CharField(max_length=255, null=True, blank=True, verbose_name=_('translator'))
     modified = models.DateTimeField(null=True, editable=False)

+    license = models.ForeignKey(License, models.PROTECT, null=True, blank=True, verbose_name=_('license'))
+    license_secondary = models.ForeignKey(License, models.PROTECT, null=True, blank=True, verbose_name=_('secondary license'), related_name='secondary')

 
     # publishing process
     mp3_status = models.SmallIntegerField(null=True, editable=False, choices=status.choices)
 
     # publishing process
     mp3_status = models.SmallIntegerField(null=True, editable=False, choices=status.choices)

-    mp3_task = models.CharField(max_length=64, null=True, editable=False)
-    mp3_tags = JSONField(null=True, editable=False)

     mp3_file = models.FileField(null=True, upload_to='archive/final', storage=OverwriteStorage(), editable=False)
     mp3_file = models.FileField(null=True, upload_to='archive/final', storage=OverwriteStorage(), editable=False)

-    mp3_published_tags = JSONField(null=True, editable=False)

     mp3_published = models.DateTimeField(null=True, editable=False)
 
     ogg_status = models.SmallIntegerField(null=True, editable=False, choices=status.choices)
     mp3_published = models.DateTimeField(null=True, editable=False)
 
     ogg_status = models.SmallIntegerField(null=True, editable=False, choices=status.choices)

-    ogg_task = models.CharField(max_length=64, null=True, editable=False)
-    ogg_tags = JSONField(null=True, editable=False)

     ogg_file = models.FileField(null=True, upload_to='archive/final', storage=OverwriteStorage(), editable=False)
     ogg_file = models.FileField(null=True, upload_to='archive/final', storage=OverwriteStorage(), editable=False)

-    ogg_published_tags = JSONField(null=True, editable=False)

     ogg_published = models.DateTimeField(null=True, editable=False)
 
     ogg_published = models.DateTimeField(null=True, editable=False)
 

+    youtube_status = models.SmallIntegerField(null=True, editable=False, choices=status.choices)
+    youtube_published = models.DateTimeField(null=True, editable=False)
+    youtube_id = models.CharField(max_length=255, blank=True, default='')
+    youtube_queued = models.DateTimeField(null=True, blank=True)

 
     class Meta:
         verbose_name = _("audiobook")
         verbose_name_plural = _("audiobooks")
         ordering = ("title",)
 
 
     class Meta:
         verbose_name = _("audiobook")
         verbose_name_plural = _("audiobooks")
         ordering = ("title",)
 

-    def __unicode__(self):
+    def __str__(self):

         return self.title
 
         return self.title
 

+    @property
+    def url(self):
+        return f'https://wolnelektury.pl/katalog/lektura/{self.slug}/'
+
+    @property
+    def parts_count(self):
+        return type(self).objects.filter(slug=self.slug).count()
+
+    @property
+    def total_duration(self):
+        return type(self).objects.filter(slug=self.slug).aggregate(s=models.Sum('duration'))['s']
+
+    @property
+    def is_first(self):
+        return not type(self).objects.filter(slug=self.slug, index__lte=self.index).exclude(pk=self.pk).exists()
+
+    @property
+    def is_last(self):
+        return not type(self).objects.filter(slug=self.slug, index__gte=self.index).exclude(pk=self.pk).exists()
+    
+    @property
+    def youtube_volume_count(self):
+        total = 0
+        prev_volume = None
+        for a in type(self).objects.filter(slug=self.slug).order_by("index"):
+            if not a.youtube_volume or a.youtube_volume != prev_volume:
+                total += 1
+            prev_volume = a.youtube_volume
+        return total
+
+    @property
+    def youtube_volume_index(self):
+        index = 0
+        prev_volume = None
+        for a in type(self).objects.filter(slug=self.slug, index__lte=self.index).order_by("index"):
+            if not a.youtube_volume or a.youtube_volume != prev_volume:
+                index += 1
+            prev_volume = a.youtube_volume
+        return index
+
+    @property
+    def is_youtube_publishable(self):
+        return (
+            not self.youtube_volume
+            or not type(self)
+            .objects.filter(slug=self.slug, youtube_volume=self.youtube_volume, index__lt=self.index)
+            .exists()
+        )
+
+    def youtube_publish(self):
+        if not self.is_youtube_publishable:
+            return False
+        self.youtube_status = status.QUEUED
+        self.youtube_queued = now()
+        self.save(update_fields=['youtube_status', 'youtube_queued'])
+

     def published(self):
         return self.mp3_published and self.ogg_published
 
     def published(self):
         return self.mp3_published and self.ogg_published
 

-    def get_source_sha1(self):
-        source_sha1 = self.source_sha1
-        if self.pk:
-            source_sha1 = type(self).objects.get(pk=self.pk).source_sha1
-            while source_sha1 == 'wait':
-                sleep(10)
-        if not source_sha1:
-            self.source_sha1 = 'wait'
-            if self.pk:
-                type(self).objects.filter(pk=self.pk).update(source_sha1='wait')
-            try:
-                f = open(force_bytes(self.source_file.path))
-                source_sha1 = sha1_file(f)
-                self.source_sha1 = source_sha1
-                if self.pk:
-                    type(self).objects.filter(pk=self.pk).update(source_sha1=source_sha1)
-            except:
-                self.source_sha1 = ''
-                if self.pk:
-                    type(self).objects.filter(pk=self.pk).update(source_sha1='')
-                return None
-        return source_sha1
+    def publish(self, user, publish=True):
+        from . import tasks
+        self.mp3_status = self.ogg_status = status.WAITING
+        self.save(update_fields=['mp3_status', 'ogg_status'])
+        tasks.Mp3Task.delay(user.id, self.pk, publish=publish).task_id
+        tasks.OggTask.delay(user.id, self.pk, publish=publish).task_id

 
 

+    def get_source_sha1(self):
+        assert self.pk or self.source_sha1
+        if not self.source_sha1:
+            with advisory_lock(f'get_source_sha1_{self.pk}'):
+                with open(self.source_file.path, 'rb') as f:
+                    self.source_sha1 = sha1_file(f)
+                self.save(update_fields=['source_sha1'])
+        return self.source_sha1

 
     def new_publish_tags(self):
         title = self.title
         if self.translator:
 
     def new_publish_tags(self):
         title = self.title
         if self.translator:

-            title += u' (tłum. %s)' % self.translator
+            title += ' (tłum. %s)' % self.translator

 
 

-        copyright = u"%s %s. Licensed to the public under %s verify at %s" % (
-                self.date, ORGANIZATION, LICENSE, self.url)
+        copyright = "%s %s." % (
+                self.date, ORGANIZATION)
+        if self.license:
+            copyright += " Licensed to the public under %s verify at %s" % (
+                self.license.uri, self.url)

 
 

-        comment = u"Audiobook nagrany w ramach projektu %s%s.\n%s" % (
-                    self.project.name,
-                    u" finansowanego przez %s" % self.project.sponsors if self.project.sponsors else "",
-                    ADVERT)
+        comment = "\n".join((
+            self.project.get_description(),
+            ADVERT
+        ))

 
         tags = {
             'album': PROJECT,
 
         tags = {
             'album': PROJECT,


@@ -124,16 +248,44 @@ class Audiobook(models.Model):
             'contact': self.url,
             'copyright': copyright,
             'date': self.date,
             'contact': self.url,
             'copyright': copyright,
             'date': self.date,

-            'genre': u'Speech',
-            'language': u'pol',
-            'license': LICENSE,
+            'genre': 'Speech',
+            'language': 'pol',

             'organization': ORGANIZATION,
             'title': title,
             'organization': ORGANIZATION,
             'title': title,

-            #'flac_sha1': self.get_source_sha1(),

             'project': self.project.name,
             'project': self.project.name,

-            'funded_by': self.project.sponsors,

         }
         }

-        if self.source_sha1 and self.source_sha1 != 'wait':
-            tags['flac_sha1'] = self.source_sha1
+        if self.license:
+            tags['license'] = self.license.uri
+        if self.project.sponsors:
+            tags['funded_by'] = self.project.sponsors
+
+        tags['flac_sha1'] = self.get_source_sha1()
+

         return tags
 
         return tags
 

+    def prepare_audio(self):
+        return self.project.config.prepare_audio(self)
+    
+    @cached_property
+    def book(self):
+        if self.slug:
+            apidata = requests.get(f'https://wolnelektury.pl/api/books/{self.slug}/').json()
+        else:
+            return {}
+        return apidata
+
+    @property
+    def document(self):
+        from librarian.document import WLDocument, parser
+        from lxml import etree
+
+        xml_url = self.book.get('xml', None)
+        if xml_url is None:
+            return None
+
+        return WLDocument(url=xml_url)
+
+    @property
+    def cover(self):
+        from librarian.cover import LogoWLCover
+        return LogoWLCover(self.document.meta).output_file.get_bytes()