Editable media inserts.
authorRadek Czajka <rczajka@rczajka.pl>
Wed, 2 Aug 2023 21:07:41 +0000 (23:07 +0200)
committerRadek Czajka <rczajka@rczajka.pl>
Wed, 2 Aug 2023 21:07:41 +0000 (23:07 +0200)
src/annoy/admin.py
src/annoy/migrations/0013_mediainsertset_mediainserttext.py [new file with mode: 0644]
src/annoy/migrations/0014_fundraising.py [new file with mode: 0644]
src/annoy/models.py
src/catalogue/fields.py
src/catalogue/management/commands/build_stale.py [new file with mode: 0644]
src/catalogue/management/commands/schedule_stale_ebooks.py [deleted file]
src/reporting/templates/reporting/main.html
src/reporting/views.py

index 8470a12..ab7be6a 100644 (file)
@@ -1,5 +1,6 @@
 from django.contrib import admin
 from django import forms
 from django.contrib import admin
 from django import forms
+from admin_ordering.admin import OrderableAdmin
 from modeltranslation.admin import TranslationAdmin
 from . import models
 
 from modeltranslation.admin import TranslationAdmin
 from . import models
 
@@ -40,3 +41,16 @@ class DynamicTextInsertAdmin(admin.ModelAdmin):
 
 
 admin.site.register(models.DynamicTextInsert, DynamicTextInsertAdmin)
 
 
 admin.site.register(models.DynamicTextInsert, DynamicTextInsertAdmin)
+
+
+class MediaInsertTextInline(OrderableAdmin, admin.TabularInline):
+    model = models.MediaInsertText
+    extra = 0
+    min_num = 1
+
+
+@admin.register(models.MediaInsertSet)
+class MediaInsertSetAdmin(admin.ModelAdmin):
+    list_display = ['file_format', 'etag']
+    inlines = [MediaInsertTextInline]
+    readonly_fields = ['etag']
diff --git a/src/annoy/migrations/0013_mediainsertset_mediainserttext.py b/src/annoy/migrations/0013_mediainsertset_mediainserttext.py
new file mode 100644 (file)
index 0000000..0c3c273
--- /dev/null
@@ -0,0 +1,34 @@
+# Generated by Django 4.0.8 on 2023-07-21 15:12
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('annoy', '0012_banner_only_authenticated'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='MediaInsertSet',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('file_format', models.CharField(choices=[('epub', 'epub'), ('mobi', 'mobi')], max_length=8)),
+                ('etag', models.CharField(blank=True, max_length=64)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='MediaInsertText',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('ordering', models.IntegerField()),
+                ('text', models.TextField()),
+                ('media_insert_set', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='annoy.mediainsertset')),
+            ],
+            options={
+                'ordering': ('ordering',),
+            },
+        ),
+    ]
diff --git a/src/annoy/migrations/0014_fundraising.py b/src/annoy/migrations/0014_fundraising.py
new file mode 100644 (file)
index 0000000..8dc1119
--- /dev/null
@@ -0,0 +1,31 @@
+# Generated by Django 4.0.8 on 2023-08-02 19:58
+import hashlib
+import json
+from django.conf import settings
+from django.db import migrations
+
+
+def migrate_fundraising(apps, schema_editor):
+    MIS = apps.get_model('annoy', 'MediaInsertSet')
+    EF = getattr(settings, 'EPUB_FUNDRAISING', None)
+    if not EF:
+        return
+    etag = hashlib.sha1(json.dumps(EF).encode('utf-8')).hexdigest()
+    for f in ('epub', 'mobi'):
+        mis = MIS.objects.create(file_format=f, etag=etag)
+        for i, t in enumerate(EF):
+            mis.mediainserttext_set.create(ordering=i+1, text=t)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('annoy', '0013_mediainsertset_mediainserttext'),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            migrate_fundraising,
+            migrations.RunPython.noop
+        )
+    ]
index c8fdf2d..61cb94c 100644 (file)
@@ -1,3 +1,5 @@
+import hashlib
+import json
 from django.apps import apps
 from django.conf import settings
 from django.db import models
 from django.apps import apps
 from django.conf import settings
 from django.db import models
@@ -109,3 +111,47 @@ class DynamicTextInsertText(models.Model):
     text_color = models.CharField(max_length=10, blank=True)
     text = models.TextField(_('text'))
     image = models.FileField(blank=True, upload_to='annoy/inserts/')
     text_color = models.CharField(max_length=10, blank=True)
     text = models.TextField(_('text'))
     image = models.FileField(blank=True, upload_to='annoy/inserts/')
+
+
+class MediaInsertSet(models.Model):
+    file_format = models.CharField(max_length=8, choices=[
+        ('epub', 'epub'),
+        ('mobi', 'mobi'),
+        ])
+    etag = models.CharField(max_length=64, blank=True)
+
+    def update_etag(self):
+        self.etag = hashlib.sha1(json.dumps(self.get_texts()).encode('utf-8')).hexdigest()
+        self.save(update_fields=['etag'])
+
+    def get_texts(self):
+        return [t.text for t in self.mediainserttext_set.all()]
+
+    @classmethod
+    def get_for_format(cls, file_format):
+        return cls.objects.filter(file_format=file_format).first()
+
+    @classmethod
+    def get_texts_for(cls, file_format):
+        self = cls.get_for_format(file_format)
+        if self is None:
+            return []
+        return self.get_texts()
+
+
+class MediaInsertText(models.Model):
+    media_insert_set = models.ForeignKey(MediaInsertSet, models.CASCADE)
+    ordering = models.IntegerField()
+    text = models.TextField()
+
+    class Meta:
+        ordering = ('ordering',)
+
+
+from django.db.models.signals import post_save, post_delete
+from django.dispatch import receiver
+
+@receiver(post_delete, sender=MediaInsertText)
+@receiver(post_save, sender=MediaInsertText)
+def update_etag(sender, instance, **kwargs):
+    instance.media_insert_set.update_etag()
index 6ae4da3..bf17d35 100644 (file)
@@ -2,13 +2,16 @@
 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
 #
 import os
 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
 #
 import os
+import pkg_resources
+import random
+from django.apps import apps
 from django.conf import settings
 from django.core.files import File
 from django.db import models
 from django.db.models.fields.files import FieldFile
 from django.utils.deconstruct import deconstructible
 from django.utils.translation import gettext_lazy as _
 from django.conf import settings
 from django.core.files import File
 from django.db import models
 from django.db.models.fields.files import FieldFile
 from django.utils.deconstruct import deconstructible
 from django.utils.translation import gettext_lazy as _
-from catalogue.constants import LANGUAGES_3TO2, EBOOK_FORMATS_WITH_CHILDREN, EBOOK_FORMATS_WITHOUT_CHILDREN
+from catalogue.constants import LANGUAGES_3TO2
 from catalogue.utils import absolute_url, remove_zip, truncate_html_words, gallery_path, gallery_url
 from waiter.utils import clear_cache
 
 from catalogue.utils import absolute_url, remove_zip, truncate_html_words, gallery_path, gallery_url
 from waiter.utils import clear_cache
 
@@ -66,6 +69,7 @@ class EbookField(models.FileField):
     """Represents an ebook file field, attachable to a model."""
     attr_class = EbookFieldFile
     ext = None
     """Represents an ebook file field, attachable to a model."""
     attr_class = EbookFieldFile
     ext = None
+    for_parents = True
     librarian2_api = False
     ZIP = None
 
     librarian2_api = False
     ZIP = None
 
@@ -124,39 +128,49 @@ class EbookField(models.FileField):
         setattr(cls, 'has_%s' % self.attname, has)
 
     def get_current_etag(self):
         setattr(cls, 'has_%s' % self.attname, has)
 
     def get_current_etag(self):
-        import pkg_resources
+        MediaInsertSet = apps.get_model('annoy', 'MediaInsertSet')
         librarian_version = pkg_resources.get_distribution("librarian").version
         librarian_version = pkg_resources.get_distribution("librarian").version
-        return librarian_version
-
-    def schedule_stale(self, queryset=None):
-        """Schedule building this format for all the books where etag is stale."""
+        etag = librarian_version
+        mis = MediaInsertSet.get_for_format(self.ext)
+        if mis is not None:
+            etag += '_' + mis.etag
+        return etag
+
+    def find_stale(self, limit):
+        """Find some books where this format is stale."""
         # If there is not ETag field, bail. That's true for xml file field.
         if not self.with_etag:
         # If there is not ETag field, bail. That's true for xml file field.
         if not self.with_etag:
-            return
+            return []
 
         etag = self.get_current_etag()
 
         etag = self.get_current_etag()
-        if queryset is None:
-            queryset = self.model.objects.all()
 
 
-        if self.format_name in EBOOK_FORMATS_WITHOUT_CHILDREN + ['html']:
+        queryset = self.model.objects.all()
+        if not self.for_parents:
             queryset = queryset.filter(children=None)
 
         queryset = queryset.exclude(**{
             f'{self.etag_field_name}__in': [
                 etag, f'{etag}{ETAG_SCHEDULED_SUFFIX}'
             queryset = queryset.filter(children=None)
 
         queryset = queryset.exclude(**{
             f'{self.etag_field_name}__in': [
                 etag, f'{etag}{ETAG_SCHEDULED_SUFFIX}'
-            ]
+           ]
         })
         })
-        for obj in queryset:
-            fieldfile = getattr(obj, self.attname)
-            priority = EBOOK_REBUILD_PRIORITY if fieldfile else EBOOK_BUILD_PRIORITY
-            fieldfile.build_delay(priority=priority)
+
+        queryset = queryset.order_by('?')[:limit]
+        return queryset
 
     @classmethod
 
     @classmethod
-    def schedule_all_stale(cls, model):
+    def find_all_stale(cls, model, limit):
         """Schedules all stale ebooks of all formats to rebuild."""
         """Schedules all stale ebooks of all formats to rebuild."""
+        found = []
         for field in model._meta.fields:
             if isinstance(field, cls):
         for field in model._meta.fields:
             if isinstance(field, cls):
-                field.schedule_stale()
+                for instance in field.find_stale(limit):
+                    found.append((
+                        field.name,
+                        instance
+                    ))
+        random.shuffle(found)
+        found = found[:limit]
+        return found
 
     @staticmethod
     def transform(wldoc):
 
     @staticmethod
     def transform(wldoc):
@@ -191,6 +205,7 @@ class XmlField(EbookField):
 
 class TxtField(EbookField):
     ext = 'txt'
 
 class TxtField(EbookField):
     ext = 'txt'
+    for_parents = False
 
     @staticmethod
     def transform(wldoc):
 
     @staticmethod
     def transform(wldoc):
@@ -199,6 +214,7 @@ class TxtField(EbookField):
 
 class Fb2Field(EbookField):
     ext = 'fb2'
 
 class Fb2Field(EbookField):
     ext = 'fb2'
+    for_parents = False
     ZIP = 'wolnelektury_pl_fb2'
 
     @staticmethod
     ZIP = 'wolnelektury_pl_fb2'
 
     @staticmethod
@@ -229,9 +245,10 @@ class EpubField(EbookField):
     @staticmethod
     def transform(wldoc):
         from librarian.builders import EpubBuilder
     @staticmethod
     def transform(wldoc):
         from librarian.builders import EpubBuilder
+        MediaInsertSet = apps.get_model('annoy', 'MediaInsertSet')
         return EpubBuilder(
                 base_url='file://' + os.path.abspath(gallery_path(wldoc.meta.url.slug)) + '/',
         return EpubBuilder(
                 base_url='file://' + os.path.abspath(gallery_path(wldoc.meta.url.slug)) + '/',
-                fundraising=settings.EPUB_FUNDRAISING
+                fundraising=MediaInsertSet.get_texts_for('epub')
             ).build(wldoc)
 
 
             ).build(wldoc)
 
 
@@ -243,14 +260,16 @@ class MobiField(EbookField):
     @staticmethod
     def transform(wldoc):
         from librarian.builders import MobiBuilder
     @staticmethod
     def transform(wldoc):
         from librarian.builders import MobiBuilder
+        MediaInsertSet = apps.get_model('annoy', 'MediaInsertSet')
         return MobiBuilder(
                 base_url='file://' + os.path.abspath(gallery_path(wldoc.meta.url.slug)) + '/',
         return MobiBuilder(
                 base_url='file://' + os.path.abspath(gallery_path(wldoc.meta.url.slug)) + '/',
-                fundraising=settings.EPUB_FUNDRAISING
+                fundraising=MediaInsertSet.get_texts_for('mobi')
             ).build(wldoc)
 
 
 class HtmlField(EbookField):
     ext = 'html'
             ).build(wldoc)
 
 
 class HtmlField(EbookField):
     ext = 'html'
+    for_parents = False
 
     def build(self, fieldfile):
         from django.core.files.base import ContentFile
 
     def build(self, fieldfile):
         from django.core.files.base import ContentFile
diff --git a/src/catalogue/management/commands/build_stale.py b/src/catalogue/management/commands/build_stale.py
new file mode 100644 (file)
index 0000000..b03e3e8
--- /dev/null
@@ -0,0 +1,41 @@
+# This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
+# Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
+#
+from time import time
+from django.conf import settings
+from django.core.management.base import BaseCommand
+
+from catalogue.fields import EbookField
+from catalogue.models import Book
+
+
+class Command(BaseCommand):
+    help = 'Schedule regenerating stale ebook files.'
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '-l', '--limit', type=int,
+            help='Limit number of files to build'
+        )
+        parser.add_argument(
+            '-t', '--time', type=int, metavar='SECONDS',
+            help='Limit timenumber of files to build'
+        )
+
+    def handle(self, **options):
+        t = time()
+        counter = 0
+        while True:
+            if options['time'] is not None and time() - t > options['time']:
+                break
+            if options['limit'] is not None and counter >= options['limit']:
+                break
+            tasks = EbookField.find_all_stale(Book, 1)
+            if not tasks:
+                break
+            for field_name, book in tasks:
+                print(field_name, book)
+                try:
+                    getattr(book, field_name).build()
+                except Exception as e:
+                    print('ERROR', e)
diff --git a/src/catalogue/management/commands/schedule_stale_ebooks.py b/src/catalogue/management/commands/schedule_stale_ebooks.py
deleted file mode 100644 (file)
index 4420a81..0000000
+++ /dev/null
@@ -1,15 +0,0 @@
-# This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
-# Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
-#
-from django.conf import settings
-from django.core.management.base import BaseCommand
-
-from catalogue.fields import EbookField
-from catalogue.models import Book
-
-
-class Command(BaseCommand):
-    help = 'Schedule regenerating stale ebook files.'
-
-    def handle(self, **options):
-        EbookField.schedule_all_stale(Book)
index a69e9d1..ece5d02 100644 (file)
@@ -1,5 +1,5 @@
-{% extends request.EXPERIMENTS.layout.value|yesno:"2022/base_simple.html,base/base.html" %}
-{% load i18n %}
+{% extends "2022/base_simple.html" %}
+{% load i18n l10n %}
 {% load reporting_stats catalogue_tags %}
 
 {% block settings %}
 {% load reporting_stats catalogue_tags %}
 
 {% block settings %}
         <li>{{ license }} ({{ license_description }})</li>
       {% endfor %}
     </ul>
         <li>{{ license }} ({{ license_description }})</li>
       {% endfor %}
     </ul>
+
+    {% localize off %}
+    <h3>Aktualność plików</h3>
+    <table border="1">
+      <tr>
+        {% for e in etags %}
+          <td>
+            <div
+                style="width: 80px; overflow: hidden; text-overflow: ellipsis; padding: 2px;"
+                title="{{ e.field }}"
+            >
+              {{ e.field }}
+            </div>
+          </td>
+        {% endfor %}
+      </tr>
+      <tr>
+        {% for e in etags %}
+          <td>
+            <div style="width: 80px; overflow: hidden; text-overflow: ellipsis; padding: 2px;">
+              {{ e.etag }}
+            </div>
+          </td>
+        {% endfor %}
+      </tr>
+      <tr>
+        {% for e in etags %}
+          <td style="vertical-align: top">
+            {% for et in e.tags %}
+              <div
+                  style="position: relative; width:80px; overflow:hidden; text-overflow: ellipsis; padding: 2px;"
+                  title="{{ et.perc }}%"
+              >
+                <span style="z-index: 0; background: {% if et.tag == e.etag %}#92BD39{% elif et.tag == '' %}#FF4C54{% else %}#FFA500{% endif %}; position: absolute;left:0; top:0; bottom: 0; width: {{ et.perc }}%;"></span>
+                <span style="position: relative; z-index: 1">
+                  {{ et.count }} {{ et.tag }}
+                </span>
+              </div>
+            {% endfor %}
+          </td>
+        {% endfor %}
+      </tr>
+    </table>
+    {% endlocalize %}
+
   </div>
 {% endblock %}
   </div>
 {% endblock %}
index 9eecf0d..74e24eb 100644 (file)
@@ -30,9 +30,40 @@ def stats_page(request):
         if extra_info.get('license'):
             licenses.add((extra_info.get('license'), extra_info.get('license_description')))
 
         if extra_info.get('license'):
             licenses.add((extra_info.get('license'), extra_info.get('license_description')))
 
+    etags = []
+    all_books = Book.objects.all()
+    all_books_count = all_books.count()
+    noparent_books = Book.objects.filter(children=None)
+    noparent_books_count = noparent_books.count()
+    for field in Book._meta.fields:
+        if not getattr(field, 'with_etag', None): continue
+        etag = field.get_current_etag()
+        d = {
+            'field': field.name,
+            'etag': etag,
+        }
+        if field.for_parents:
+            books = all_books
+            n_books = all_books_count
+        else:
+            books = noparent_books
+            n_books = noparent_books_count
+        tags = books.values_list(field.etag_field.name).order_by(
+            '-' + field.etag_field.name).distinct().annotate(c=Count('*'))
+        d['tags'] = [
+            {
+                'tag': t[0],
+                'count': t[1],
+                'perc': round(100 * t[1] / n_books, 2)
+            }
+            for t in tags
+        ]
+        etags.append(d)
+
     return render(request, 'reporting/main.html', {
         'media_types': media_types,
         'licenses': licenses,
     return render(request, 'reporting/main.html', {
         'media_types': media_types,
         'licenses': licenses,
+        'etags': etags,
     })
 
 
     })