Editable media inserts.

diff --git a/src/annoy/admin.py b/src/annoy/admin.py
index 8470a12..ab7be6a 100644 (file)
--- a/src/annoy/admin.py
+++ b/src/annoy/admin.py
@@ -1,5 +1,6 @@
 from django.contrib import admin
 from django import forms
 from django.contrib import admin
 from django import forms

+from admin_ordering.admin import OrderableAdmin

 from modeltranslation.admin import TranslationAdmin
 from . import models
 
 from modeltranslation.admin import TranslationAdmin
 from . import models
 


@@ -40,3 +41,16 @@ class DynamicTextInsertAdmin(admin.ModelAdmin):
 
 
 admin.site.register(models.DynamicTextInsert, DynamicTextInsertAdmin)
 
 
 admin.site.register(models.DynamicTextInsert, DynamicTextInsertAdmin)

+
+
+class MediaInsertTextInline(OrderableAdmin, admin.TabularInline):
+    model = models.MediaInsertText
+    extra = 0
+    min_num = 1
+
+
+@admin.register(models.MediaInsertSet)
+class MediaInsertSetAdmin(admin.ModelAdmin):
+    list_display = ['file_format', 'etag']
+    inlines = [MediaInsertTextInline]
+    readonly_fields = ['etag']

diff --git a/src/annoy/migrations/0013_mediainsertset_mediainserttext.py b/src/annoy/migrations/0013_mediainsertset_mediainserttext.py
new file mode 100644 (file)
index 0000000..0c3c273
--- /dev/null
+++ b/src/annoy/migrations/0013_mediainsertset_mediainserttext.py
@@ -0,0 +1,34 @@
+# Generated by Django 4.0.8 on 2023-07-21 15:12
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('annoy', '0012_banner_only_authenticated'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='MediaInsertSet',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('file_format', models.CharField(choices=[('epub', 'epub'), ('mobi', 'mobi')], max_length=8)),
+                ('etag', models.CharField(blank=True, max_length=64)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='MediaInsertText',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('ordering', models.IntegerField()),
+                ('text', models.TextField()),
+                ('media_insert_set', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='annoy.mediainsertset')),
+            ],
+            options={
+                'ordering': ('ordering',),
+            },
+        ),
+    ]

diff --git a/src/annoy/migrations/0014_fundraising.py b/src/annoy/migrations/0014_fundraising.py
new file mode 100644 (file)
index 0000000..8dc1119
--- /dev/null
+++ b/src/annoy/migrations/0014_fundraising.py
@@ -0,0 +1,31 @@
+# Generated by Django 4.0.8 on 2023-08-02 19:58
+import hashlib
+import json
+from django.conf import settings
+from django.db import migrations
+
+
+def migrate_fundraising(apps, schema_editor):
+    MIS = apps.get_model('annoy', 'MediaInsertSet')
+    EF = getattr(settings, 'EPUB_FUNDRAISING', None)
+    if not EF:
+        return
+    etag = hashlib.sha1(json.dumps(EF).encode('utf-8')).hexdigest()
+    for f in ('epub', 'mobi'):
+        mis = MIS.objects.create(file_format=f, etag=etag)
+        for i, t in enumerate(EF):
+            mis.mediainserttext_set.create(ordering=i+1, text=t)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('annoy', '0013_mediainsertset_mediainserttext'),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            migrate_fundraising,
+            migrations.RunPython.noop
+        )
+    ]

diff --git a/src/annoy/models.py b/src/annoy/models.py
index c8fdf2d..61cb94c 100644 (file)
--- a/src/annoy/models.py
+++ b/src/annoy/models.py
@@ -1,3 +1,5 @@
+import hashlib
+import json

 from django.apps import apps
 from django.conf import settings
 from django.db import models
 from django.apps import apps
 from django.conf import settings
 from django.db import models


@@ -109,3 +111,47 @@ class DynamicTextInsertText(models.Model):
     text_color = models.CharField(max_length=10, blank=True)
     text = models.TextField(_('text'))
     image = models.FileField(blank=True, upload_to='annoy/inserts/')
     text_color = models.CharField(max_length=10, blank=True)
     text = models.TextField(_('text'))
     image = models.FileField(blank=True, upload_to='annoy/inserts/')

+
+
+class MediaInsertSet(models.Model):
+    file_format = models.CharField(max_length=8, choices=[
+        ('epub', 'epub'),
+        ('mobi', 'mobi'),
+        ])
+    etag = models.CharField(max_length=64, blank=True)
+
+    def update_etag(self):
+        self.etag = hashlib.sha1(json.dumps(self.get_texts()).encode('utf-8')).hexdigest()
+        self.save(update_fields=['etag'])
+
+    def get_texts(self):
+        return [t.text for t in self.mediainserttext_set.all()]
+
+    @classmethod
+    def get_for_format(cls, file_format):
+        return cls.objects.filter(file_format=file_format).first()
+
+    @classmethod
+    def get_texts_for(cls, file_format):
+        self = cls.get_for_format(file_format)
+        if self is None:
+            return []
+        return self.get_texts()
+
+
+class MediaInsertText(models.Model):
+    media_insert_set = models.ForeignKey(MediaInsertSet, models.CASCADE)
+    ordering = models.IntegerField()
+    text = models.TextField()
+
+    class Meta:
+        ordering = ('ordering',)
+
+
+from django.db.models.signals import post_save, post_delete
+from django.dispatch import receiver
+
+@receiver(post_delete, sender=MediaInsertText)
+@receiver(post_save, sender=MediaInsertText)
+def update_etag(sender, instance, **kwargs):
+    instance.media_insert_set.update_etag()

diff --git a/src/catalogue/fields.py b/src/catalogue/fields.py
index 6ae4da3..bf17d35 100644 (file)
--- a/src/catalogue/fields.py
+++ b/src/catalogue/fields.py
@@ -2,13 +2,16 @@
 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
 #
 import os
 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
 #
 import os

+import pkg_resources
+import random
+from django.apps import apps

 from django.conf import settings
 from django.core.files import File
 from django.db import models
 from django.db.models.fields.files import FieldFile
 from django.utils.deconstruct import deconstructible
 from django.utils.translation import gettext_lazy as _
 from django.conf import settings
 from django.core.files import File
 from django.db import models
 from django.db.models.fields.files import FieldFile
 from django.utils.deconstruct import deconstructible
 from django.utils.translation import gettext_lazy as _

-from catalogue.constants import LANGUAGES_3TO2, EBOOK_FORMATS_WITH_CHILDREN, EBOOK_FORMATS_WITHOUT_CHILDREN
+from catalogue.constants import LANGUAGES_3TO2

 from catalogue.utils import absolute_url, remove_zip, truncate_html_words, gallery_path, gallery_url
 from waiter.utils import clear_cache
 
 from catalogue.utils import absolute_url, remove_zip, truncate_html_words, gallery_path, gallery_url
 from waiter.utils import clear_cache
 


@@ -66,6 +69,7 @@ class EbookField(models.FileField):
     """Represents an ebook file field, attachable to a model."""
     attr_class = EbookFieldFile
     ext = None
     """Represents an ebook file field, attachable to a model."""
     attr_class = EbookFieldFile
     ext = None

+    for_parents = True

     librarian2_api = False
     ZIP = None
 
     librarian2_api = False
     ZIP = None
 


@@ -124,39 +128,49 @@ class EbookField(models.FileField):
         setattr(cls, 'has_%s' % self.attname, has)
 
     def get_current_etag(self):
         setattr(cls, 'has_%s' % self.attname, has)
 
     def get_current_etag(self):

-        import pkg_resources
+        MediaInsertSet = apps.get_model('annoy', 'MediaInsertSet')

         librarian_version = pkg_resources.get_distribution("librarian").version
         librarian_version = pkg_resources.get_distribution("librarian").version

-        return librarian_version
-
-    def schedule_stale(self, queryset=None):
-        """Schedule building this format for all the books where etag is stale."""
+        etag = librarian_version
+        mis = MediaInsertSet.get_for_format(self.ext)
+        if mis is not None:
+            etag += '_' + mis.etag
+        return etag
+
+    def find_stale(self, limit):
+        """Find some books where this format is stale."""

         # If there is not ETag field, bail. That's true for xml file field.
         if not self.with_etag:
         # If there is not ETag field, bail. That's true for xml file field.
         if not self.with_etag:

-            return
+            return []

 
         etag = self.get_current_etag()
 
         etag = self.get_current_etag()

-        if queryset is None:
-            queryset = self.model.objects.all()

 
 

-        if self.format_name in EBOOK_FORMATS_WITHOUT_CHILDREN + ['html']:
+        queryset = self.model.objects.all()
+        if not self.for_parents:

             queryset = queryset.filter(children=None)
 
         queryset = queryset.exclude(**{
             f'{self.etag_field_name}__in': [
                 etag, f'{etag}{ETAG_SCHEDULED_SUFFIX}'
             queryset = queryset.filter(children=None)
 
         queryset = queryset.exclude(**{
             f'{self.etag_field_name}__in': [
                 etag, f'{etag}{ETAG_SCHEDULED_SUFFIX}'

-            ]
+           ]

         })
         })

-        for obj in queryset:
-            fieldfile = getattr(obj, self.attname)
-            priority = EBOOK_REBUILD_PRIORITY if fieldfile else EBOOK_BUILD_PRIORITY
-            fieldfile.build_delay(priority=priority)
+
+        queryset = queryset.order_by('?')[:limit]
+        return queryset

 
     @classmethod
 
     @classmethod

-    def schedule_all_stale(cls, model):
+    def find_all_stale(cls, model, limit):

         """Schedules all stale ebooks of all formats to rebuild."""
         """Schedules all stale ebooks of all formats to rebuild."""

+        found = []

         for field in model._meta.fields:
             if isinstance(field, cls):
         for field in model._meta.fields:
             if isinstance(field, cls):

-                field.schedule_stale()
+                for instance in field.find_stale(limit):
+                    found.append((
+                        field.name,
+                        instance
+                    ))
+        random.shuffle(found)
+        found = found[:limit]
+        return found

 
     @staticmethod
     def transform(wldoc):
 
     @staticmethod
     def transform(wldoc):


@@ -191,6 +205,7 @@ class XmlField(EbookField):
 
 class TxtField(EbookField):
     ext = 'txt'
 
 class TxtField(EbookField):
     ext = 'txt'

+    for_parents = False

 
     @staticmethod
     def transform(wldoc):
 
     @staticmethod
     def transform(wldoc):


@@ -199,6 +214,7 @@ class TxtField(EbookField):
 
 class Fb2Field(EbookField):
     ext = 'fb2'
 
 class Fb2Field(EbookField):
     ext = 'fb2'

+    for_parents = False

     ZIP = 'wolnelektury_pl_fb2'
 
     @staticmethod
     ZIP = 'wolnelektury_pl_fb2'
 
     @staticmethod


@@ -229,9 +245,10 @@ class EpubField(EbookField):
     @staticmethod
     def transform(wldoc):
         from librarian.builders import EpubBuilder
     @staticmethod
     def transform(wldoc):
         from librarian.builders import EpubBuilder

+        MediaInsertSet = apps.get_model('annoy', 'MediaInsertSet')

         return EpubBuilder(
                 base_url='file://' + os.path.abspath(gallery_path(wldoc.meta.url.slug)) + '/',
         return EpubBuilder(
                 base_url='file://' + os.path.abspath(gallery_path(wldoc.meta.url.slug)) + '/',

-                fundraising=settings.EPUB_FUNDRAISING
+                fundraising=MediaInsertSet.get_texts_for('epub')

             ).build(wldoc)
 
 
             ).build(wldoc)
 
 


@@ -243,14 +260,16 @@ class MobiField(EbookField):
     @staticmethod
     def transform(wldoc):
         from librarian.builders import MobiBuilder
     @staticmethod
     def transform(wldoc):
         from librarian.builders import MobiBuilder

+        MediaInsertSet = apps.get_model('annoy', 'MediaInsertSet')

         return MobiBuilder(
                 base_url='file://' + os.path.abspath(gallery_path(wldoc.meta.url.slug)) + '/',
         return MobiBuilder(
                 base_url='file://' + os.path.abspath(gallery_path(wldoc.meta.url.slug)) + '/',

-                fundraising=settings.EPUB_FUNDRAISING
+                fundraising=MediaInsertSet.get_texts_for('mobi')

             ).build(wldoc)
 
 
 class HtmlField(EbookField):
     ext = 'html'
             ).build(wldoc)
 
 
 class HtmlField(EbookField):
     ext = 'html'

+    for_parents = False

 
     def build(self, fieldfile):
         from django.core.files.base import ContentFile
 
     def build(self, fieldfile):
         from django.core.files.base import ContentFile

diff --git a/src/catalogue/management/commands/build_stale.py b/src/catalogue/management/commands/build_stale.py
new file mode 100644 (file)
index 0000000..b03e3e8
--- /dev/null
+++ b/src/catalogue/management/commands/build_stale.py
@@ -0,0 +1,41 @@
+# This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
+# Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
+#
+from time import time
+from django.conf import settings
+from django.core.management.base import BaseCommand
+
+from catalogue.fields import EbookField
+from catalogue.models import Book
+
+
+class Command(BaseCommand):
+    help = 'Schedule regenerating stale ebook files.'
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '-l', '--limit', type=int,
+            help='Limit number of files to build'
+        )
+        parser.add_argument(
+            '-t', '--time', type=int, metavar='SECONDS',
+            help='Limit timenumber of files to build'
+        )
+
+    def handle(self, **options):
+        t = time()
+        counter = 0
+        while True:
+            if options['time'] is not None and time() - t > options['time']:
+                break
+            if options['limit'] is not None and counter >= options['limit']:
+                break
+            tasks = EbookField.find_all_stale(Book, 1)
+            if not tasks:
+                break
+            for field_name, book in tasks:
+                print(field_name, book)
+                try:
+                    getattr(book, field_name).build()
+                except Exception as e:
+                    print('ERROR', e)

diff --git a/src/catalogue/management/commands/schedule_stale_ebooks.py b/src/catalogue/management/commands/schedule_stale_ebooks.py
deleted file mode 100644 (file)
index 4420a81..0000000
--- a/src/catalogue/management/commands/schedule_stale_ebooks.py
+++ /dev/null
@@ -1,15 +0,0 @@
-# This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
-# Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
-#
-from django.conf import settings
-from django.core.management.base import BaseCommand
-
-from catalogue.fields import EbookField
-from catalogue.models import Book
-
-
-class Command(BaseCommand):
-    help = 'Schedule regenerating stale ebook files.'
-
-    def handle(self, **options):
-        EbookField.schedule_all_stale(Book)

diff --git a/src/reporting/templates/reporting/main.html b/src/reporting/templates/reporting/main.html
index a69e9d1..ece5d02 100644 (file)
--- a/src/reporting/templates/reporting/main.html
+++ b/src/reporting/templates/reporting/main.html
@@ -1,5 +1,5 @@
-{% extends request.EXPERIMENTS.layout.value|yesno:"2022/base_simple.html,base/base.html" %}
-{% load i18n %}
+{% extends "2022/base_simple.html" %}
+{% load i18n l10n %}

 {% load reporting_stats catalogue_tags %}
 
 {% block settings %}
 {% load reporting_stats catalogue_tags %}
 
 {% block settings %}


@@ -48,5 +48,50 @@
         <li>{{ license }} ({{ license_description }})</li>
       {% endfor %}
     </ul>
         <li>{{ license }} ({{ license_description }})</li>
       {% endfor %}
     </ul>

+
+    {% localize off %}
+    <h3>Aktualność plików</h3>
+    <table border="1">
+      <tr>
+        {% for e in etags %}
+          <td>
+            <div
+                style="width: 80px; overflow: hidden; text-overflow: ellipsis; padding: 2px;"
+                title="{{ e.field }}"
+            >
+              {{ e.field }}
+            </div>
+          </td>
+        {% endfor %}
+      </tr>
+      <tr>
+        {% for e in etags %}
+          <td>
+            <div style="width: 80px; overflow: hidden; text-overflow: ellipsis; padding: 2px;">
+              {{ e.etag }}
+            </div>
+          </td>
+        {% endfor %}
+      </tr>
+      <tr>
+        {% for e in etags %}
+          <td style="vertical-align: top">
+            {% for et in e.tags %}
+              <div
+                  style="position: relative; width:80px; overflow:hidden; text-overflow: ellipsis; padding: 2px;"
+                  title="{{ et.perc }}%"
+              >
+                <span style="z-index: 0; background: {% if et.tag == e.etag %}#92BD39{% elif et.tag == '' %}#FF4C54{% else %}#FFA500{% endif %}; position: absolute;left:0; top:0; bottom: 0; width: {{ et.perc }}%;"></span>
+                <span style="position: relative; z-index: 1">
+                  {{ et.count }} {{ et.tag }}
+                </span>
+              </div>
+            {% endfor %}
+          </td>
+        {% endfor %}
+      </tr>
+    </table>
+    {% endlocalize %}
+

   </div>
 {% endblock %}
   </div>
 {% endblock %}

diff --git a/src/reporting/views.py b/src/reporting/views.py
index 9eecf0d..74e24eb 100644 (file)
--- a/src/reporting/views.py
+++ b/src/reporting/views.py
@@ -30,9 +30,40 @@ def stats_page(request):
         if extra_info.get('license'):
             licenses.add((extra_info.get('license'), extra_info.get('license_description')))
 
         if extra_info.get('license'):
             licenses.add((extra_info.get('license'), extra_info.get('license_description')))
 

+    etags = []
+    all_books = Book.objects.all()
+    all_books_count = all_books.count()
+    noparent_books = Book.objects.filter(children=None)
+    noparent_books_count = noparent_books.count()
+    for field in Book._meta.fields:
+        if not getattr(field, 'with_etag', None): continue
+        etag = field.get_current_etag()
+        d = {
+            'field': field.name,
+            'etag': etag,
+        }
+        if field.for_parents:
+            books = all_books
+            n_books = all_books_count
+        else:
+            books = noparent_books
+            n_books = noparent_books_count
+        tags = books.values_list(field.etag_field.name).order_by(
+            '-' + field.etag_field.name).distinct().annotate(c=Count('*'))
+        d['tags'] = [
+            {
+                'tag': t[0],
+                'count': t[1],
+                'perc': round(100 * t[1] / n_books, 2)
+            }
+            for t in tags
+        ]
+        etags.append(d)
+

     return render(request, 'reporting/main.html', {
         'media_types': media_types,
         'licenses': licenses,
     return render(request, 'reporting/main.html', {
         'media_types': media_types,
         'licenses': licenses,

+        'etags': etags,

     })
 
 
     })