From: Radek Czajka Date: Wed, 2 Aug 2023 21:07:41 +0000 (+0200) Subject: Editable media inserts. X-Git-Url: https://git.mdrn.pl/wolnelektury.git/commitdiff_plain/b2c4c0093c7829668c1bf6f5a7493e1aff697181?hp=cafca91e28661f24abbcda91c733de7e77d16e86 Editable media inserts. --- diff --git a/src/annoy/admin.py b/src/annoy/admin.py index 8470a1203..ab7be6aca 100644 --- a/src/annoy/admin.py +++ b/src/annoy/admin.py @@ -1,5 +1,6 @@ from django.contrib import admin from django import forms +from admin_ordering.admin import OrderableAdmin from modeltranslation.admin import TranslationAdmin from . import models @@ -40,3 +41,16 @@ class DynamicTextInsertAdmin(admin.ModelAdmin): admin.site.register(models.DynamicTextInsert, DynamicTextInsertAdmin) + + +class MediaInsertTextInline(OrderableAdmin, admin.TabularInline): + model = models.MediaInsertText + extra = 0 + min_num = 1 + + +@admin.register(models.MediaInsertSet) +class MediaInsertSetAdmin(admin.ModelAdmin): + list_display = ['file_format', 'etag'] + inlines = [MediaInsertTextInline] + readonly_fields = ['etag'] diff --git a/src/annoy/migrations/0013_mediainsertset_mediainserttext.py b/src/annoy/migrations/0013_mediainsertset_mediainserttext.py new file mode 100644 index 000000000..0c3c273f8 --- /dev/null +++ b/src/annoy/migrations/0013_mediainsertset_mediainserttext.py @@ -0,0 +1,34 @@ +# Generated by Django 4.0.8 on 2023-07-21 15:12 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('annoy', '0012_banner_only_authenticated'), + ] + + operations = [ + migrations.CreateModel( + name='MediaInsertSet', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('file_format', models.CharField(choices=[('epub', 'epub'), ('mobi', 'mobi')], max_length=8)), + ('etag', models.CharField(blank=True, max_length=64)), + ], + ), + migrations.CreateModel( + name='MediaInsertText', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('ordering', models.IntegerField()), + ('text', models.TextField()), + ('media_insert_set', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='annoy.mediainsertset')), + ], + options={ + 'ordering': ('ordering',), + }, + ), + ] diff --git a/src/annoy/migrations/0014_fundraising.py b/src/annoy/migrations/0014_fundraising.py new file mode 100644 index 000000000..8dc111934 --- /dev/null +++ b/src/annoy/migrations/0014_fundraising.py @@ -0,0 +1,31 @@ +# Generated by Django 4.0.8 on 2023-08-02 19:58 +import hashlib +import json +from django.conf import settings +from django.db import migrations + + +def migrate_fundraising(apps, schema_editor): + MIS = apps.get_model('annoy', 'MediaInsertSet') + EF = getattr(settings, 'EPUB_FUNDRAISING', None) + if not EF: + return + etag = hashlib.sha1(json.dumps(EF).encode('utf-8')).hexdigest() + for f in ('epub', 'mobi'): + mis = MIS.objects.create(file_format=f, etag=etag) + for i, t in enumerate(EF): + mis.mediainserttext_set.create(ordering=i+1, text=t) + + +class Migration(migrations.Migration): + + dependencies = [ + ('annoy', '0013_mediainsertset_mediainserttext'), + ] + + operations = [ + migrations.RunPython( + migrate_fundraising, + migrations.RunPython.noop + ) + ] diff --git a/src/annoy/models.py b/src/annoy/models.py index c8fdf2dad..61cb94cad 100644 --- a/src/annoy/models.py +++ b/src/annoy/models.py @@ -1,3 +1,5 @@ +import hashlib +import json from django.apps import apps from django.conf import settings from django.db import models @@ -109,3 +111,47 @@ class DynamicTextInsertText(models.Model): text_color = models.CharField(max_length=10, blank=True) text = models.TextField(_('text')) image = models.FileField(blank=True, upload_to='annoy/inserts/') + + +class MediaInsertSet(models.Model): + file_format = models.CharField(max_length=8, choices=[ + ('epub', 'epub'), + ('mobi', 'mobi'), + ]) + etag = models.CharField(max_length=64, blank=True) + + def update_etag(self): + self.etag = hashlib.sha1(json.dumps(self.get_texts()).encode('utf-8')).hexdigest() + self.save(update_fields=['etag']) + + def get_texts(self): + return [t.text for t in self.mediainserttext_set.all()] + + @classmethod + def get_for_format(cls, file_format): + return cls.objects.filter(file_format=file_format).first() + + @classmethod + def get_texts_for(cls, file_format): + self = cls.get_for_format(file_format) + if self is None: + return [] + return self.get_texts() + + +class MediaInsertText(models.Model): + media_insert_set = models.ForeignKey(MediaInsertSet, models.CASCADE) + ordering = models.IntegerField() + text = models.TextField() + + class Meta: + ordering = ('ordering',) + + +from django.db.models.signals import post_save, post_delete +from django.dispatch import receiver + +@receiver(post_delete, sender=MediaInsertText) +@receiver(post_save, sender=MediaInsertText) +def update_etag(sender, instance, **kwargs): + instance.media_insert_set.update_etag() diff --git a/src/catalogue/fields.py b/src/catalogue/fields.py index 6ae4da3c4..bf17d3581 100644 --- a/src/catalogue/fields.py +++ b/src/catalogue/fields.py @@ -2,13 +2,16 @@ # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information. # import os +import pkg_resources +import random +from django.apps import apps from django.conf import settings from django.core.files import File from django.db import models from django.db.models.fields.files import FieldFile from django.utils.deconstruct import deconstructible from django.utils.translation import gettext_lazy as _ -from catalogue.constants import LANGUAGES_3TO2, EBOOK_FORMATS_WITH_CHILDREN, EBOOK_FORMATS_WITHOUT_CHILDREN +from catalogue.constants import LANGUAGES_3TO2 from catalogue.utils import absolute_url, remove_zip, truncate_html_words, gallery_path, gallery_url from waiter.utils import clear_cache @@ -66,6 +69,7 @@ class EbookField(models.FileField): """Represents an ebook file field, attachable to a model.""" attr_class = EbookFieldFile ext = None + for_parents = True librarian2_api = False ZIP = None @@ -124,39 +128,49 @@ class EbookField(models.FileField): setattr(cls, 'has_%s' % self.attname, has) def get_current_etag(self): - import pkg_resources + MediaInsertSet = apps.get_model('annoy', 'MediaInsertSet') librarian_version = pkg_resources.get_distribution("librarian").version - return librarian_version - - def schedule_stale(self, queryset=None): - """Schedule building this format for all the books where etag is stale.""" + etag = librarian_version + mis = MediaInsertSet.get_for_format(self.ext) + if mis is not None: + etag += '_' + mis.etag + return etag + + def find_stale(self, limit): + """Find some books where this format is stale.""" # If there is not ETag field, bail. That's true for xml file field. if not self.with_etag: - return + return [] etag = self.get_current_etag() - if queryset is None: - queryset = self.model.objects.all() - if self.format_name in EBOOK_FORMATS_WITHOUT_CHILDREN + ['html']: + queryset = self.model.objects.all() + if not self.for_parents: queryset = queryset.filter(children=None) queryset = queryset.exclude(**{ f'{self.etag_field_name}__in': [ etag, f'{etag}{ETAG_SCHEDULED_SUFFIX}' - ] + ] }) - for obj in queryset: - fieldfile = getattr(obj, self.attname) - priority = EBOOK_REBUILD_PRIORITY if fieldfile else EBOOK_BUILD_PRIORITY - fieldfile.build_delay(priority=priority) + + queryset = queryset.order_by('?')[:limit] + return queryset @classmethod - def schedule_all_stale(cls, model): + def find_all_stale(cls, model, limit): """Schedules all stale ebooks of all formats to rebuild.""" + found = [] for field in model._meta.fields: if isinstance(field, cls): - field.schedule_stale() + for instance in field.find_stale(limit): + found.append(( + field.name, + instance + )) + random.shuffle(found) + found = found[:limit] + return found @staticmethod def transform(wldoc): @@ -191,6 +205,7 @@ class XmlField(EbookField): class TxtField(EbookField): ext = 'txt' + for_parents = False @staticmethod def transform(wldoc): @@ -199,6 +214,7 @@ class TxtField(EbookField): class Fb2Field(EbookField): ext = 'fb2' + for_parents = False ZIP = 'wolnelektury_pl_fb2' @staticmethod @@ -229,9 +245,10 @@ class EpubField(EbookField): @staticmethod def transform(wldoc): from librarian.builders import EpubBuilder + MediaInsertSet = apps.get_model('annoy', 'MediaInsertSet') return EpubBuilder( base_url='file://' + os.path.abspath(gallery_path(wldoc.meta.url.slug)) + '/', - fundraising=settings.EPUB_FUNDRAISING + fundraising=MediaInsertSet.get_texts_for('epub') ).build(wldoc) @@ -243,14 +260,16 @@ class MobiField(EbookField): @staticmethod def transform(wldoc): from librarian.builders import MobiBuilder + MediaInsertSet = apps.get_model('annoy', 'MediaInsertSet') return MobiBuilder( base_url='file://' + os.path.abspath(gallery_path(wldoc.meta.url.slug)) + '/', - fundraising=settings.EPUB_FUNDRAISING + fundraising=MediaInsertSet.get_texts_for('mobi') ).build(wldoc) class HtmlField(EbookField): ext = 'html' + for_parents = False def build(self, fieldfile): from django.core.files.base import ContentFile diff --git a/src/catalogue/management/commands/build_stale.py b/src/catalogue/management/commands/build_stale.py new file mode 100644 index 000000000..b03e3e8e6 --- /dev/null +++ b/src/catalogue/management/commands/build_stale.py @@ -0,0 +1,41 @@ +# This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later. +# Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information. +# +from time import time +from django.conf import settings +from django.core.management.base import BaseCommand + +from catalogue.fields import EbookField +from catalogue.models import Book + + +class Command(BaseCommand): + help = 'Schedule regenerating stale ebook files.' + + def add_arguments(self, parser): + parser.add_argument( + '-l', '--limit', type=int, + help='Limit number of files to build' + ) + parser.add_argument( + '-t', '--time', type=int, metavar='SECONDS', + help='Limit timenumber of files to build' + ) + + def handle(self, **options): + t = time() + counter = 0 + while True: + if options['time'] is not None and time() - t > options['time']: + break + if options['limit'] is not None and counter >= options['limit']: + break + tasks = EbookField.find_all_stale(Book, 1) + if not tasks: + break + for field_name, book in tasks: + print(field_name, book) + try: + getattr(book, field_name).build() + except Exception as e: + print('ERROR', e) diff --git a/src/catalogue/management/commands/schedule_stale_ebooks.py b/src/catalogue/management/commands/schedule_stale_ebooks.py deleted file mode 100644 index 4420a815e..000000000 --- a/src/catalogue/management/commands/schedule_stale_ebooks.py +++ /dev/null @@ -1,15 +0,0 @@ -# This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later. -# Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information. -# -from django.conf import settings -from django.core.management.base import BaseCommand - -from catalogue.fields import EbookField -from catalogue.models import Book - - -class Command(BaseCommand): - help = 'Schedule regenerating stale ebook files.' - - def handle(self, **options): - EbookField.schedule_all_stale(Book) diff --git a/src/reporting/templates/reporting/main.html b/src/reporting/templates/reporting/main.html index a69e9d165..ece5d0268 100644 --- a/src/reporting/templates/reporting/main.html +++ b/src/reporting/templates/reporting/main.html @@ -1,5 +1,5 @@ -{% extends request.EXPERIMENTS.layout.value|yesno:"2022/base_simple.html,base/base.html" %} -{% load i18n %} +{% extends "2022/base_simple.html" %} +{% load i18n l10n %} {% load reporting_stats catalogue_tags %} {% block settings %} @@ -48,5 +48,50 @@
  • {{ license }} ({{ license_description }})
    • {% endfor %} + + {% localize off %} +

    Aktualność plików

    + + + {% for e in etags %} + + {% endfor %} + + + {% for e in etags %} + + {% endfor %} + + + {% for e in etags %} + + {% endfor %} + +
    +
    + {{ e.field }} +
    +
    +
    + {{ e.etag }} +
    +
    + {% for et in e.tags %} +
    + + + {{ et.count }} {{ et.tag }} + +
    + {% endfor %} +
    + {% endlocalize %} + {% endblock %} diff --git a/src/reporting/views.py b/src/reporting/views.py index 9eecf0d55..74e24eb80 100644 --- a/src/reporting/views.py +++ b/src/reporting/views.py @@ -30,9 +30,40 @@ def stats_page(request): if extra_info.get('license'): licenses.add((extra_info.get('license'), extra_info.get('license_description'))) + etags = [] + all_books = Book.objects.all() + all_books_count = all_books.count() + noparent_books = Book.objects.filter(children=None) + noparent_books_count = noparent_books.count() + for field in Book._meta.fields: + if not getattr(field, 'with_etag', None): continue + etag = field.get_current_etag() + d = { + 'field': field.name, + 'etag': etag, + } + if field.for_parents: + books = all_books + n_books = all_books_count + else: + books = noparent_books + n_books = noparent_books_count + tags = books.values_list(field.etag_field.name).order_by( + '-' + field.etag_field.name).distinct().annotate(c=Count('*')) + d['tags'] = [ + { + 'tag': t[0], + 'count': t[1], + 'perc': round(100 * t[1] / n_books, 2) + } + for t in tags + ] + etags.append(d) + return render(request, 'reporting/main.html', { 'media_types': media_types, 'licenses': licenses, + 'etags': etags, })