from django.contrib import admin
from django import forms
+from admin_ordering.admin import OrderableAdmin
from modeltranslation.admin import TranslationAdmin
from . import models
admin.site.register(models.DynamicTextInsert, DynamicTextInsertAdmin)
+
+
+class MediaInsertTextInline(OrderableAdmin, admin.TabularInline):
+ model = models.MediaInsertText
+ extra = 0
+ min_num = 1
+
+
+@admin.register(models.MediaInsertSet)
+class MediaInsertSetAdmin(admin.ModelAdmin):
+ list_display = ['file_format', 'etag']
+ inlines = [MediaInsertTextInline]
+ readonly_fields = ['etag']
--- /dev/null
+# Generated by Django 4.0.8 on 2023-07-21 15:12
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('annoy', '0012_banner_only_authenticated'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='MediaInsertSet',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('file_format', models.CharField(choices=[('epub', 'epub'), ('mobi', 'mobi')], max_length=8)),
+ ('etag', models.CharField(blank=True, max_length=64)),
+ ],
+ ),
+ migrations.CreateModel(
+ name='MediaInsertText',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('ordering', models.IntegerField()),
+ ('text', models.TextField()),
+ ('media_insert_set', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='annoy.mediainsertset')),
+ ],
+ options={
+ 'ordering': ('ordering',),
+ },
+ ),
+ ]
--- /dev/null
+# Generated by Django 4.0.8 on 2023-08-02 19:58
+import hashlib
+import json
+from django.conf import settings
+from django.db import migrations
+
+
+def migrate_fundraising(apps, schema_editor):
+ MIS = apps.get_model('annoy', 'MediaInsertSet')
+ EF = getattr(settings, 'EPUB_FUNDRAISING', None)
+ if not EF:
+ return
+ etag = hashlib.sha1(json.dumps(EF).encode('utf-8')).hexdigest()
+ for f in ('epub', 'mobi'):
+ mis = MIS.objects.create(file_format=f, etag=etag)
+ for i, t in enumerate(EF):
+ mis.mediainserttext_set.create(ordering=i+1, text=t)
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('annoy', '0013_mediainsertset_mediainserttext'),
+ ]
+
+ operations = [
+ migrations.RunPython(
+ migrate_fundraising,
+ migrations.RunPython.noop
+ )
+ ]
+import hashlib
+import json
from django.apps import apps
from django.conf import settings
from django.db import models
text_color = models.CharField(max_length=10, blank=True)
text = models.TextField(_('text'))
image = models.FileField(blank=True, upload_to='annoy/inserts/')
+
+
+class MediaInsertSet(models.Model):
+ file_format = models.CharField(max_length=8, choices=[
+ ('epub', 'epub'),
+ ('mobi', 'mobi'),
+ ])
+ etag = models.CharField(max_length=64, blank=True)
+
+ def update_etag(self):
+ self.etag = hashlib.sha1(json.dumps(self.get_texts()).encode('utf-8')).hexdigest()
+ self.save(update_fields=['etag'])
+
+ def get_texts(self):
+ return [t.text for t in self.mediainserttext_set.all()]
+
+ @classmethod
+ def get_for_format(cls, file_format):
+ return cls.objects.filter(file_format=file_format).first()
+
+ @classmethod
+ def get_texts_for(cls, file_format):
+ self = cls.get_for_format(file_format)
+ if self is None:
+ return []
+ return self.get_texts()
+
+
+class MediaInsertText(models.Model):
+ media_insert_set = models.ForeignKey(MediaInsertSet, models.CASCADE)
+ ordering = models.IntegerField()
+ text = models.TextField()
+
+ class Meta:
+ ordering = ('ordering',)
+
+
+from django.db.models.signals import post_save, post_delete
+from django.dispatch import receiver
+
+@receiver(post_delete, sender=MediaInsertText)
+@receiver(post_save, sender=MediaInsertText)
+def update_etag(sender, instance, **kwargs):
+ instance.media_insert_set.update_etag()
# Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
#
import os
+import pkg_resources
+import random
+from django.apps import apps
from django.conf import settings
from django.core.files import File
from django.db import models
from django.db.models.fields.files import FieldFile
from django.utils.deconstruct import deconstructible
from django.utils.translation import gettext_lazy as _
-from catalogue.constants import LANGUAGES_3TO2, EBOOK_FORMATS_WITH_CHILDREN, EBOOK_FORMATS_WITHOUT_CHILDREN
+from catalogue.constants import LANGUAGES_3TO2
from catalogue.utils import absolute_url, remove_zip, truncate_html_words, gallery_path, gallery_url
from waiter.utils import clear_cache
"""Represents an ebook file field, attachable to a model."""
attr_class = EbookFieldFile
ext = None
+ for_parents = True
librarian2_api = False
ZIP = None
setattr(cls, 'has_%s' % self.attname, has)
def get_current_etag(self):
- import pkg_resources
+ MediaInsertSet = apps.get_model('annoy', 'MediaInsertSet')
librarian_version = pkg_resources.get_distribution("librarian").version
- return librarian_version
-
- def schedule_stale(self, queryset=None):
- """Schedule building this format for all the books where etag is stale."""
+ etag = librarian_version
+ mis = MediaInsertSet.get_for_format(self.ext)
+ if mis is not None:
+ etag += '_' + mis.etag
+ return etag
+
+ def find_stale(self, limit):
+ """Find some books where this format is stale."""
# If there is not ETag field, bail. That's true for xml file field.
if not self.with_etag:
- return
+ return []
etag = self.get_current_etag()
- if queryset is None:
- queryset = self.model.objects.all()
- if self.format_name in EBOOK_FORMATS_WITHOUT_CHILDREN + ['html']:
+ queryset = self.model.objects.all()
+ if not self.for_parents:
queryset = queryset.filter(children=None)
queryset = queryset.exclude(**{
f'{self.etag_field_name}__in': [
etag, f'{etag}{ETAG_SCHEDULED_SUFFIX}'
- ]
+ ]
})
- for obj in queryset:
- fieldfile = getattr(obj, self.attname)
- priority = EBOOK_REBUILD_PRIORITY if fieldfile else EBOOK_BUILD_PRIORITY
- fieldfile.build_delay(priority=priority)
+
+ queryset = queryset.order_by('?')[:limit]
+ return queryset
@classmethod
- def schedule_all_stale(cls, model):
+ def find_all_stale(cls, model, limit):
"""Schedules all stale ebooks of all formats to rebuild."""
+ found = []
for field in model._meta.fields:
if isinstance(field, cls):
- field.schedule_stale()
+ for instance in field.find_stale(limit):
+ found.append((
+ field.name,
+ instance
+ ))
+ random.shuffle(found)
+ found = found[:limit]
+ return found
@staticmethod
def transform(wldoc):
class TxtField(EbookField):
ext = 'txt'
+ for_parents = False
@staticmethod
def transform(wldoc):
class Fb2Field(EbookField):
ext = 'fb2'
+ for_parents = False
ZIP = 'wolnelektury_pl_fb2'
@staticmethod
@staticmethod
def transform(wldoc):
from librarian.builders import EpubBuilder
+ MediaInsertSet = apps.get_model('annoy', 'MediaInsertSet')
return EpubBuilder(
base_url='file://' + os.path.abspath(gallery_path(wldoc.meta.url.slug)) + '/',
- fundraising=settings.EPUB_FUNDRAISING
+ fundraising=MediaInsertSet.get_texts_for('epub')
).build(wldoc)
@staticmethod
def transform(wldoc):
from librarian.builders import MobiBuilder
+ MediaInsertSet = apps.get_model('annoy', 'MediaInsertSet')
return MobiBuilder(
base_url='file://' + os.path.abspath(gallery_path(wldoc.meta.url.slug)) + '/',
- fundraising=settings.EPUB_FUNDRAISING
+ fundraising=MediaInsertSet.get_texts_for('mobi')
).build(wldoc)
class HtmlField(EbookField):
ext = 'html'
+ for_parents = False
def build(self, fieldfile):
from django.core.files.base import ContentFile
--- /dev/null
+# This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
+# Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
+#
+from time import time
+from django.conf import settings
+from django.core.management.base import BaseCommand
+
+from catalogue.fields import EbookField
+from catalogue.models import Book
+
+
+class Command(BaseCommand):
+ help = 'Schedule regenerating stale ebook files.'
+
+ def add_arguments(self, parser):
+ parser.add_argument(
+ '-l', '--limit', type=int,
+ help='Limit number of files to build'
+ )
+ parser.add_argument(
+ '-t', '--time', type=int, metavar='SECONDS',
+ help='Limit timenumber of files to build'
+ )
+
+ def handle(self, **options):
+ t = time()
+ counter = 0
+ while True:
+ if options['time'] is not None and time() - t > options['time']:
+ break
+ if options['limit'] is not None and counter >= options['limit']:
+ break
+ tasks = EbookField.find_all_stale(Book, 1)
+ if not tasks:
+ break
+ for field_name, book in tasks:
+ print(field_name, book)
+ try:
+ getattr(book, field_name).build()
+ except Exception as e:
+ print('ERROR', e)
+++ /dev/null
-# This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
-# Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
-#
-from django.conf import settings
-from django.core.management.base import BaseCommand
-
-from catalogue.fields import EbookField
-from catalogue.models import Book
-
-
-class Command(BaseCommand):
- help = 'Schedule regenerating stale ebook files.'
-
- def handle(self, **options):
- EbookField.schedule_all_stale(Book)
-{% extends request.EXPERIMENTS.layout.value|yesno:"2022/base_simple.html,base/base.html" %}
-{% load i18n %}
+{% extends "2022/base_simple.html" %}
+{% load i18n l10n %}
{% load reporting_stats catalogue_tags %}
{% block settings %}
<li>{{ license }} ({{ license_description }})</li>
{% endfor %}
</ul>
+
+ {% localize off %}
+ <h3>Aktualność plików</h3>
+ <table border="1">
+ <tr>
+ {% for e in etags %}
+ <td>
+ <div
+ style="width: 80px; overflow: hidden; text-overflow: ellipsis; padding: 2px;"
+ title="{{ e.field }}"
+ >
+ {{ e.field }}
+ </div>
+ </td>
+ {% endfor %}
+ </tr>
+ <tr>
+ {% for e in etags %}
+ <td>
+ <div style="width: 80px; overflow: hidden; text-overflow: ellipsis; padding: 2px;">
+ {{ e.etag }}
+ </div>
+ </td>
+ {% endfor %}
+ </tr>
+ <tr>
+ {% for e in etags %}
+ <td style="vertical-align: top">
+ {% for et in e.tags %}
+ <div
+ style="position: relative; width:80px; overflow:hidden; text-overflow: ellipsis; padding: 2px;"
+ title="{{ et.perc }}%"
+ >
+ <span style="z-index: 0; background: {% if et.tag == e.etag %}#92BD39{% elif et.tag == '' %}#FF4C54{% else %}#FFA500{% endif %}; position: absolute;left:0; top:0; bottom: 0; width: {{ et.perc }}%;"></span>
+ <span style="position: relative; z-index: 1">
+ {{ et.count }} {{ et.tag }}
+ </span>
+ </div>
+ {% endfor %}
+ </td>
+ {% endfor %}
+ </tr>
+ </table>
+ {% endlocalize %}
+
</div>
{% endblock %}
if extra_info.get('license'):
licenses.add((extra_info.get('license'), extra_info.get('license_description')))
+ etags = []
+ all_books = Book.objects.all()
+ all_books_count = all_books.count()
+ noparent_books = Book.objects.filter(children=None)
+ noparent_books_count = noparent_books.count()
+ for field in Book._meta.fields:
+ if not getattr(field, 'with_etag', None): continue
+ etag = field.get_current_etag()
+ d = {
+ 'field': field.name,
+ 'etag': etag,
+ }
+ if field.for_parents:
+ books = all_books
+ n_books = all_books_count
+ else:
+ books = noparent_books
+ n_books = noparent_books_count
+ tags = books.values_list(field.etag_field.name).order_by(
+ '-' + field.etag_field.name).distinct().annotate(c=Count('*'))
+ d['tags'] = [
+ {
+ 'tag': t[0],
+ 'count': t[1],
+ 'perc': round(100 * t[1] / n_books, 2)
+ }
+ for t in tags
+ ]
+ etags.append(d)
+
return render(request, 'reporting/main.html', {
'media_types': media_types,
'licenses': licenses,
+ 'etags': etags,
})
fnp / wolnelektury.git / commitdiff