From ad422879d55a62e02c71024531aa4a2277dedaf3 Mon Sep 17 00:00:00 2001 From: Radek Czajka Date: Tue, 26 Sep 2023 13:08:48 +0200 Subject: [PATCH] Isbns, and minor fixes. --- src/cover/utils.py | 5 +- src/depot/publishers/base.py | 8 +- src/documents/views.py | 2 +- src/isbn/__init__.py | 0 src/isbn/admin.py | 11 ++ src/isbn/apps.py | 6 + .../management/commands/isbn_import_wl.py | 29 ++++ src/isbn/migrations/0001_initial.py | 74 +++++++++ ...a_isbn_wl_data_alter_isbn_book_and_more.py | 49 ++++++ ...s_alter_isbn_bn_data_alter_isbn_wl_data.py | 28 ++++ src/isbn/migrations/0004_alter_isbn_form.py | 31 ++++ src/isbn/migrations/__init__.py | 0 src/isbn/models.py | 150 ++++++++++++++++++ src/isbn/product_forms.py | 10 ++ src/isbn/templates/isbn/list.html | 75 +++++++++ src/isbn/urls.py | 7 + src/isbn/views.py | 9 ++ src/redakcja/urls.py | 1 + 18 files changed, 489 insertions(+), 6 deletions(-) create mode 100644 src/isbn/__init__.py create mode 100644 src/isbn/admin.py create mode 100644 src/isbn/apps.py create mode 100644 src/isbn/management/commands/isbn_import_wl.py create mode 100644 src/isbn/migrations/0001_initial.py create mode 100644 src/isbn/migrations/0002_isbn_bn_data_isbn_wl_data_alter_isbn_book_and_more.py create mode 100644 src/isbn/migrations/0003_isbn_notes_alter_isbn_bn_data_alter_isbn_wl_data.py create mode 100644 src/isbn/migrations/0004_alter_isbn_form.py create mode 100644 src/isbn/migrations/__init__.py create mode 100644 src/isbn/models.py create mode 100644 src/isbn/product_forms.py create mode 100644 src/isbn/templates/isbn/list.html create mode 100644 src/isbn/urls.py create mode 100644 src/isbn/views.py diff --git a/src/cover/utils.py b/src/cover/utils.py index 56fb24a7..6815d29d 100644 --- a/src/cover/utils.py +++ b/src/cover/utils.py @@ -99,7 +99,10 @@ def get_wikimedia_data(url): entity = client.get(qitem) meta['title'] = entity.label.get('pl', str(entity.label)) author = entity.get(client.get(WIKIDATA.CREATOR)) - meta['author'] = author.label.get('pl', str(author.label)) + if author is not None: + meta['author'] = author.label.get('pl', str(author.label)) + else: + meta['author'] = '' if meta['license_name'] == 'Public domain': meta['license_name'] = 'domena publiczna' diff --git a/src/depot/publishers/base.py b/src/depot/publishers/base.py index 5cfcbef4..88ee56b2 100644 --- a/src/depot/publishers/base.py +++ b/src/depot/publishers/base.py @@ -41,7 +41,7 @@ class BasePublisher: '{}'.format( slugify(p.readable()), p.readable(), - ) + ) if p is not None else '' for p in wlbook.meta.authors ) + '
' description += '{}
'.format( @@ -54,21 +54,21 @@ class BasePublisher: '{}'.format( slugify(p), p, - ) + ) if p is not None else '' for p in wlbook.meta.epochs ) + ' ' description += 'Rodzaj: ' + ', '.join( '{}'.format( slugify(p), p, - ) + ) if p is not None else '' for p in wlbook.meta.kinds ) + ' ' description += 'Gatunek: ' + ', '.join( '{}'.format( slugify(p), p, - ) + ) if p is not None else '' for p in wlbook.meta.genres ) + '

' diff --git a/src/documents/views.py b/src/documents/views.py index 0fbb3b9f..ecc92199 100644 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -369,7 +369,7 @@ def book(request, slug): doc = None else: try: - stats = doc.get_statistic() + stats = doc.get_statistics() except: pass diff --git a/src/isbn/__init__.py b/src/isbn/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/isbn/admin.py b/src/isbn/admin.py new file mode 100644 index 00000000..15947ab7 --- /dev/null +++ b/src/isbn/admin.py @@ -0,0 +1,11 @@ +from django.contrib import admin +from . import models + +@admin.register(models.Isbn) +class IsbnAdmin(admin.ModelAdmin): + search_fields = ['pool__prefix', 'suffix'] + + +admin.site.register(models.IsbnPool) + +# Register your models here. diff --git a/src/isbn/apps.py b/src/isbn/apps.py new file mode 100644 index 00000000..befb64f8 --- /dev/null +++ b/src/isbn/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class IsbnConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "isbn" diff --git a/src/isbn/management/commands/isbn_import_wl.py b/src/isbn/management/commands/isbn_import_wl.py new file mode 100644 index 00000000..0ed87456 --- /dev/null +++ b/src/isbn/management/commands/isbn_import_wl.py @@ -0,0 +1,29 @@ +import json +from django.core.management.base import BaseCommand +from isbn.models import IsbnPool + + +class Command(BaseCommand): + def add_arguments(self, parser): + parser.add_argument('filename') + + def handle(self, filename, **options): + with open(filename) as f: + data = json.load(f) + pool_map = {} + for d in data: + f = d['fields'] + if d['model'] == 'isbn.isbnpool': + pool_map[d['pk']], created = IsbnPool.objects.get_or_create( + prefix=f['prefix'], + suffix_from=f['suffix_from'], + suffix_to=f['suffix_to'], + ref_from=f['ref_from'], + ) + if d['model'] == 'isbn.onixrecord': + pool = pool_map[f['isbn_pool']] + isbn, created = pool.isbn_set.get_or_create( + suffix=f['suffix'], + ) + isbn.wl_data = json.dumps(f, indent=4, ensure_ascii=False) + isbn.save(update_fields=['wl_data']) diff --git a/src/isbn/migrations/0001_initial.py b/src/isbn/migrations/0001_initial.py new file mode 100644 index 00000000..f4d3eeab --- /dev/null +++ b/src/isbn/migrations/0001_initial.py @@ -0,0 +1,74 @@ +# Generated by Django 4.1.9 on 2023-09-12 18:10 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ("catalogue", "0050_audience_woblink"), + ] + + operations = [ + migrations.CreateModel( + name="IsbnPool", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("prefix", models.CharField(max_length=10)), + ("suffix_from", models.IntegerField()), + ("suffix_to", models.IntegerField()), + ("ref_from", models.IntegerField()), + ( + "purpose", + models.CharField( + choices=[("WL", "Wolne Lektury"), ("GENERAL", "Ogólne")], + max_length=8, + ), + ), + ], + ), + migrations.CreateModel( + name="Isbn", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("datestamp", models.DateField()), + ("suffix", models.IntegerField()), + ("form", models.CharField(choices=[], max_length=32)), + ( + "book", + models.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, to="catalogue.book" + ), + ), + ( + "pool", + models.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, to="isbn.isbnpool" + ), + ), + ], + options={ + "ordering": ["pool", "suffix"], + "unique_together": {("pool", "suffix")}, + }, + ), + ] diff --git a/src/isbn/migrations/0002_isbn_bn_data_isbn_wl_data_alter_isbn_book_and_more.py b/src/isbn/migrations/0002_isbn_bn_data_isbn_wl_data_alter_isbn_book_and_more.py new file mode 100644 index 00000000..8a1c3727 --- /dev/null +++ b/src/isbn/migrations/0002_isbn_bn_data_isbn_wl_data_alter_isbn_book_and_more.py @@ -0,0 +1,49 @@ +# Generated by Django 4.1.9 on 2023-09-25 15:10 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("catalogue", "0050_audience_woblink"), + ("isbn", "0001_initial"), + ] + + operations = [ + migrations.AddField( + model_name="isbn", + name="bn_data", + field=models.TextField(default=""), + preserve_default=False, + ), + migrations.AddField( + model_name="isbn", + name="wl_data", + field=models.TextField(default=""), + preserve_default=False, + ), + migrations.AlterField( + model_name="isbn", + name="book", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.PROTECT, + to="catalogue.book", + ), + ), + migrations.AlterField( + model_name="isbn", + name="datestamp", + field=models.DateField(blank=True, null=True), + ), + migrations.AlterField( + model_name="isbn", + name="form", + field=models.CharField( + blank=True, choices=[("pdf", "pdf")], max_length=32, null=True + ), + ), + ] diff --git a/src/isbn/migrations/0003_isbn_notes_alter_isbn_bn_data_alter_isbn_wl_data.py b/src/isbn/migrations/0003_isbn_notes_alter_isbn_bn_data_alter_isbn_wl_data.py new file mode 100644 index 00000000..7c44c18e --- /dev/null +++ b/src/isbn/migrations/0003_isbn_notes_alter_isbn_bn_data_alter_isbn_wl_data.py @@ -0,0 +1,28 @@ +# Generated by Django 4.1.9 on 2023-09-26 10:52 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("isbn", "0002_isbn_bn_data_isbn_wl_data_alter_isbn_book_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="isbn", + name="notes", + field=models.TextField(blank=True), + ), + migrations.AlterField( + model_name="isbn", + name="bn_data", + field=models.TextField(blank=True), + ), + migrations.AlterField( + model_name="isbn", + name="wl_data", + field=models.TextField(blank=True), + ), + ] diff --git a/src/isbn/migrations/0004_alter_isbn_form.py b/src/isbn/migrations/0004_alter_isbn_form.py new file mode 100644 index 00000000..fe2b0b4b --- /dev/null +++ b/src/isbn/migrations/0004_alter_isbn_form.py @@ -0,0 +1,31 @@ +# Generated by Django 4.1.9 on 2023-09-26 13:08 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("isbn", "0003_isbn_notes_alter_isbn_bn_data_alter_isbn_wl_data"), + ] + + operations = [ + migrations.AlterField( + model_name="isbn", + name="form", + field=models.CharField( + blank=True, + choices=[ + ("html", "html"), + ("txt", "txt"), + ("pdf", "pdf"), + ("epub", "epub"), + ("mobi", "mobi"), + ("mp3", "mp3"), + ("paperback", "paperback"), + ], + max_length=32, + null=True, + ), + ), + ] diff --git a/src/isbn/migrations/__init__.py b/src/isbn/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/isbn/models.py b/src/isbn/models.py new file mode 100644 index 00000000..9db0513c --- /dev/null +++ b/src/isbn/models.py @@ -0,0 +1,150 @@ +from django.apps import apps +from django.db import models +from lxml import etree +import requests +from .product_forms import FORMS + + +class IsbnPool(models.Model): + PURPOSE_GENERAL = 'GENERAL' + PURPOSE_WL = 'WL' + PURPOSE_CHOICES = ( + (PURPOSE_WL, 'Wolne Lektury'), + (PURPOSE_GENERAL, 'Ogólne'), + ) + + prefix = models.CharField(max_length=10) + suffix_from = models.IntegerField() + suffix_to = models.IntegerField() + ref_from = models.IntegerField() + purpose = models.CharField(max_length=8, choices=PURPOSE_CHOICES) + + def __str__(self): + return '-'.join(( + self.prefix[:3], + self.prefix[3:5], + self.prefix[5:], + 'X' * (12 - len(self.prefix)), + 'X' + )) + + @staticmethod + def check_digit(prefix12): + digits = [int(d) for d in prefix12] + return str((-sum(digits[0::2]) + 7 * sum(digits[1::2])) % 10) + + def get_code(self, suffix, dashes=False): + suffix_length = 12 - len(self.prefix) + suffix_str = f'{suffix:0{suffix_length}d}' + prefix12 = self.prefix + suffix_str + check_digit = self.check_digit(prefix12) + if dashes: + isbn = '-'.join(( + self.prefix[:3], + self.prefix[3:5], + self.prefix[5:], + suffix_str, + check_digit + )) + else: + isbn = ''.join(( + prefix12, check_digit + )) + return isbn + + + @property + def size(self): + return self.suffix_to - self.suffix_from + 1 + + @property + def entries(self): + return self.isbn_set.count() + + @property + def fill_percentage(self): + return 100 * self.entries / self.size + + def bn_record_id_for(self, suffix): + return self.ref_from + suffix + + def import_all_bn_data(self): + for suffix in range(self.suffix_from, self.suffix_to + 1): + print(suffix) + self.import_bn_data_for(suffix) + + def import_bn_data_for(self, suffix): + record_id = self.bn_record_id_for(suffix) + content = requests.get( + f'https://e-isbn.pl/IsbnWeb/record/export_onix.xml?record_id={record_id}').content + elem = etree.fromstring(content) + product = elem.find('{http://ns.editeur.org/onix/3.0/reference}Product') + if product is not None: + isbn, created = self.isbn_set.get_or_create( + suffix=suffix + ) + isbn.bn_data = etree.tostring(product, pretty_print=True, encoding='unicode') + isbn.save(update_fields=['bn_data']) + + +class Isbn(models.Model): + pool = models.ForeignKey(IsbnPool, models.PROTECT) + suffix = models.IntegerField() + datestamp = models.DateField(blank=True, null=True) + book = models.ForeignKey( + 'catalogue.Book', models.PROTECT, null=True, blank=True + ) + form = models.CharField( + max_length=32, choices=[ + (form, form) + for form, config in FORMS + ], null=True, blank=True + ) + bn_data = models.TextField(blank=True) + wl_data = models.TextField(blank=True) + notes = models.TextField(blank=True) + + class Meta: + ordering = ['pool', 'suffix'] + unique_together = ['pool', 'suffix'] + + def __str__(self): + return self.get_code(True) + + def get_code(self, dashes=True): + return self.pool.get_code(self.suffix, dashes=dashes) + + @classmethod + def import_from_documents(cls): + Book = apps.get_model('documents', 'Book') + for book in Book.objects.all(): + try: + catalogue_book = book.catalogue_book + if catalogue_book is None: + continue + except: + continue + try: + meta = book.wldocument(publishable=False, librarian2=True).meta + except: + continue + for form in ('html', 'txt', 'pdf', 'epub', 'mobi'): + isbn = getattr(meta, f'isbn_{form}') + if isbn is not None: + parts = isbn.split('-') + assert parts[0] == 'ISBN' + suffix = int(parts[-2]) + prefix = ''.join(parts[1:-2]) + pool = IsbnPool.objects.get(prefix=prefix) + isbn, created = pool.isbn_set.get_or_create( + suffix=suffix, + ) + if isbn.book is None: + isbn.book = catalogue_book + else: + assert isbn.book is catalogue_book + if isbn.form is None: + isbn.form = form + else: + assert isbn.form == form + isbn.save(update_fields=['book', 'form']) diff --git a/src/isbn/product_forms.py b/src/isbn/product_forms.py new file mode 100644 index 00000000..206ca1d2 --- /dev/null +++ b/src/isbn/product_forms.py @@ -0,0 +1,10 @@ +FORMS = [ + ('html', ('EC', 'E105')), + ('txt', ('EB', 'E112')), + ('pdf', ('EB', 'E107')), + ('epub', ('ED', 'E101')), + ('mobi', ('ED', 'E127')), + ('mp3', ('AN', 'A103')), + ('paperback', ('BC', '')), +] + diff --git a/src/isbn/templates/isbn/list.html b/src/isbn/templates/isbn/list.html new file mode 100644 index 00000000..e9f00121 --- /dev/null +++ b/src/isbn/templates/isbn/list.html @@ -0,0 +1,75 @@ +{% extends "documents/base.html" %} +{% load pagination_tags %} +{% load l10n %} + + +{% block content %} + {% localize off %} + + {% for pool in pools %} + + + + + + {% endfor %} +
+ {{ pool.get_purpose_display }} + + ({{ pool }}) + +
+ {% with p=pool.fill_percentage %} +
{% if p > 30 %}{{ pool.entries }} / {{ pool.size }}{% endif %}
+ {% if p <= 30 %}{{ pool.entries }} / {{ pool.size }}{% endif %} + {% endwith %} +
+
+ {% endlocalize %} + + {% autopaginate list 100 %} + + + + + + + + + + + {% for isbn in list %} + + + + + + + + + + {% endfor %} + +
ISBNKsiążkaFormaCzas
+ {{ isbn.get_code }} + + {{ isbn.book|default_if_none:'—' }} + + {{ isbn.form|default_if_none:'—' }} + + {{ isbn.datestamp|default_if_none:'—' }} + + {% if isbn.wl_data %} + WL + {% endif %} + + {% if isbn.bn_data %} + BN + {% endif %} + + {% if isbn.notes %} + not. + {% endif %} +
+ {% paginate %} +{% endblock %} diff --git a/src/isbn/urls.py b/src/isbn/urls.py new file mode 100644 index 00000000..a27781a7 --- /dev/null +++ b/src/isbn/urls.py @@ -0,0 +1,7 @@ +from django.urls import path +from . import views + + +urlpatterns = [ + path('', views.isbn_list, name='isbn_list'), +] diff --git a/src/isbn/views.py b/src/isbn/views.py new file mode 100644 index 00000000..70c101ac --- /dev/null +++ b/src/isbn/views.py @@ -0,0 +1,9 @@ +from django.shortcuts import render +from .models import Isbn, IsbnPool + + +def isbn_list(request): + return render(request, 'isbn/list.html', { + 'pools': IsbnPool.objects.all(), + 'list': Isbn.objects.all(), + }) diff --git a/src/redakcja/urls.py b/src/redakcja/urls.py index e4078851..6fb77fbe 100644 --- a/src/redakcja/urls.py +++ b/src/redakcja/urls.py @@ -27,6 +27,7 @@ urlpatterns = [ path('cover/', include('cover.urls')), path('depot/', include('depot.urls')), path('wlxml/', include('wlxml.urls')), + path('isbn/', include('isbn.urls')), path('api/', include('redakcja.api.urls')), ] -- 2.20.1