From 0251adf97488c11e936a714976261febfb8c5268 Mon Sep 17 00:00:00 2001 From: Radek Czajka Date: Fri, 23 Sep 2022 15:40:17 +0200 Subject: [PATCH] Catalogue: wikidata suggestions --- src/catalogue/models.py | 10 +- .../static/catalogue/wikidata_admin.css | 23 +++++ .../static/catalogue/wikidata_admin.js | 61 ++++++++++++ src/catalogue/urls.py | 2 + src/catalogue/views.py | 49 +++++++++ src/catalogue/wikidata.py | 99 +++++++++++++------ 6 files changed, 210 insertions(+), 34 deletions(-) create mode 100644 src/catalogue/static/catalogue/wikidata_admin.css create mode 100644 src/catalogue/static/catalogue/wikidata_admin.js diff --git a/src/catalogue/models.py b/src/catalogue/models.py index d671425d..b974de14 100644 --- a/src/catalogue/models.py +++ b/src/catalogue/models.py @@ -7,10 +7,10 @@ from django.utils.translation import gettext_lazy as _ from admin_ordering.models import OrderableModel from wikidata.client import Client from .constants import WIKIDATA -from .wikidata import WikidataMixin +from .wikidata import WikidataModel -class Author(WikidataMixin, models.Model): +class Author(WikidataModel): slug = models.SlugField(max_length=255, null=True, blank=True, unique=True) first_name = models.CharField(_("first name"), max_length=255, blank=True) last_name = models.CharField(_("last name"), max_length=255, blank=True) @@ -110,7 +110,7 @@ class NotableBook(OrderableModel): book = models.ForeignKey('Book', models.CASCADE) -class Category(WikidataMixin, models.Model): +class Category(WikidataModel): name = models.CharField(_("name"), max_length=255) slug = models.SlugField(max_length=255, unique=True) @@ -138,7 +138,7 @@ class Kind(Category): verbose_name_plural = _('kinds') -class Book(WikidataMixin, models.Model): +class Book(WikidataModel): slug = models.SlugField(max_length=255, blank=True, null=True, unique=True) authors = models.ManyToManyField(Author, blank=True, verbose_name=_("authors")) translators = models.ManyToManyField( @@ -325,7 +325,7 @@ class WorkRate(models.Model): return (decimal.Decimal(book.estimated_chars) / 1800 * self.per_normpage).quantize(decimal.Decimal('1.00'), rounding=decimal.ROUND_HALF_UP) -class Place(WikidataMixin, models.Model): +class Place(WikidataModel): name = models.CharField(_('name'), max_length=255, blank=True) locative = models.CharField(_('locative'), max_length=255, blank=True, help_text=_('in…')) diff --git a/src/catalogue/static/catalogue/wikidata_admin.css b/src/catalogue/static/catalogue/wikidata_admin.css new file mode 100644 index 00000000..92ac69d9 --- /dev/null +++ b/src/catalogue/static/catalogue/wikidata_admin.css @@ -0,0 +1,23 @@ +.wikidata-hint { + background-image: url('https://www.wikidata.org/static/favicon/wikidata.ico'); + background-repeat: no-repeat; + background-position: 2px 50%; + background-size: 16px auto; + padding: 2px 2px 2px 20px; + cursor: pointer; + color: black; + background-color: white; + border-radius: 10px; +} + +#id_wikidata { + transition: .2s background-position; + background-image: url('https://www.wikidata.org/static/favicon/wikidata.ico'); + background-size: 64px 64px; + background-repeat: no-repeat; + background-position: -64px 50%; +} +#id_wikidata.wikidata-processing { + background-position: 100% 50%; + transition: 10s background-position; +} diff --git a/src/catalogue/static/catalogue/wikidata_admin.js b/src/catalogue/static/catalogue/wikidata_admin.js new file mode 100644 index 00000000..2ba5a1c9 --- /dev/null +++ b/src/catalogue/static/catalogue/wikidata_admin.js @@ -0,0 +1,61 @@ +(function($) { + $(function() { + + let model = $('body').attr('class').match(/model-([^\s]*)/)[1]; + $("#id_wikidata").each(show_wikidata_hints).on('change', show_wikidata_hints); + + function show_wikidata_hints() { + $(".wikidata-hint").remove(); + $wdinput = $(this); + let qid = $wdinput.val(); + $wdinput.addClass('wikidata-processing'); + $.ajax( + '/catalogue/wikidata/' + model + '/' + qid, + { + success: function(result) { + for (att in result) { + let val = result[att]; + let $input = $("#id_" + att); + if (val && val != $input.val()) { + let el = $(''); + if (val.wd) { + el.on('click', function() { + set_value_from_wikidata_id( + $input, val.model, val.wd, + function() { + $(this).remove(); + } + ); + }); + el.text(val.label); + } else { + el.on('click', function() { + $input.val(val); + $(this).remove(); + }); + el.text(val); + } + $input.parent().append(el); + } + }; + + $wdinput.removeClass('wikidata-processing'); + }, + } + ); + } + + function set_value_from_wikidata_id($input, model, wikidata_id, callback) { + $.post({ + url: '/catalogue/wikidata/' + model + '/' + wikidata_id, + data: { + csrfmiddlewaretoken: $('[name=csrfmiddlewaretoken]').val(), + }, + success: function(result) { + $input.val(result.id); + callback(); + }, + }) + } + }); +})(jQuery); diff --git a/src/catalogue/urls.py b/src/catalogue/urls.py index 2b4301bf..d4fd6b6a 100644 --- a/src/catalogue/urls.py +++ b/src/catalogue/urls.py @@ -18,4 +18,6 @@ urlpatterns = [ path('terms/author/', views.AuthorTerms.as_view()), path('terms/editor/', views.EditorTerms.as_view()), + + path('wikidata//', views.WikidataView.as_view()), ] diff --git a/src/catalogue/views.py b/src/catalogue/views.py index 6d4a2249..65b29808 100644 --- a/src/catalogue/views.py +++ b/src/catalogue/views.py @@ -1,13 +1,19 @@ # This file is part of FNP-Redakcja, licensed under GNU Affero GPLv3 or later. # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information. # +from django.apps import apps from django.db.models import Prefetch +from django.http import Http404 +from django.utils.formats import localize_input from django.contrib.auth.models import User from django.views.generic import DetailView, TemplateView from . import models import documents.models from rest_framework.generics import ListAPIView from rest_framework.filters import SearchFilter +from rest_framework.permissions import IsAdminUser +from rest_framework.response import Response +from rest_framework.views import APIView from rest_framework import serializers @@ -86,3 +92,46 @@ class WLURITerms(Terms): class serializer_class(serializers.Serializer): label = serializers.CharField(source='wluri') + +class WikidataView(APIView): + permission_classes = [IsAdminUser] + + def get_object(self, model, qid, save): + try: + Model = apps.get_model('catalogue', model) + except LookupError: + raise Http404 + if not issubclass(Model, models.WikidataModel): + raise Http404 + + obj = Model.objects.filter(wikidata=qid).first() + if obj is None: + obj = Model(wikidata=qid) + if not obj.pk and save: + obj.save() + else: + obj.wikidata_populate(save=False) + d = { + "id": obj.pk, + } + for attname in dir(Model.Wikidata): + if attname.startswith("_"): + continue + for fieldname, lang in obj.wikidata_fields_for_attribute(attname): + d[fieldname] = getattr(obj, fieldname) + + if isinstance(d[fieldname], models.WikidataModel): + d[attname] = { + "model": type(d[fieldname])._meta.model_name, + "wd": d[fieldname].wikidata, + "label": str(d[fieldname]) or d[fieldname]._wikidata_label, + } + else: + d[fieldname] = localize_input(d[fieldname]) + return Response(d) + + def get(self, request, model, qid): + return self.get_object(model, qid, save=False) + + def post(self, request, model, qid): + return self.get_object(model, qid, save=True) diff --git a/src/catalogue/wikidata.py b/src/catalogue/wikidata.py index 610245d5..b5f3e94f 100644 --- a/src/catalogue/wikidata.py +++ b/src/catalogue/wikidata.py @@ -2,6 +2,7 @@ # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information. # from datetime import date +from django.conf import settings from django.db import models from django.db.models.signals import m2m_changed from django.utils.html import format_html @@ -9,9 +10,11 @@ from django.utils.translation import gettext_lazy as _ from django.dispatch import receiver from wikidata.client import Client from wikidata.datavalue import DatavalueError +from modeltranslation.translator import translator +from modeltranslation.settings import AVAILABLE_LANGUAGES -class WikidataMixin(models.Model): +class WikidataModel(models.Model): wikidata = models.CharField( max_length=255, blank=True, @@ -20,8 +23,8 @@ class WikidataMixin(models.Model): class Meta: abstract = True - - def wikidata_populate(self, client, entity, attname, wd): + + def wikidata_populate_field(self, client, entity, attname, wd, save, lang): model_field = self._meta.get_field(attname) if isinstance(model_field, models.ManyToManyField): if getattr(self, attname).all().exists(): @@ -32,53 +35,87 @@ class WikidataMixin(models.Model): wdvalue = None if wd == "description": - wdvalue = entity.description.get("pl", str(entity.description)) + wdvalue = entity.description.get(lang, str(entity.description)) elif wd == "label": - wdvalue = entity.label.get("pl", str(entity.label)) + wdvalue = entity.label.get(lang, str(entity.label)) else: try: + # TODO: lang? wdvalue = entity.get(client.get(wd)) except DatavalueError: pass - self.set_field_from_wikidata(attname, wdvalue) - + self.set_field_from_wikidata(attname, wdvalue, save=save) + + def wikidata_populate(self, save=True): + Wikidata = type(self).Wikidata + client = Client() + # Probably should getlist + entity = client.get(self.wikidata) + for attname in dir(Wikidata): + if attname.startswith("_"): + continue + wd = getattr(Wikidata, attname) + + self.wikidata_populate_attribute(client, entity, attname, wd, save=save) + if hasattr(Wikidata, '_supplement'): + for attname, wd in Wikidata._supplement(self): + self.wikidata_populate_attribute(client, entity, attname, wd, save=save) + + def wikidata_fields_for_attribute(self, attname): + field = getattr(type(self), attname) + if type(self) in translator._registry: + opts = translator.get_options_for_model(type(self)) + if attname in opts.fields: + tfields = opts.fields[attname] + for tf in tfields: + yield tf.name, tf.language + return + + yield attname, settings.LANGUAGE_CODE + + def wikidata_populate_attribute(self, client, entity, attname, wd, save): + for fieldname, lang in self.wikidata_fields_for_attribute(attname): + self.wikidata_populate_field(client, entity, fieldname, wd, save, lang) + def save(self, **kwargs): - super().save() - if self.wikidata and hasattr(self, "Wikidata"): - Wikidata = type(self).Wikidata - client = Client() - # Probably should getlist - entity = client.get(self.wikidata) - for attname in dir(Wikidata): - if attname.startswith("_"): - continue - wd = getattr(Wikidata, attname) - - self.wikidata_populate(client, entity, attname, wd) - if hasattr(Wikidata, '_supplement'): - for attname, wd in Wikidata._supplement(self): - self.wikidata_populate(client, entity, attname, wd) + am_new = self.pk is None + super().save() + if am_new and self.wikidata and hasattr(self, "Wikidata"): + self.wikidata_populate() kwargs.update(force_insert=False, force_update=True) super().save(**kwargs) - def set_field_from_wikidata(self, attname, wdvalue): + def set_field_from_wikidata(self, attname, wdvalue, save, language='pl'): if not wdvalue: return # Find out what this model field is model_field = self._meta.get_field(attname) if isinstance(model_field, models.ForeignKey): rel_model = model_field.related_model - if issubclass(rel_model, WikidataMixin): - # welp, we can try and find by WD identifier. - wdvalue, created = rel_model.objects.get_or_create(wikidata=wdvalue.id) + if issubclass(rel_model, WikidataModel): + label = wdvalue.label.get(language, str(wdvalue.label)) + try: + wdvalue = rel_model.objects.get(wikidata=wdvalue.id) + except rel_model.DoesNotExist: + wdvalue = rel_model(wikidata=wdvalue.id) + if save: + wdvalue.save() + wdvalue._wikidata_label = label setattr(self, attname, wdvalue) elif isinstance(model_field, models.ManyToManyField): rel_model = model_field.related_model - if issubclass(rel_model, WikidataMixin): - wdvalue, created = rel_model.objects.get_or_create(wikidata=wdvalue.id) + if issubclass(rel_model, WikidataModel): + label = wdvalue.label.get(language, str(wdvalue.label)) + try: + wdvalue = rel_model.objects.get(wikidata=wdvalue.id) + except rel_model.DoesNotExist: + wdvalue = rel_model(wikidata=wdvalue.id) + if save: + wdvalue.save() + wdvalue._wikidata_label = label getattr(self, attname).set([wdvalue]) else: # How to get original title? @@ -86,7 +123,7 @@ class WikidataMixin(models.Model): if isinstance(model_field, models.IntegerField): wdvalue = wdvalue.year elif not isinstance(wdvalue, str): - wdvalue = wdvalue.label.get("pl", str(wdvalue.label)) + wdvalue = wdvalue.label.get(language, str(wdvalue.label)) setattr(self, attname, wdvalue) def wikidata_link(self): @@ -102,6 +139,10 @@ class WikidataMixin(models.Model): class WikidataAdminMixin: + class Media: + css = {"screen": ("catalogue/wikidata_admin.css",)} + js = ("catalogue/wikidata_admin.js",) + def save_related(self, request, form, formsets, change): super().save_related(request, form, formsets, change) form.instance.save() -- 2.20.1