From: Radek Czajka Date: Sat, 11 Apr 2020 09:14:41 +0000 (+0200) Subject: Wikidata in catalogue. X-Git-Url: https://git.mdrn.pl/redakcja.git/commitdiff_plain/4bb5959d246fa0528b83fc3e21f9e5782ec34236?ds=sidebyside;hp=5c0ed2a6cb007fccf2cc4b58199f285707f7e974 Wikidata in catalogue. --- diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 9b00c430..313d092d 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -8,6 +8,7 @@ oauth2 httplib2 # oauth2 dependency python-slugify python-docx==0.8.10 +Wikidata==0.6.1 librarian==1.8.1 diff --git a/src/catalogue/admin.py b/src/catalogue/admin.py index 189cd718..bd783b99 100644 --- a/src/catalogue/admin.py +++ b/src/catalogue/admin.py @@ -1,17 +1,21 @@ from django.contrib import admin from . import models +from .wikidata import WikidataAdminMixin +class AuthorAdmin(WikidataAdminMixin, admin.ModelAdmin): + list_display = "first_name", "last_name", "notes" + search_fields = ["first_name", "last_name", "wikidata"] + prepopulated_fields = {"slug": ("first_name", "last_name")} -class AuthorAdmin(admin.ModelAdmin): - search_fields = ['name'] admin.site.register(models.Author, AuthorAdmin) -class BookAdmin(admin.ModelAdmin): - raw_id_fields = ['authors'] - autocomplete_fields = ['translators'] +class BookAdmin(WikidataAdminMixin, admin.ModelAdmin): + list_display = "title", "notes" + autocomplete_fields = ["authors", "translators"] + prepopulated_fields = {"slug": ("title",)} -admin.site.register(models.Book, BookAdmin) +admin.site.register(models.Book, BookAdmin) diff --git a/src/catalogue/constants.py b/src/catalogue/constants.py new file mode 100644 index 00000000..f8180f76 --- /dev/null +++ b/src/catalogue/constants.py @@ -0,0 +1,9 @@ +class WIKIDATA: + AUTHOR = "P50" + LANGUAGE = "P407" + DATE_OF_DEATH = "P570" + LAST_NAME = "P734" + GIVEN_NAME = "P735" + TRANSLATOR = "P655" + BASED_ON = "P629" + TITLE = "P1476" diff --git a/src/catalogue/migrations/0008_auto_20200410_1741.py b/src/catalogue/migrations/0008_auto_20200410_1741.py new file mode 100644 index 00000000..f83652ea --- /dev/null +++ b/src/catalogue/migrations/0008_auto_20200410_1741.py @@ -0,0 +1,71 @@ +# Generated by Django 3.0.4 on 2020-04-10 17:41 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('catalogue', '0007_auto_20200322_2326'), + ] + + operations = [ + migrations.RenameField( + model_name='author', + old_name='name', + new_name='last_name', + ), + migrations.RemoveField( + model_name='book', + name='translator', + ), + migrations.RemoveField( + model_name='book', + name='uri', + ), + migrations.AddField( + model_name='author', + name='first_name', + field=models.CharField(blank=True, max_length=255), + ), + migrations.AddField( + model_name='author', + name='notes', + field=models.TextField(blank=True), + ), + migrations.AddField( + model_name='author', + name='priority', + field=models.PositiveSmallIntegerField(choices=[(0, 'Low'), (1, 'Medium'), (2, 'High')], default=0), + ), + migrations.AddField( + model_name='author', + name='slug', + field=models.SlugField(blank=True, null=True, unique=True), + ), + migrations.AddField( + model_name='author', + name='wikidata', + field=models.CharField(blank=True, max_length=255), + ), + migrations.AddField( + model_name='book', + name='slug', + field=models.SlugField(blank=True, max_length=255, null=True, unique=True), + ), + migrations.AddField( + model_name='book', + name='translators', + field=models.ManyToManyField(blank=True, related_name='translated_book_set', related_query_name='translated_book', to='catalogue.Author'), + ), + migrations.AddField( + model_name='book', + name='wikidata', + field=models.CharField(blank=True, max_length=255), + ), + migrations.AlterField( + model_name='book', + name='authors', + field=models.ManyToManyField(blank=True, to='catalogue.Author'), + ), + ] diff --git a/src/catalogue/migrations/0009_auto_20200411_1114.py b/src/catalogue/migrations/0009_auto_20200411_1114.py new file mode 100644 index 00000000..f1dc1391 --- /dev/null +++ b/src/catalogue/migrations/0009_auto_20200411_1114.py @@ -0,0 +1,38 @@ +# Generated by Django 3.0.4 on 2020-04-11 11:14 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('catalogue', '0008_auto_20200410_1741'), + ] + + operations = [ + migrations.AlterField( + model_name='author', + name='last_name', + field=models.CharField(blank=True, max_length=255), + ), + migrations.AlterField( + model_name='author', + name='wikidata', + field=models.CharField(blank=True, help_text='If you have a Wikidata ID, like "Q1337", enter it and save.', max_length=255, null=True, unique=True), + ), + migrations.AlterField( + model_name='book', + name='language', + field=models.CharField(blank=True, max_length=3), + ), + migrations.AlterField( + model_name='book', + name='title', + field=models.CharField(blank=True, max_length=255), + ), + migrations.AlterField( + model_name='book', + name='wikidata', + field=models.CharField(blank=True, help_text='If you have a Wikidata ID, like "Q1337", enter it and save.', max_length=255, null=True, unique=True), + ), + ] diff --git a/src/catalogue/models.py b/src/catalogue/models.py index 4a236b62..c2ed3812 100644 --- a/src/catalogue/models.py +++ b/src/catalogue/models.py @@ -1,38 +1,67 @@ from django.db import models from django.utils.translation import gettext_lazy as _ +from .constants import WIKIDATA +from .wikidata import WikidataMixin -class Author(models.Model): - name = models.CharField(max_length=255) +class Author(WikidataMixin, models.Model): + slug = models.SlugField(null=True, blank=True, unique=True) + first_name = models.CharField(max_length=255, blank=True) + last_name = models.CharField(max_length=255, blank=True) year_of_death = models.SmallIntegerField(null=True, blank=True) - status = models.PositiveSmallIntegerField(null=True, blank=True, choices=[ - (1, _('Alive')), - (2, _('Dead')), - (3, _('Long dead')), - (4, _('Unknown')), - ]) + status = models.PositiveSmallIntegerField( + null=True, + blank=True, + choices=[ + (1, _("Alive")), + (2, _("Dead")), + (3, _("Long dead")), + (4, _("Unknown")), + ], + ) + notes = models.TextField(blank=True) + priority = models.PositiveSmallIntegerField( + default=0, choices=[(0, _("Low")), (1, _("Medium")), (2, _("High"))] + ) - def __str__(self): - return self.name + class Wikidata: + first_name = WIKIDATA.GIVEN_NAME + last_name = WIKIDATA.LAST_NAME + year_of_death = WIKIDATA.DATE_OF_DEATH + notes = "description" + def __str__(self): + return f"{self.first_name} {self.last_name}" -class Book(models.Model): - uri = models.CharField(max_length=255) +class Book(WikidataMixin, models.Model): + slug = models.SlugField(max_length=255, blank=True, null=True, unique=True) authors = models.ManyToManyField(Author, blank=True) - translators = models.ManyToManyField(Author, related_name='translated_book_set', related_query_name='translated_book', blank=True) - title = models.CharField(max_length=255) - language = models.CharField(max_length=3) - based_on = models.ForeignKey('self', models.PROTECT, related_name='translation', null=True, blank=True) - + translators = models.ManyToManyField( + Author, + related_name="translated_book_set", + related_query_name="translated_book", + blank=True, + ) + title = models.CharField(max_length=255, blank=True) + language = models.CharField(max_length=3, blank=True) + based_on = models.ForeignKey( + "self", models.PROTECT, related_name="translation", null=True, blank=True + ) scans_source = models.CharField(max_length=255, blank=True) text_source = models.CharField(max_length=255, blank=True) notes = models.TextField(blank=True) - priority = models.PositiveSmallIntegerField(default=0, choices=[ - (0, _('Low')), - (1, _('Medium')), - (2, _('High')), - ]) + priority = models.PositiveSmallIntegerField( + default=0, choices=[(0, _("Low")), (1, _("Medium")), (2, _("High"))] + ) + + class Wikidata: + authors = WIKIDATA.AUTHOR + translators = WIKIDATA.TRANSLATOR + title = WIKIDATA.TITLE + language = WIKIDATA.LANGUAGE + based_on = WIKIDATA.BASED_ON + notes = "description" def __str__(self): return self.title diff --git a/src/catalogue/wikidata.py b/src/catalogue/wikidata.py new file mode 100644 index 00000000..4e63c095 --- /dev/null +++ b/src/catalogue/wikidata.py @@ -0,0 +1,87 @@ +from datetime import date +from django.db import models +from django.db.models.signals import m2m_changed +from django.utils.translation import gettext_lazy as _ +from django.dispatch import receiver +from wikidata.client import Client +from wikidata.datavalue import DatavalueError + + +class WikidataMixin(models.Model): + wikidata = models.CharField( + max_length=255, + null=True, + blank=True, + unique=True, + help_text=_('If you have a Wikidata ID, like "Q1337", enter it and save.'), + ) + + class Meta: + abstract = True + + def save(self, **kwargs): + super().save() + if self.wikidata and hasattr(self, "Wikidata"): + Wikidata = type(self).Wikidata + client = Client() + # Probably should getlist + entity = client.get(self.wikidata) + for attname in dir(Wikidata): + if attname.startswith("_"): + continue + wd = getattr(Wikidata, attname) + + model_field = self._meta.get_field(attname) + if isinstance(model_field, models.ManyToManyField): + if getattr(self, attname).all().exists(): + continue + else: + if getattr(self, attname): + continue + + wdvalue = None + if wd == "description": + wdvalue = entity.description.get("pl", str(entity.description)) + elif wd == "label": + wdvalue = entity.label.get("pl", str(entity.label)) + else: + try: + wdvalue = entity.get(client.get(wd)) + except DatavalueError: + pass + + self.set_field_from_wikidata(attname, wdvalue) + + kwargs.update(force_insert=False, force_update=True) + super().save(**kwargs) + + def set_field_from_wikidata(self, attname, wdvalue): + if not wdvalue: + return + # Find out what this model field is + model_field = self._meta.get_field(attname) + if isinstance(model_field, models.ForeignKey): + rel_model = model_field.related_model + if issubclass(rel_model, WikidataMixin): + # welp, we can try and find by WD identifier. + wdvalue, created = rel_model.objects.get_or_create(wikidata=wdvalue.id) + setattr(self, attname, wdvalue) + elif isinstance(model_field, models.ManyToManyField): + rel_model = model_field.related_model + if issubclass(rel_model, WikidataMixin): + wdvalue, created = rel_model.objects.get_or_create(wikidata=wdvalue.id) + getattr(self, attname).set([wdvalue]) + else: + # How to get original title? + if isinstance(wdvalue, date): + if isinstance(model_field, models.IntegerField): + wdvalue = wdvalue.year + elif not isinstance(wdvalue, str): + wdvalue = wdvalue.label.get("pl", str(wdvalue.label)) + setattr(self, attname, wdvalue) + + +class WikidataAdminMixin: + def save_related(self, request, form, formsets, change): + super().save_related(request, form, formsets, change) + form.instance.save()