From 4b200d823f0ec79279da035a46d0d888693ca2fe Mon Sep 17 00:00:00 2001
From: Radek Czajka <rczajka@rczajka.pl>
Date: Tue, 27 Sep 2022 10:14:48 +0200
Subject: [PATCH] Author photos and nicer Wikidata imports.

---
 requirements/requirements.txt                 |  2 +-
 src/catalogue/admin.py                        |  2 +
 src/catalogue/constants.py                    |  1 +
 ...photo_author_photo_attribution_and_more.py | 33 +++++++++
 src/catalogue/models.py                       | 10 +++
 .../static/catalogue/wikidata_admin.css       | 23 ++++--
 .../static/catalogue/wikidata_admin.js        | 50 ++++++++++---
 src/catalogue/urls.py                         |  1 +
 src/catalogue/views.py                        | 66 ++++++++++++++---
 src/catalogue/wikidata.py                     | 69 +++++++++++++-----
 src/catalogue/wikimedia.py                    | 70 +++++++++++++++++++
 11 files changed, 286 insertions(+), 41 deletions(-)
 create mode 100644 src/catalogue/migrations/0039_author_photo_author_photo_attribution_and_more.py
 create mode 100644 src/catalogue/wikimedia.py

diff --git a/requirements/requirements.txt b/requirements/requirements.txt
index eba62b92..519d5745 100644
--- a/requirements/requirements.txt
+++ b/requirements/requirements.txt
@@ -8,7 +8,7 @@ oauth2
 httplib2 # oauth2 dependency
 python-slugify
 python-docx==0.8.11
-Wikidata==0.6.1
+Wikidata==0.7
 
 librarian==2.4.8
 
diff --git a/src/catalogue/admin.py b/src/catalogue/admin.py
index 087bfd74..f00e7fc9 100644
--- a/src/catalogue/admin.py
+++ b/src/catalogue/admin.py
@@ -67,6 +67,8 @@ class AuthorAdmin(WikidataAdminMixin, TabbedTranslationAdmin):
                     "notes",
                     "gazeta_link",
                     "culturepl_link",
+                    "plwiki",
+                    "photo", "photo_source", "photo_attribution",
                 ]
             },
         ),
diff --git a/src/catalogue/constants.py b/src/catalogue/constants.py
index 0acc116a..1faee93a 100644
--- a/src/catalogue/constants.py
+++ b/src/catalogue/constants.py
@@ -2,6 +2,7 @@
 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
 #
 class WIKIDATA:
+    IMAGE = 'P18'
     PLACE_OF_BIRTH = 'P19'
     PLACE_OF_DEATH = 'P20'
     GENDER = "P21"
diff --git a/src/catalogue/migrations/0039_author_photo_author_photo_attribution_and_more.py b/src/catalogue/migrations/0039_author_photo_author_photo_attribution_and_more.py
new file mode 100644
index 00000000..2f94a562
--- /dev/null
+++ b/src/catalogue/migrations/0039_author_photo_author_photo_attribution_and_more.py
@@ -0,0 +1,33 @@
+# Generated by Django 4.0.6 on 2022-09-26 16:21
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('catalogue', '0038_book_original_year'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='author',
+            name='photo',
+            field=models.ImageField(blank=True, null=True, upload_to='catalogue/author/'),
+        ),
+        migrations.AddField(
+            model_name='author',
+            name='photo_attribution',
+            field=models.CharField(blank=True, max_length=255),
+        ),
+        migrations.AddField(
+            model_name='author',
+            name='photo_source',
+            field=models.CharField(blank=True, max_length=255),
+        ),
+        migrations.AddField(
+            model_name='author',
+            name='plwiki',
+            field=models.CharField(blank=True, max_length=255),
+        ),
+    ]
diff --git a/src/catalogue/models.py b/src/catalogue/models.py
index 92275ecf..db840da5 100644
--- a/src/catalogue/models.py
+++ b/src/catalogue/models.py
@@ -9,6 +9,7 @@ from admin_ordering.models import OrderableModel
 from wikidata.client import Client
 from .constants import WIKIDATA
 from .wikidata import WikidataModel
+from .wikimedia import WikiMedia
 
 
 class Author(WikidataModel):
@@ -51,8 +52,13 @@ class Author(WikidataModel):
         ],
     )
     notes = models.TextField(_("notes"), blank=True)
+
     gazeta_link = models.CharField(_("gazeta link"), max_length=255, blank=True)
     culturepl_link = models.CharField(_("culture.pl link"), max_length=255, blank=True)
+    plwiki = models.CharField(blank=True, max_length=255)
+    photo = models.ImageField(blank=True, null=True, upload_to='catalogue/author/')
+    photo_source = models.CharField(blank=True, max_length=255)
+    photo_attribution = models.CharField(max_length=255, blank=True)
 
     description = models.TextField(_("description"), blank=True)
 
@@ -78,6 +84,10 @@ class Author(WikidataModel):
         place_of_death = WIKIDATA.PLACE_OF_DEATH
         gender = WIKIDATA.GENDER
         notes = "description"
+        plwiki = "plwiki"
+        photo = WikiMedia.download(WIKIDATA.IMAGE)
+        photo_source = WikiMedia.descriptionurl(WIKIDATA.IMAGE)
+        photo_attribution = WikiMedia.attribution(WIKIDATA.IMAGE)
 
         def _supplement(obj):
             if not obj.first_name and not obj.last_name:
diff --git a/src/catalogue/static/catalogue/wikidata_admin.css b/src/catalogue/static/catalogue/wikidata_admin.css
index 92ac69d9..24e27de0 100644
--- a/src/catalogue/static/catalogue/wikidata_admin.css
+++ b/src/catalogue/static/catalogue/wikidata_admin.css
@@ -1,13 +1,26 @@
 .wikidata-hint {
     background-image: url('https://www.wikidata.org/static/favicon/wikidata.ico');
     background-repeat: no-repeat;
-    background-position: 2px 50%;
+    background-position: 5px 50%;
     background-size: 16px auto;
-    padding: 2px 2px 2px 20px;
+    padding: 5px 5px 5px 26px;
     cursor: pointer;
-    color: black;
-    background-color: white;
+    background-color: black;
+    color: white;
     border-radius: 10px;
+    display: inline-block;
+    overflow: hidden;
+    vertical-align: middle;
+}
+@media (prefers-color-scheme: dark) {
+    .wikidata-hint {
+        color: black;
+        background-color: white;
+    }
+}
+.wikidata-hint img {
+    height: 48px;
+    margin: -5px -5px -5px 5px;
 }
 
 #id_wikidata {
@@ -21,3 +34,5 @@
     background-position: 100% 50%;
     transition: 10s background-position;
 }
+
+
diff --git a/src/catalogue/static/catalogue/wikidata_admin.js b/src/catalogue/static/catalogue/wikidata_admin.js
index a8aa51ee..f13510bc 100644
--- a/src/catalogue/static/catalogue/wikidata_admin.js
+++ b/src/catalogue/static/catalogue/wikidata_admin.js
@@ -18,25 +18,44 @@
                             let val = result[att];
                             let $input = $("#id_" + att);
                             if (val && val != $input.val()) {
+                                let already_set = false;
                                 let el = $('<span class="wikidata-hint">');
+
                                 if (val.wd) {
+                                    if (val.id && val.id == $input.val()) {
+                                        already_set = true;
+                                    } else {
+                                        // A representation of a WD Entity.
+                                        el.on('click', function() {
+                                            set_value_from_wikidata_id(
+                                                $input, val.model, val.wd,
+                                                () => {$(this).remove();}
+                                            );
+                                        });
+                                        el.text(val.label);
+                                    }
+                                } else if (val.img) {
+                                    // A downloadable remote image.
+                                    let img = $('<img height="32">');
+                                    img.attr('src', val.img);
+                                    el.append(img);
                                     el.on('click', function() {
-                                        set_value_from_wikidata_id(
-                                            $input, val.model, val.wd,
-                                            function() {
-                                                $(this).remove();
-                                            }
+                                        set_file_from_url(
+                                            $input, val.download,
+                                            () => {$(this).remove();}
                                         );
                                     });
-                                    el.text(val.label);
                                 } else {
+                                    // A plain literal.
                                     el.on('click', function() {
                                         $input.val(val);
                                         $(this).remove();
                                     });
                                     el.text(val);
                                 }
-                                $input.parent().append(el);
+                                if (!already_set) {
+                                    $input.parent().append(el);
+                                }
                             }
                         };
 
@@ -55,10 +74,25 @@
                     csrfmiddlewaretoken: $('[name=csrfmiddlewaretoken]').val(),
                 },
                 success: function(result) {
-                    $input.val(result.id);
+                    $input.append($('<option>').attr('value', result.id).text(result.__str__));                   
+                    $input.val(result.id).trigger('change');
                     callback();
                 },
             })
         }
+
+        function set_file_from_url($input, url, callback) {
+            filename = decodeURIComponent(url.match(/.*\/(.*)/)[1]);
+            $.ajax({
+                url: url,
+                success: function(content) {
+                    let file = new File([content], filename);
+                    let container = new DataTransfer(); 
+                    container.items.add(file);
+                    $input[0].files = container.files;
+                    callback()
+                }
+            });
+        }
     });
 })(jQuery);
diff --git a/src/catalogue/urls.py b/src/catalogue/urls.py
index b7625388..4eaf7b07 100644
--- a/src/catalogue/urls.py
+++ b/src/catalogue/urls.py
@@ -9,6 +9,7 @@ urlpatterns = [
     path("", views.CatalogueView.as_view(), name="catalogue"),
     path("author/<slug:slug>/", views.AuthorView.as_view(), name="catalogue_author"),
     path("book/<slug:slug>/", views.BookView.as_view(), name="catalogue_book"),
+    path("book/<slug:slug>.json", views.BookAPIView.as_view(), name="catalogue_book_api"),
 
     path('terms/epoch/', views.EpochTerms.as_view()),
     path('terms/kind/', views.KindTerms.as_view()),
diff --git a/src/catalogue/views.py b/src/catalogue/views.py
index c507aca4..376d3287 100644
--- a/src/catalogue/views.py
+++ b/src/catalogue/views.py
@@ -14,7 +14,7 @@ from django.views.generic import DetailView, TemplateView
 import apiclient
 from . import models
 import documents.models
-from rest_framework.generics import ListAPIView
+from rest_framework.generics import ListAPIView, RetrieveAPIView
 from rest_framework.filters import SearchFilter
 from rest_framework.permissions import IsAdminUser
 from rest_framework.response import Response
@@ -49,6 +49,47 @@ class AuthorView(TemplateView):
 class BookView(DetailView):
     model = models.Book
 
+class BookAPIView(RetrieveAPIView):
+    queryset = models.Book.objects.all()
+    lookup_field = 'slug'
+
+    class serializer_class(serializers.ModelSerializer):
+        class AuthorSerializer(serializers.ModelSerializer):
+            literal = serializers.CharField(source='name')
+
+            class Meta:
+                model = models.Author
+                fields = ['literal']
+
+        def category_serializer(m):
+            class CategorySerializer(serializers.ModelSerializer):
+                literal = serializers.CharField(source='name')
+                class Meta:
+                    model = m
+                    fields = ['literal']
+            return CategorySerializer
+
+        authors = AuthorSerializer(many=True)
+        translators = AuthorSerializer(many=True)
+        epochs = category_serializer(models.Epoch)(many=True)
+        kinds = category_serializer(models.Kind)(many=True)
+        genres = category_serializer(models.Genre)(many=True)
+
+        class Meta:
+            model = models.Book
+            fields = [
+                'title',
+                'authors',
+                'translators',
+                'epochs',
+                'kinds',
+                'genres',
+                'scans_source',
+                'text_source',
+                'original_year',
+                'pd_year',
+            ]
+    
 
 class TermSearchFilter(SearchFilter):
     search_param = 'term'
@@ -115,9 +156,10 @@ class WikidataView(APIView):
         if not obj.pk and save:
             obj.save()
         else:
-            obj.wikidata_populate(save=False)
+            obj.wikidata_populate(save=False, force=True)
         d = {
             "id": obj.pk,
+            "__str__": str(obj),
         }
         for attname in dir(Model.Wikidata):
             if attname.startswith("_"):
@@ -126,18 +168,24 @@ class WikidataView(APIView):
                 d[fieldname] = getattr(obj, fieldname)
 
                 if isinstance(d[fieldname], models.WikidataModel):
-                    d[attname] = {
+                    d[fieldname] = {
                         "model": type(d[fieldname])._meta.model_name,
+                        "id": d[fieldname].pk,
                         "wd": d[fieldname].wikidata,
                         "label": str(d[fieldname]) or d[fieldname]._wikidata_label,
                     }
                 elif hasattr(d[fieldname], 'all'):
-                    d[attname] = [
-                            {"model": type(item)._meta.model_name,
-                                "wd": item.wikidata,
-                                "label": str(item) or item._wikidata_label
-                                } for item in d[attname].all()
-                            ]
+                    d[fieldname] = [
+                        {
+                            "model": type(item)._meta.model_name,
+                            "wd": item.wikidata,
+                            "label": str(item) or item._wikidata_label
+                        } for item in d[fieldname].all()
+                    ]
+                elif hasattr(d[fieldname], 'as_hint_json'):
+                    d[fieldname] = d[fieldname].as_hint_json()
+                elif hasattr(d[fieldname], 'storage'):
+                    d[fieldname] = d[fieldname].url if d[fieldname] else None
                 else:
                     d[fieldname] = localize_input(d[fieldname])
         return Response(d)
diff --git a/src/catalogue/wikidata.py b/src/catalogue/wikidata.py
index 3ec843a7..90f1a5cb 100644
--- a/src/catalogue/wikidata.py
+++ b/src/catalogue/wikidata.py
@@ -7,11 +7,11 @@ from django.db import models
 from django.db.models.signals import m2m_changed
 from django.utils.html import format_html
 from django.utils.translation import gettext_lazy as _
-from django.dispatch import receiver
 from wikidata.client import Client
 from wikidata.datavalue import DatavalueError
 from modeltranslation.translator import translator
 from modeltranslation.settings import AVAILABLE_LANGUAGES
+from .wikimedia import Downloadable
 
 
 class WikidataModel(models.Model):
@@ -23,31 +23,52 @@ class WikidataModel(models.Model):
 
     class Meta:
         abstract = True
-        
-    def wikidata_populate_field(self, client, entity, attname, wd, save, lang):
-        model_field = self._meta.get_field(attname)
-        if isinstance(model_field, models.ManyToManyField):
-            if getattr(self, attname).all().exists():
-                return
-        else:
-            if getattr(self, attname):
-                return
 
+    def get_wikidata_property(self, client, entity, wd, lang):
         wdvalue = None
-        if wd == "description":
+
+        if callable(wd):
+            return wd(
+                lambda next_arg:
+                self.get_wikidata_property(
+                    client, entity, next_arg, lang
+                )
+            )
+        elif wd == "description":
             wdvalue = entity.description.get(lang, str(entity.description))
         elif wd == "label":
             wdvalue = entity.label.get(lang, str(entity.label))
-        else:
+        elif wd[0] == 'P':
             try:
                 # TODO: lang?
                 wdvalue = entity.get(client.get(wd))
             except DatavalueError:
                 pass
+        else:
+            try:
+                # wiki links identified as 'plwiki' etc.
+                wdvalue = entity.attributes['sitelinks'][wd]['url']
+            except KeyError:
+                pass
 
+        return wdvalue
+        
+        
+    def wikidata_populate_field(self, client, entity, attname, wd, save, lang, force=False):
+        if not force:
+            model_field = self._meta.get_field(attname)
+            if isinstance(model_field, models.ManyToManyField):
+                if getattr(self, attname).all().exists():
+                    return
+            else:
+                if getattr(self, attname):
+                    return
+
+        wdvalue = self.get_wikidata_property(client, entity, wd, lang)
+            
         self.set_field_from_wikidata(attname, wdvalue, save=save)
 
-    def wikidata_populate(self, save=True):
+    def wikidata_populate(self, save=True, force=False):
         Wikidata = type(self).Wikidata
         client = Client()
         # Probably should getlist
@@ -57,10 +78,10 @@ class WikidataModel(models.Model):
                 continue
             wd = getattr(Wikidata, attname)
 
-            self.wikidata_populate_attribute(client, entity, attname, wd, save=save)
+            self.wikidata_populate_attribute(client, entity, attname, wd, save=save, force=force)
         if hasattr(Wikidata, '_supplement'):
             for attname, wd in Wikidata._supplement(self):
-                self.wikidata_populate_attribute(client, entity, attname, wd, save=save)
+                self.wikidata_populate_attribute(client, entity, attname, wd, save=save, force=force)
 
     def wikidata_fields_for_attribute(self, attname):
         field = getattr(type(self), attname)
@@ -78,9 +99,9 @@ class WikidataModel(models.Model):
 
         yield attname, settings.LANGUAGE_CODE
 
-    def wikidata_populate_attribute(self, client, entity, attname, wd, save):
+    def wikidata_populate_attribute(self, client, entity, attname, wd, save, force=False):
         for fieldname, lang in self.wikidata_fields_for_attribute(attname):
-            self.wikidata_populate_field(client, entity, fieldname, wd, save, lang)
+            self.wikidata_populate_field(client, entity, fieldname, wd, save, lang, force=force)
                 
     def save(self, **kwargs):
         am_new = self.pk is None
@@ -97,6 +118,7 @@ class WikidataModel(models.Model):
             return
         # Find out what this model field is
         model_field = self._meta.get_field(attname)
+        skip_set = False
         if isinstance(model_field, models.ForeignKey):
             rel_model = model_field.related_model
             if issubclass(rel_model, WikidataModel):
@@ -126,9 +148,18 @@ class WikidataModel(models.Model):
             if isinstance(wdvalue, date):
                 if isinstance(model_field, models.IntegerField):
                     wdvalue = wdvalue.year
-            elif not isinstance(wdvalue, str):
+
+            # If downloadable (and not save)?
+            elif isinstance(wdvalue, Downloadable):
+                if save:
+                    wdvalue.apply_to_field(self, attname)
+                    skip_set = True
+
+            elif hasattr(wdvalue, 'label'):
                 wdvalue = wdvalue.label.get(language, str(wdvalue.label))
-            setattr(self, attname, wdvalue)
+
+            if not skip_set:
+                setattr(self, attname, wdvalue)
 
     def wikidata_link(self):
         if self.wikidata:
diff --git a/src/catalogue/wikimedia.py b/src/catalogue/wikimedia.py
new file mode 100644
index 00000000..534ca17a
--- /dev/null
+++ b/src/catalogue/wikimedia.py
@@ -0,0 +1,70 @@
+from urllib.parse import unquote
+from django.core.files.base import ContentFile
+from cover.utils import get_wikimedia_data, URLOpener
+
+
+class WikiMedia:
+    def get_description_url(imgdata):
+        if imgdata is None:
+            return None
+        return imgdata.attributes['imageinfo'][0]['descriptionurl']
+
+    @classmethod
+    def descriptionurl(cls, arg):
+        def transform(get_value):
+            value = get_value(arg)
+            if value is None:
+                return None
+            return cls.get_description_url(value)
+        return transform
+
+    @classmethod
+    def attribution(cls, arg):
+        def transform(get_value):
+            value = get_value(arg)
+            if value is None:
+                return None
+            media_data = get_wikimedia_data(
+                cls.get_description_url(value)
+            )
+            parts = [
+                media_data['title'],
+                media_data['author'],
+                media_data['license_name'],
+            ]
+            parts = [p for p in parts if p]
+            attribution = ', '.join(parts)
+            return attribution
+        return transform
+
+    @classmethod
+    def download(cls, arg):
+        def transform(get_value):
+            value = get_value(arg)
+            if value is None:
+                return None
+            media_data = get_wikimedia_data(
+                cls.get_description_url(value)
+            )
+            download_url = media_data['download_url']
+            return Downloadable(download_url)
+        return transform
+
+
+class Downloadable:
+    def __init__(self, url):
+        self.url = url
+
+    def apply_to_field(self, obj, attname):
+        t = URLOpener().open(self.url).read()
+        getattr(obj, attname).save(
+            unquote(self.url.rsplit('/', 1)[-1]),
+            ContentFile(t),
+            save=False
+        )
+
+    def as_hint_json(self):
+        return {
+            'download': self.url,
+            'img': self.url,
+        }
-- 
2.20.1