From 52d79280796679d28fb526063c77c43851ae52c4 Mon Sep 17 00:00:00 2001 From: Radek Czajka Date: Fri, 25 Feb 2022 14:20:55 +0100 Subject: [PATCH 1/1] Importing covers from WikiCommons, MNW. --- src/cover/forms.py | 16 ++--- src/cover/locale/pl/LC_MESSAGES/django.mo | Bin 1502 -> 1828 bytes src/cover/locale/pl/LC_MESSAGES/django.po | 42 ++++++----- src/cover/templates/cover/add_image.html | 4 +- src/cover/utils.py | 81 ++++++++++++++++++++-- src/cover/views.py | 6 +- 6 files changed, 114 insertions(+), 35 deletions(-) diff --git a/src/cover/forms.py b/src/cover/forms.py index ea51e46b..c4f06fad 100644 --- a/src/cover/forms.py +++ b/src/cover/forms.py @@ -10,7 +10,7 @@ from cover.models import Image from django.utils.safestring import mark_safe from PIL import Image as PILImage -from cover.utils import get_flickr_data, FlickrError, URLOpener +from cover.utils import get_import_data, FlickrError, URLOpener class ImageAddForm(forms.ModelForm): @@ -41,8 +41,8 @@ class ImageAddForm(forms.ModelForm): same_source = Image.objects.filter(source_url=source_url) if same_source: raise forms.ValidationError(mark_safe( - ugettext('Image already in repository' - % same_source.first().get_absolute_url()))) + ugettext('Image already in repository.') + % {'url': same_source.first().get_absolute_url()})) return source_url def clean(self): @@ -91,15 +91,15 @@ class ReadonlyImageEditForm(ImageEditForm): raise AssertionError("ReadonlyImageEditForm should not be saved.") -class FlickrForm(forms.Form): - source_url = forms.URLField(label=_('Flickr URL')) +class ImportForm(forms.Form): + source_url = forms.URLField(label=_('WikiCommons, MNW or Flickr URL')) def clean_source_url(self): url = self.cleaned_data['source_url'] try: - flickr_data = get_flickr_data(url) + import_data = get_import_data(url) except FlickrError as e: raise forms.ValidationError(e) for field_name in ('license_url', 'license_name', 'author', 'title', 'download_url'): - self.cleaned_data[field_name] = flickr_data[field_name] - return flickr_data['source_url'] + self.cleaned_data[field_name] = import_data[field_name] + return import_data['source_url'] diff --git a/src/cover/locale/pl/LC_MESSAGES/django.mo b/src/cover/locale/pl/LC_MESSAGES/django.mo index 1c297fe04ceca54bd840ece7384be86ab5307166..756253c1b2dbceb7c17b71df344b8eb2ecb212b4 100644 GIT binary patch literal 1828 zcma)*O>7%Q6vwB~@?ky-<)a(`Jq4vCkagB>D@9Hmpl*YLk~Bm~Q*i)Ib|=o*yEE?0 zZe5$4BE%IgAn_q_LgL1OThs$m@f9I)Mcla~&ivonp(aJZNVC70_cd?c?ECf1$xj5v zbC}O#{(?D)`Q-_GFv9zUco?jJ4}$aH17HJu1bhp86ubsj!3dlIKLtbZ4mbh+4xRze z-Y>*S@D=bOa29+KoC7)THpp?`1vzdDWcx12{`({SM$z6!cN+6qOg`&q za9s8oA7{rloDa%S49~jBmAhoxPb|vBDH-=j8*-{+z0R!bEzHWr@;xs9 zW<4^_tRj1pU3;rgZ~j(d`IBVXFuSP?SM_N)`pSWU*(UhL{biK2`Y z+pbACBlg4T$W{GqZDgv1x4DasP^dGVA;!A1Efrb4F4QXFt6?P!ODoc6!KRbOClYT`qYW9W zHZ}UP)t0p2WNe}7mZL)Oy0Kjw^_4fWV-w$&W$T*dPO^wNrkUgIYt%_ftIMlb?xt8N zhoy!!nKC%3*MUiE>uF=-!J5B z6HddY0-x&ieCc}drZUP6>#=TCtTQ@~OB~%X*i(r1NyxIYsy}FLciT zap+4*<=~)qym?HIob9Jtx>4_}+a0Z3pZfRo4#{N4j(R|fHLa+NpP?^}9abY3mz->i zt-~|&o ZwH+L~JKx2F16x=f?WD+hSw1)re*!Wt&+h;L delta 768 zcmZ|MKWGzC9Ki8snnX=a)wWL6@^-m!NfZAZBBr&L+B1nUDLCoHobhOyE0@z^2Qvub zA_zwZ7Z)7_7Y7$X5TSzNBnskE>g=dfC%?a>#mOJ{-Y2>Ddw+iKo{ww}_cll4>xwu+ zJ4)N29iz4P&=Fg>AAjOr{Dn#Ujl;N&2QV2^DurX1!$~}ZmobhF+=nfc`76l$o?7LH zEN~ws@iC6#`XE8Q*cpF~GVwjiMjtVOU-1iv_x7=)G0r=oYw4Vs%^POU$Q5*U%F3NSZe~rBM5F57 z@w)%Zn=%J^w$lxbS`bad=O>Js>nvN^E|%u%a+Wi;eXi|g{GfF pq$y0V?G)ogPZB3}#cz0Su=ek*nhu;LRsZnzZs_V&D|(uI@E3>Kc)S1r diff --git a/src/cover/locale/pl/LC_MESSAGES/django.po b/src/cover/locale/pl/LC_MESSAGES/django.po index d40bd54a..5418ccf5 100644 --- a/src/cover/locale/pl/LC_MESSAGES/django.po +++ b/src/cover/locale/pl/LC_MESSAGES/django.po @@ -5,42 +5,36 @@ # msgid "" msgstr "" -"Project-Id-Version: PACKAGE VERSION\n" +"Project-Id-Version: \n" "Report-Msgid-Bugs-To: \n" -"PO-Revision-Date: 2014-02-25 13:30+0100\n" +"PO-Revision-Date: 2022-02-25 14:10+0100\n" "Last-Translator: Radek Czajka \n" -"Language-Team: LANGUAGE \n" -"Language: \n" +"Language-Team: \n" +"Language: pl\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" "Plural-Forms: nplurals=3; plural=(n==1 ? 0 : n%10>=2 && n%10<=4 && (n%100<10 " "|| n%100>=20) ? 1 : 2);\n" -"X-Generator: Poedit 1.5.4\n" +"X-Generator: Poedit 3.0\n" -#: cover/forms.py:34 +#: cover/forms.py:34 cover/forms.py:44 #, python-format msgid "Image already in repository." msgstr "Obraz jest już w repozytorium." -#: cover/forms.py:44 -#, fuzzy, python-format -#| msgid "Image already in repository." -msgid "Image already in repository" -msgstr "Obraz jest już w repozytorium." - #: cover/forms.py:53 msgid "No image specified" -msgstr "" +msgstr "Nie wskazano obrazu" #: cover/forms.py:61 cover/forms.py:78 #, python-format msgid "Image too small: %sx%s, minimal dimensions %sx%s" -msgstr "" +msgstr "Obraz za mały: %sx%s, minimalny rozmiar %sx%s" #: cover/forms.py:95 -msgid "Flickr URL" -msgstr "URL z Flickra" +msgid "WikiCommons, MNW or Flickr URL" +msgstr "URL z WikiCommons, MNW albo Flickra" #: cover/models.py:23 msgid "title" @@ -85,8 +79,8 @@ msgid "Add image" msgstr "Dodaj obrazek" #: cover/templates/cover/add_image.html:19 -msgid "Load from Flickr" -msgstr "Pobierz z Flickra" +msgid "Load from WikiCommons, MNW, Flickr" +msgstr "Pobierz z WikiCommons, MNW albo Flickra" #: cover/templates/cover/image_detail.html:6 #: cover/templates/cover/image_detail.html:9 @@ -117,3 +111,15 @@ msgstr "Obrazki na okładki" #: cover/templates/cover/image_list.html:15 msgid "Add new" msgstr "Dodaj nowy" + +#, fuzzy, python-format +#~| msgid "Image already in repository." +#~ msgid "Image already in repository" +#~ msgstr "Obraz jest już w repozytorium." + +#, python-format +#~ msgid "Image already in repository" +#~ msgstr "Obraz jest już w repozytorium." + +#~ msgid "Flickr URL" +#~ msgstr "URL z Flickra" diff --git a/src/cover/templates/cover/add_image.html b/src/cover/templates/cover/add_image.html index 56f5665e..092933e8 100644 --- a/src/cover/templates/cover/add_image.html +++ b/src/cover/templates/cover/add_image.html @@ -12,11 +12,11 @@
{% csrf_token %} - + {% bootstrap_form ff %} {% buttons %} - + {% endbuttons %}
diff --git a/src/cover/utils.py b/src/cover/utils.py index d13554fa..3bf3e9f9 100644 --- a/src/cover/utils.py +++ b/src/cover/utils.py @@ -1,17 +1,16 @@ # This file is part of FNP-Redakcja, licensed under GNU Affero GPLv3 or later. # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information. # +import csv +from io import StringIO import json import re from urllib.request import FancyURLopener -from django.contrib.sites.models import Site - - class URLOpener(FancyURLopener): @property def version(self): - return 'FNP Redakcja (http://%s)' % Site.objects.get_current() + return 'FNP Redakcja' class FlickrError(Exception): @@ -69,3 +68,77 @@ def get_flickr_data(url): 'title': title, 'download_url': download_url, } + + +def get_wikimedia_data(url): + """ + >>> get_wikimedia_data('https://commons.wikimedia.org/wiki/File:Valdai_IverskyMon_asv2018_img47.jpg') + {'title': 'Valdai IverskyMon asv2018 img47', 'author': 'A.Savin', 'source_url': 'https://commons.wikimedia.org/wiki/File:Valdai_IverskyMon_asv2018_img47.jpg', 'download_url': 'https://upload.wikimedia.org/wikipedia/commons/4/43/Valdai_IverskyMon_asv2018_img47.jpg', 'license_url': 'http://artlibre.org/licence/lal/en', 'license_name': 'FAL'} + + """ + file_name = url.rsplit('/', 1)[-1].rsplit(':', 1)[-1] + d = json.loads(URLOpener().open('https://commons.wikimedia.org/w/api.php?action=query&titles=File:{}&prop=imageinfo&iiprop=url|user|extmetadata&iimetadataversion=latest&format=json'.format(file_name)).read().decode('utf-8')) + d = list(d['query']['pages'].values())[0]['imageinfo'][0] + ext = d['extmetadata'] + + meta = { + 'title': ext['ObjectName']['value'], + 'author': d['user'], + 'source_url': d['descriptionurl'], + 'download_url': d['url'], + 'license_url': ext['LicenseUrl']['value'], + 'license_name': ext['LicenseShortName']['value'], + } + + return meta + + +def get_mnw_data(url): + """ + >>> get_mnw_data('https://cyfrowe.mnw.art.pl/pl/katalog/794032') + {'title': 'Pejzaż z podwójnym świerkiem', 'author': 'nieznany, Altdorfer, Albrecht (ca 1480-1538)', 'source_url': 'https://cyfrowe.mnw.art.pl/pl/katalog/794032', 'download_url': 'https://cyfrowe-cdn.mnw.art.pl/upload/multimedia/49/58/49583b3e9b23e2d25f372fe6021ae220.jpg', 'license_url': 'https://pl.wikipedia.org/wiki/Domena_publiczna', 'license_name': 'DOMENA PUBLICZNA'} + + """ + nr = url.rsplit('/', 1)[-1] + d = list( + csv.DictReader( + StringIO( + URLOpener().open( + 'https://cyfrowe-api.mnw.art.pl/api/object/{}/csv'.format(nr) + ).read().decode('utf-8') + ) + ) + )[0] + + authors = [] + i = 0 + while f'authors.{i}.name' in d: + authors.append(d[f'authors.{i}.name']) + i += 1 + + license_url = d['copyrights.0.link'] + license_name = d['copyrights.0.name'] + if license_name == 'DOMENA PUBLICZNA': + license_name = 'domena publiczna' + license_url = '' + + return { + 'title': d['title'], + 'author': ', '.join(authors), + 'source_url': url, + 'download_url': 'https://cyfrowe-cdn.mnw.art.pl/upload/multimedia/{}.{}'.format( + d['image.filePath'], + d['image.extension'], + ), + 'license_url': license_url, + 'license_name': license_name, + } + + +def get_import_data(url): + if re.match(r'(https?://)?(www\.|secure\.)?flickr\.com/', url): + return get_flickr_data(url) + if re.match(r'(https?://)?(commons|upload)\.wikimedia\.org/', url): + return get_wikimedia_data(url) + if re.match(r'(https?://)?cyfrowe\.mnw\.art\.pl/', url): + return get_mnw_data(url) diff --git a/src/cover/views.py b/src/cover/views.py index bfc3ee33..a7b96fbb 100644 --- a/src/cover/views.py +++ b/src/cover/views.py @@ -133,8 +133,8 @@ def image_list(request): def add_image(request): form = ff = None if request.method == 'POST': - if request.POST.get('form_id') == 'flickr': - ff = forms.FlickrForm(request.POST) + if request.POST.get('form_id') == 'import': + ff = forms.ImportForm(request.POST) if ff.is_valid(): form = forms.ImageAddForm(ff.cleaned_data) else: @@ -145,7 +145,7 @@ def add_image(request): if form is None: form = forms.ImageAddForm() if ff is None: - ff = forms.FlickrForm() + ff = forms.ImportForm() return render(request, 'cover/add_image.html', { 'form': form, 'ff': ff, -- 2.20.1