2ce39f0665881b1ae27d6b386235db39bce4fab3
[redakcja.git] / src / catalogue / management / commands / import_catalogue_from_wl_dump.py
1 import json
2 from urllib.request import urlopen
3 import sys
4 from django.core.management import BaseCommand
5 from slugify import slugify
6 import wikidata
7 from catalogue.models import Book, Author
8
9
10 def parse_name(name):
11     name_pieces = name.rsplit(" ", 1)
12     if len(name_pieces) == 1:
13         return name_pieces[0], ""
14     else:
15         return name_pieces
16
17
18 def find_wikidata(link, lang):
19     link = link.rstrip()
20     title = link.rsplit("/", 1)[-1]
21     title = link.split("#", 1)[0]
22     title = title.replace(" ", "_")
23     data = json.load(
24         urlopen(
25             f"https://www.wikidata.org/w/api.php?action=wbgetentities&sites={lang}wiki&titles={title}&format=json"
26         )
27     )
28     wikidata_id = list(data["entities"].keys())[0]
29     if not wikidata_id.startswith("Q"):
30         return None
31     return wikidata_id
32
33
34 class Command(BaseCommand):
35     def add_arguments(self, parser):
36         parser.add_argument("path")
37
38     def handle(self, path, **kwargs):
39         with open(path) as f:
40             data = json.load(f)
41
42         for pass_n in (1, 2):
43             for item in data:
44                 if item["model"] == "pdcounter.bookstub":
45                     if pass_n != 2:
46                         continue
47                     notes = []
48                     print(item["fields"]["author"], item["fields"]["title"])
49                     slug = item["fields"]["slug"]
50                     book, created = Book.objects.get_or_create(slug=slug)
51                     if item["fields"]["translator"] and not book.translators.exists():
52                         notes.append("tłum.: " + item["fields"]["translator"])
53                     book.title = book.title or item["fields"]["title"]
54                     book.pd_year = book.pd_year or item["fields"]["pd"]
55                     notes = "\n".join(notes)
56                     if notes and notes not in book.notes:
57                         book.notes = "\n".join([notes, book.notes])
58                     book.save()
59
60                     if not book.authors.exists():
61                         first_name, last_name = parse_name(item["fields"]["author"])
62                         author_slug = slugify(item["fields"]["author"])
63                         author = (
64                             Author.objects.filter(slug=author_slug).first()
65                             or Author.objects.filter(
66                                 first_name=first_name, last_name=last_name
67                             ).first()
68                             or Author()
69                         )
70                         author.slug = author.slug or author_slug
71                         author.first_name = author.first_name or first_name
72                         author.last_name = author.last_name or last_name
73                         author.save()
74                         book.authors.set([author])
75                 elif item["model"] == "pdcounter.author":
76                     if pass_n != 1:
77                         continue
78                     slug = item["fields"]["slug"]
79                     author, created = Author.objects.get_or_create(slug=slug)
80                     if not author.first_name and not author.last_name:
81                         author.first_name, author.last_name = parse_name(
82                             item["fields"]["name"]
83                         )
84                         author.year_of_death = (
85                             author.year_of_death or item["fields"]["death"]
86                         )
87                         author.notes = author.notes or item["fields"]["description"]
88                         author.gazeta_link = (
89                             author.gazeta_link or item["fields"]["gazeta_link"]
90                         )
91                         author.save()
92                         wiki_link = item["fields"]["wiki_link"]
93                         assert not wiki_link  # Welp
94                 elif item["model"] == "catalogue.book":
95                     if pass_n != 2:
96                         continue
97                     if item["fields"]["parent"]:
98                         continue
99                     print(item["fields"]["slug"])
100                     slug = item["fields"]["slug"]
101                     book, created = Book.objects.get_or_create(slug=slug)
102                     book.title = book.title or item["fields"]["title"]
103                     book.language = book.language or item["fields"]["language"]
104                     book.gazeta_link = book.gazeta_link or item["fields"]["gazeta_link"]
105                     if item["fields"]["wiki_link"]:
106                         book.wikidata = (
107                             book.wikidata
108                             or find_wikidata(item["fields"]["wiki_link"], "pl")
109                             or ""
110                         )
111
112                     extra_info = json.loads(item["fields"]["extra_info"])
113                     if book.pd_year is None and extra_info.get(
114                         "released_to_public_domain_at"
115                     ):
116                         book.pd_year = int(
117                             extra_info["released_to_public_domain_at"].split("-", 1)[0]
118                         )
119
120                     book.save()
121
122                     if not book.authors.exists():
123                         authors = []
124                         for astr in extra_info.get("authors", []):
125                             parts = astr.split(", ")
126                             if len(parts) == 1:
127                                 first_name = parts[0]
128                                 last_name = ""
129                             else:
130                                 last_name, first_name = parts
131                             aslug = slugify(f"{first_name} {last_name}".strip())
132                             author = (
133                                 Author.objects.filter(slug=aslug).first()
134                                 or Author.objects.filter(
135                                     first_name=first_name, last_name=last_name
136                                 ).first()
137                                 or Author.objects.filter(name_de=astr).first()
138                                 or Author.objects.filter(name_lt=astr).first()
139                             )
140                             # Not trying to create the author or set properties, because here we don't know the dc:creator@xml:lang property.
141                             if author is not None:
142                                 authors.append(author)
143                         book.authors.set(authors)
144                 elif item["model"] == "catalogue.tag":
145                     if pass_n != 1:
146                         continue
147                     if item["fields"]["category"] != "author":
148                         continue
149                     slug = item["fields"]["slug"]
150                     author, created = Author.objects.get_or_create(slug=slug)
151                     author.name_de = author.name_de or item["fields"]["name_de"] or ""
152                     author.name_lt = author.name_lt or item["fields"]["name_lt"] or ""
153                     if not author.first_name and not author.last_name:
154                         author.first_name, author.last_name = parse_name(
155                             item["fields"]["name_pl"]
156                         )
157                     author.culturepl_link = (
158                         author.culturepl_link or item["fields"]["culturepl_link"] or ""
159                     )
160                     author.gazeta_link = (
161                         author.gazeta_link or item["fields"]["gazeta_link"] or ""
162                     )
163                     author.description = (
164                         author.description or item["fields"]["description_pl"] or ""
165                     )
166                     author.description_de = (
167                         author.description_de or item["fields"]["description_de"] or ""
168                     )
169                     author.description_lt = (
170                         author.description_lt or item["fields"]["description_lt"] or ""
171                     )
172
173                     if not author.wikidata:
174                         for field, value in item["fields"].items():
175                             if field.startswith("wiki_link_") and value:
176                                 wd = find_wikidata(value, field.rsplit("_", 1)[-1])
177                                 if wd:
178                                     author.wikidata = wd
179                                     break
180                     author.save()
181
182                 else:
183                     print(item)
184                     break