Epub debugging.
[redakcja.git] / src / depot / publishers / woblink.py
1 from datetime import date
2 import io
3 import json
4 import re
5 from time import sleep
6 from django.conf import settings
7 from django.utils.html import escape, format_html
8 from django.utils.safestring import mark_safe
9 from librarian.builders.html import SnippetHtmlBuilder
10 from librarian.functions import lang_code_3to2
11 from catalogue.models import Audience, Author, Thema
12 from .. import models
13 from .base import BasePublisher
14 from .woblink_constants import WOBLINK_CATEGORIES
15
16
17 class WoblinkError(ValueError):
18     pass
19
20 class NoPrice(WoblinkError):
21     def as_html(self):
22         return format_html(
23             'Brak <a href="/admin/depot/site/{site}">określonej ceny</a>.',
24             site=self.args[0].id
25         )
26
27 class NoIsbn(WoblinkError):
28     def as_html(self):
29         return 'Brak ISBN.'
30
31 class AuthorLiteralForeign(WoblinkError):
32     def as_html(self):
33         return format_html(
34             'Nie obsługiwane: autor „{author}” w języku {lang}.',
35             author=str(self.args[0]),
36             lang=self.args[0].lang,
37         )
38
39 class AuthorNotInCatalogue(WoblinkError):
40     def as_html(self):
41         return format_html(
42             'Brak autora „{author}” w katalogu.',
43             author=str(self.args[0])
44         )
45
46 class AuthorNoWoblink(WoblinkError):
47     def as_html(self):
48         return format_html(
49             'Autor <a href="/admin/catalogue/author/{author_id}/">{author}</a> bez identyfikatora Woblink.',
50             author_id=self.args[0].id,
51             author=self.args[0].name
52         )
53
54 class NoThema(WoblinkError):
55     def as_html(self):
56         return format_html('Brak Thema.')
57
58 class UnknownThema(WoblinkError):
59     def as_html(self):
60         return format_html(
61             'Nieznana Thema {code}.',
62             code=self.args[0]
63         )
64
65
66 class ThemaUnknownWoblink(WoblinkError):
67     def as_html(self):
68         return format_html(
69             'Thema <a href="/admin/catalogue/thema/{id}/">{code}</a> przypisana do nieznanej kategorii Woblink.',
70             id=self.args[0].id,
71             code=self.args[0].code,
72         )
73
74 class NoWoblinkCategory(WoblinkError):
75     def as_html(self):
76         return 'Brak kategorii Woblink.'
77
78 class WoblinkWarning(Warning):
79     pass
80
81 class NoMainThemaWarning(WoblinkWarning):
82     def as_html(self):
83         return format_html(
84             'Brak głównej kategorii Thema.'
85         )
86
87 class ThemaNoWoblink(WoblinkWarning):
88     def as_html(self):
89         return format_html(
90             'Thema <a href="/admin/catalogue/thema/{id}/">{code}</a> nie przypisana do kategorii Woblink.',
91             id=self.args[0].id,
92             code=self.args[0].code,
93         )
94
95 class AuthorLiteralForeignWarning(WoblinkWarning):
96     def as_html(self):
97         return format_html(
98             'Nie obsługiwane: autor „{author}” w języku {lang}.',
99             author=str(self.args[0]),
100             lang=self.args[0].lang,
101         )
102
103 class AuthorNotInCatalogueWarning(WoblinkWarning):
104     def as_html(self):
105         return format_html(
106             'Brak autora „{author}” w katalogu.',
107             author=str(self.args[0])
108         )
109
110 class AuthorNoWoblinkWarning(WoblinkWarning):
111     def as_html(self):
112         return format_html(
113             'Autor <a href="/admin/catalogue/author/{author_id}/">{author}</a> bez identyfikatora Woblink.',
114             author_id=self.args[0].id,
115             author=self.args[0].name
116         )
117
118
119
120
121 class Woblink(BasePublisher):
122     BASE_URL = 'https://publisher.woblink.com/'
123     ADD_URL = BASE_URL + 'catalog/add'
124     STEP1_URL = BASE_URL + 'catalog/edit/%s'
125     STEP2_URL = BASE_URL + 'catalog/edit/%s/2'
126     STEP3_URL = BASE_URL + 'catalog/edit/%s/3'
127     STEP4_URL = BASE_URL + 'catalog/edit/%s/4'
128     STEP5_URL = BASE_URL + 'catalog/edit/%s/5'
129     UPLOAD_URL = BASE_URL + 'file/upload-%s'
130     JOB_STATUS_URL = BASE_URL + 'task/status'
131     GENERATE_DEMO_URL = BASE_URL + 'task/run/generate-%s-demo/%s/%d'
132     CHECK_DEMO_URL = BASE_URL + 'task/run/check-%s-demo/%s'
133
134     SEARCH_CATALOGUE_URL = BASE_URL + '{category}/autocomplete/{term}'
135
136     ROLE_AUTHOR = 1
137     ROLE_TRANSLATOR = 4
138
139     def login(self):
140         response = self.session.get('https://publisher.woblink.com/login')
141         token = re.search(
142             r'name="_csrf_token" value="([^"]+)"',
143             response.text
144         ).group(1)
145         data = {
146             '_csrf_token': token,
147             '_username': self.username,
148             '_password': self.password,
149         }
150         response = self.session.post(
151             'https://publisher.woblink.com/login_check',
152             data=data,
153         )
154
155     def search_catalogue(self, category, term):
156         return self.session.get(
157             self.SEARCH_CATALOGUE_URL.format(category=category, term=term)
158         ).json()
159
160     def search_author_catalogue(self, term):
161         return [
162             {
163                 'id': item['autId'],
164                 'text': item['autFullname']
165             }
166             for item in self.search_catalogue('author', term)
167         ]
168     def search_series_catalogue(self, term):
169         return [
170             {
171                 'id': item['id'],
172                 'text': item['name']
173             }
174             for item in self.search_catalogue('series', term)
175         ]
176         
177     def get_isbn(self, meta, errors=None):
178         if not meta.isbn_epub:
179             if errors is not None:
180                 errors.append(NoIsbn())
181         return meta.isbn_epub
182
183     def get_authors_data(self, meta, errors=None):
184         authors = []
185         for role, items, obligatory in [
186                 (self.ROLE_AUTHOR, meta.authors, True),
187                 (self.ROLE_TRANSLATOR, meta.translators, False)
188         ]:
189             for person_literal in items:
190                 if person_literal.lang != 'pl':
191                     if errors is not None:
192                         if obligatory:
193                              errors.append(AuthorLiteralForeign(person_literal))
194                         else:
195                             errors.append(AuthorLiteralForeignWarning(person_literal))
196                     continue
197                 aobj = Author.get_by_literal(str(person_literal))
198                 if aobj is None:
199                     if errors is not None:
200                         if obligatory:
201                              errors.append(AuthorNotInCatalogue(person_literal))
202                         else:
203                             errors.append(AuthorNotInCatalogueWarning(person_literal))
204                     continue
205                 if not aobj.woblink:
206                     if errors is not None:
207                         if obligatory:
208                              errors.append(AuthorNoWoblink(aobj))
209                         else:
210                             errors.append(AuthorNoWoblinkWarning(aobj))
211                     continue
212                 authors.append((role, aobj.woblink))
213         return authors
214
215     def get_genres(self, meta, errors=None):
216         thema_codes = []
217         if meta.thema_main:
218             thema_codes.append(meta.thema_main)
219         else:
220             if errors is not None:
221                 errors.append(NoMainThemaWarning())
222         thema_codes.extend(meta.thema)
223
224         thema_codes.extend(
225             Audience.objects.filter(code__in=meta.audiences).exclude(
226                 thema='').values_list('thema', flat=True)
227         )
228
229         if not thema_codes:
230             if errors is not None:
231                 errors.append(NoThema())
232         category_ids = []
233         for code in thema_codes:
234             try:
235                 thema = Thema.objects.get(code=code)
236             except Thema.DoesNotExist:
237                 if errors is not None:
238                     errors.append(UnknownThema(code))
239             else:
240                 if thema.woblink_category is None:
241                     if errors is not None:
242                         errors.append(ThemaNoWoblink(thema))
243                 elif thema.woblink_category not in WOBLINK_CATEGORIES:
244                     if errors is not None:
245                         errors.append(ThemaUnknownWoblink(thema))
246                 elif thema.woblink_category not in category_ids:
247                     category_ids.append(thema.woblink_category)
248         if not category_ids:
249             if errors is not None:
250                 errors.append(NoWoblinkCategory())
251         return category_ids
252
253     def get_series(self, meta, errors=None):
254         return list(Audience.objects.filter(code__in=meta.audiences).exclude(
255             woblink=None).values_list('woblink', flat=True))
256
257     def get_abstract(self, wldoc, errors=None, description_add=None):
258         description = self.get_description(wldoc, description_add)
259         parts = description.split('\n', 1)
260         if len(parts) == 1 or len(parts[0]) > 240:
261             # No newline found here.
262             # Try to find last sentence end..
263             parts = re.split(r' \.', description[240::-1], 1)
264             if len(parts) == 2:
265                 p1 = parts[1][::-1] + '.'
266                 p2 = description[len(p1) + 1:]
267             else:
268                 # No sentence end found.
269                 # Just find a space.
270                 p1 = description[:240].rsplit(' ', 1)[0]
271                 p2 = description[len(p1) + 1:]
272                 p1 += '…'
273                 p2 = '…' + p2
274             parts = [p1, p2]
275
276         m = re.search(r'<[^>]+$', parts[0])
277         if m is not None:
278             parts[0] = parts[0][:-len(m.group(0))]
279             parts[1] = m.group(0) + parts[1]
280
281         opened = []
282         for tag in re.findall(r'<[^>]*[^/>]>', parts[0]):
283             if tag[1] == '/':
284                 opened.pop()
285             else:
286                 opened.append(tag)
287         for tag in reversed(opened):
288             parts[0] += '</' + tag[1:-1].split()[0] + '>'
289             parts[1] = tag + parts[1]
290         return {
291             'header': parts[0],
292             'rest': parts[1],
293         }
294
295     def get_lang2code(self, meta, errors=None):
296         return lang_code_3to2(meta.language)
297
298     def get_price(self, site, wldoc, errors=None):
299         try:
300             stats = wldoc.get_statistics()['total']
301         except:
302             if errors:
303                 errors.append(NoPrice(site))
304             return 0
305         words = stats['words_with_fn']
306         pages = stats['chars_with_fn'] / 1800
307         price = site.get_price(words, pages)
308         if price is None:
309             if errors:
310                 errors.append(NoPrice(site))
311             return 0
312
313         return price
314
315     def can_publish(self, site, book):
316         d = {
317             'warnings': [],
318             'errors': [],
319             'info': [],
320         }
321         try:
322             wldoc = book.wldocument(librarian2=True)
323         except:
324             d['errors'].append('Nieprawidłowy dokument.')
325             return d
326         errors = []
327         book_data = self.get_book_data(site, wldoc, errors)
328         for error in errors:
329             if not isinstance(error, Warning):
330                 errlist = d['errors']
331             else:
332                 errlist = d['warnings']
333             errlist.append(error.as_html())
334
335         if book_data.get('isbn'):
336             d['info'].append(format_html(
337                 'ISBN: {isbn}',
338                 isbn=book_data['isbn'],
339             ))
340
341         if book_data.get('genres'):
342             d['info'].append(format_html(
343                 'W kategoriach: {cat} ({price} zł)',
344                 cat=', '.join(self.describe_category(g) for g in book_data['genres']),
345                 price=book_data['price'],
346             ))
347         d['info'].append(mark_safe(
348             '<strong>' + book_data['abstract']['header'] +
349             '</strong><br/>' + book_data['abstract']['rest']
350         ))
351
352         return d
353
354     def describe_category(self, category):
355         t = []
356         while category:
357             c = WOBLINK_CATEGORIES[category]
358             t.append(c['name'])
359             category = c.get('parent')
360         return ' / '.join(reversed(t))
361
362     def create_book(self, isbn):
363         isbn = ''.join(c for c in isbn if c.isdigit())
364         assert len(isbn) == 13
365         response = self.session.post(
366             self.ADD_URL,
367             data={
368                 'AddPublication[pubType]': 'ebook',
369                 'AddPublication[pubHasIsbn]': '1',
370                 'AddPublication[pubIsbn]': isbn,
371                  ##AddPubation[save]
372             }
373         )
374         m = re.search(r'/(\d+)$', response.url)
375         if m is not None:
376             return m.group(1)
377
378     def send_book(self, site_book_publish, changes=None):
379         site_book = site_book_publish.site_book
380         book = site_book.book
381         site = site_book.site
382         wldoc = book.wldocument(librarian2=True, changes=changes, publishable=False) # TODO pub
383         meta = wldoc.meta
384
385         book_data = self.get_book_data(site, wldoc)
386
387         if not site_book.external_id:
388             woblink_id = self.create_book(book_data['isbn'])
389             assert woblink_id
390             site_book.external_id = woblink_id
391             site_book.save(update_fields=['external_id'])
392         woblink_id = site_book.external_id
393
394         self.edit_step1(woblink_id, book_data)
395         self.edit_step2(woblink_id, book_data)
396         self.edit_step3(woblink_id, book_data)
397         cover_id = self.send_cover(woblink_id, wldoc)
398
399         texts = site.get_texts()
400         epub_id, epub_demo = self.send_epub(
401             woblink_id, wldoc, book.gallery_path(),
402             fundraising=texts
403         )
404         mobi_id, mobi_demo = self.send_mobi(
405             woblink_id, wldoc, book.gallery_path(),
406             fundraising=texts
407         )
408         self.edit_step4(
409             woblink_id, book_data,
410             cover_id, epub_id, epub_demo, mobi_id, mobi_demo,
411         )
412         self.edit_step5(woblink_id, book_data)
413
414     def get_book_data(self, site, wldoc, errors=None):
415         return {
416             "title": wldoc.meta.title,
417             "isbn": self.get_isbn(wldoc.meta, errors=errors),
418             "authors": self.get_authors_data(wldoc.meta, errors=errors),
419             "abstract": self.get_abstract(
420                 wldoc, errors=errors, description_add=site.description_add
421             ),
422             "lang2code": self.get_lang2code(wldoc.meta, errors=errors),
423             "genres": self.get_genres(wldoc.meta, errors=errors),
424             "price": self.get_price(site, wldoc, errors=errors),
425             "series": self.get_series(wldoc.meta, errors=errors),
426         }
427
428     def with_form_name(self, data, name):
429         return {
430             f"{name}[{k}]": v
431             for (k, v) in data.items()
432         }
433
434     def edit_step1(self, woblink_id, book_data):
435         data = book_data
436
437         authors_data = [
438             {
439                 "AhpPubId": woblink_id,
440                 "AhpAutId": author_id,
441                 "AhpType": author_type,
442             }
443             for (author_type, author_id) in data['authors']
444         ]
445
446         series_data = [
447             {
448                 'PublicationId': woblink_id,
449                 'SeriesId': series_id,
450             }
451             for series_id in data['series']
452         ]
453
454         d = {
455             'pubTitle': book_data['title'],
456             'npwAuthorHasPublications': json.dumps(authors_data),
457             'pubShortNote': data['abstract']['header'],
458             'pubNote': data['abstract']['rest'],
459             'pubCulture': data['lang2code'],
460             'npwPublicationHasAwards': '[]',
461             'npwPublicationHasSeriess': json.dumps(series_data),
462         }
463         d = self.with_form_name(d, 'EditPublicationStep1')
464         d['roles'] = [author_type for (author_type, author_id) in data['authors']]
465         r = self.session.post(self.STEP1_URL % woblink_id, data=d)
466         return r
467
468
469     def edit_step2(self, woblink_id, book_data):
470         gd = {}
471         legacy = None
472         for i, g in enumerate(book_data['genres']):
473             gdata = WOBLINK_CATEGORIES[g]
474             if legacy is None:
475                 legacy = gdata.get('legacy')
476             if p := gdata.get('parent'):
477                 gd.setdefault(p, {'isMain': False})
478                 gd[p].setdefault('children', [])
479                 gd[p]['children'].append(str(g))
480                 gd[p].setdefault('mainChild', str(g))
481                 if legacy is None:
482                     legacy = WOBLINK_CATEGORIES[p].get('legacy')
483             else:
484                 gd.setdefault(g, {})
485                 gd[g]['isMain'] = True
486         gd = [
487             {
488                 "pubId": woblink_id,
489                 "category": str(k),
490                 **v
491             }
492             for k, v in gd.items()
493         ]
494
495         data = {
496             'npwPublicationHasNewGenres': json.dumps(gd),
497             'genre': legacy or '',
498         }
499         data = self.with_form_name(data, 'AddPublicationStep2')
500         return self.session.post(self.STEP2_URL % woblink_id, data=data)
501
502     def edit_step3(self, woblink_id, book_data):
503         d = {
504             'pubBasePrice': book_data['price'],
505             'pubPremiereDate': date.today().isoformat(),
506             'pubIsLicenseIndefinite': '1',
507             'pubFileFormat': 'epub+mobi',
508             'pubIsAcs': '0',
509             'pubPublisherIndex': '',
510             'save_and_continue': '',
511         }
512         d = self.with_form_name(d, 'EditPublicationStep3')
513         return self.session.post(self.STEP3_URL % woblink_id, data=d)
514
515     def edit_step4(self, woblink_id, book_data, cover_id, epub_id, epub_demo, mobi_id, mobi_demo):
516         d = {
517             'pubCoverResId': cover_id,
518             'pubEpubResId': epub_id,
519             'pubEpubDemoResId': epub_demo,
520             'pubMobiResId': mobi_id,
521             'pubMobiDemoResId': mobi_demo,
522             'pubFileFormat': 'epub+mobi',
523             'pubId': woblink_id,
524             'save_and_continue': '',
525         }
526         d = self.with_form_name(d, 'EditPublicationStep4')
527         return self.session.post(self.STEP4_URL % woblink_id, data=d)
528
529     def edit_step5(self, woblink_id, book_data):
530         d = {'save': ''}
531         d = self.with_form_name(d, 'EditPublicationStep5')
532         return self.session.post(self.STEP5_URL % woblink_id, data=d)
533
534     def wait_for_job(self, job_id):
535         while True:
536             response = self.session.post(
537                 self.JOB_STATUS_URL,
538                 data={'ids[]': job_id}
539             )
540             data = response.json()[job_id]
541             if data['ready']:
542                 assert data['successful']
543                 return data.get('returnValue')
544             sleep(2)
545
546     def upload_file(self, woblink_id, filename, content, field_name, mime_type):
547         form_name = f'Upload{field_name}'
548         id_field = f'pub{field_name}ResId'
549         field_name = field_name.lower()
550
551         data = {
552             'pubId': woblink_id,
553         }
554         files = {
555             field_name: (filename, content, mime_type)
556         }
557         
558         response = self.session.post(
559             self.UPLOAD_URL % field_name,
560             data=self.with_form_name(data, form_name),
561             files=self.with_form_name(files, form_name),
562         )
563         resp_data = response.json()
564         assert resp_data['success'] is True
565         file_id = resp_data[id_field]
566         if 'jobId' in resp_data:
567             self.wait_for_job(resp_data['jobId'])
568         return file_id
569
570     def generate_demo(self, woblink_id, file_format, check=True):
571         percent = 10
572         while True:
573             job_id = self.session.get(
574                 self.GENERATE_DEMO_URL % (file_format, woblink_id, percent),
575             ).json()['jobId']
576             try:
577                 file_id = self.wait_for_job(job_id)
578                 if check:
579                     self.wait_for_job(
580                         self.session.get(
581                             self.CHECK_DEMO_URL % (file_format, woblink_id)
582                         ).json()['jobId']
583                     )
584             except AssertionError:
585                 if percent < 50:
586                     percent += 10
587                 else:
588                     raise
589             else:
590                 break
591
592         return file_id
593
594     def send_epub(self, woblink_id, doc, gallery_path, fundraising=None):
595         from librarian.builders import EpubBuilder
596         content = EpubBuilder(
597             base_url='file://' + gallery_path + '/',
598             fundraising=fundraising or [],
599         ).build(doc).get_file()
600         file_id = self.upload_file(
601             woblink_id,
602             doc.meta.url.slug + '.epub',
603             content,
604             'Epub',
605             'application/epub+zip'
606         )
607         demo_id = self.generate_demo(woblink_id, 'epub')
608         return file_id, demo_id
609
610     def send_mobi(self, woblink_id, doc, gallery_path, fundraising=None):
611         from librarian.builders import MobiBuilder
612         content = MobiBuilder(
613             base_url='file://' + gallery_path + '/',
614             fundraising=fundraising or [],
615         ).build(doc).get_file()
616         file_id = self.upload_file(
617             woblink_id,
618             doc.meta.url.slug + '.mobi',
619             content,
620             'Mobi',
621             'application/x-mobipocket-ebook'
622         )
623         demo_id = self.generate_demo(woblink_id, 'mobi', check=False)
624         return file_id, demo_id
625
626     def send_cover(self, woblink_id, doc):
627         from librarian.cover import make_cover
628         # TODO Labe
629         # A5 @ 300ppi.
630         cover = make_cover(doc.meta, cover_class='m-label', width=1748, height=2480)
631         content = io.BytesIO()
632         cover.final_image().save(content, cover.format)
633         content.seek(0)
634         file_id = self.upload_file(
635             woblink_id,
636             doc.meta.url.slug + '.jpeg',
637             content,
638             'Cover',
639             cover.mime_type()
640         )
641         return file_id