Avoid hanging conjunction on covers.
[librarian.git] / src / librarian / html.py
1 # -*- coding: utf-8 -*-
2 #
3 # This file is part of Librarian, licensed under GNU Affero GPLv3 or later.
4 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
5 #
6 from __future__ import print_function, unicode_literals
7
8 import os
9 import re
10 import copy
11
12 from lxml import etree
13 from librarian import XHTMLNS, ParseError, OutputFile
14 from librarian import functions
15 from PIL import Image
16
17 from lxml.etree import XMLSyntaxError, XSLTApplyError
18 import six
19
20
21 functions.reg_substitute_entities()
22 functions.reg_person_name()
23
24 STYLESHEETS = {
25     'legacy': 'xslt/book2html.xslt',
26     'full': 'xslt/wl2html_full.xslt',
27     'partial': 'xslt/wl2html_partial.xslt'
28 }
29
30
31 def get_stylesheet(name):
32     return os.path.join(os.path.dirname(__file__), STYLESHEETS[name])
33
34
35 def html_has_content(text):
36     return etree.ETXPath(
37         '//p|//{%(ns)s}p|//h1|//{%(ns)s}h1' % {'ns': str(XHTMLNS)}
38     )(text)
39
40
41 def transform_abstrakt(abstrakt_element):
42     style_filename = get_stylesheet('legacy')
43     style = etree.parse(style_filename)
44     xml = etree.tostring(abstrakt_element, encoding='unicode')
45     document = etree.parse(six.StringIO(
46         xml.replace('abstrakt', 'dlugi_cytat')
47     ))  # HACK
48     result = document.xslt(style)
49     html = re.sub('<a name="sec[0-9]*"/>', '',
50                   etree.tostring(result, encoding='unicode'))
51     return re.sub('</?blockquote[^>]*>', '', html)
52
53
54 def add_image_sizes(tree, gallery_path, gallery_url, base_url):
55     widths = [360, 600, 1200, 1800, 2400]
56
57     for i, ilustr in enumerate(tree.findall('//ilustr')):
58         rel_path = ilustr.attrib['src']
59         img_url = six.moves.urllib.parse.urljoin(base_url, rel_path)
60
61         f = six.moves.urllib.request.urlopen(img_url)
62         img = Image.open(f)
63         ext = {'GIF': 'gif', 'PNG': 'png'}.get(img.format, 'jpg')
64
65         srcset = []
66         # Needed widths: predefined and original, limited by
67         # whichever is smaller.
68         img_widths = [
69             w for w in
70             sorted(
71                 set(widths + [img.size[0]])
72             )
73             if w <= min(widths[-1], img.size[0])
74         ]
75         largest = None
76         for w in widths:
77             fname = '%d.W%d.%s' % (i, w, ext)
78             fpath = gallery_path + fname
79             if not os.path.exists(fpath):
80                 height = round(img.size[1] * w / img.size[0])
81                 th = img.resize((w, height))
82                 th.save(fpath)
83             th_url = gallery_url + fname
84             srcset.append(" ".join((
85                 th_url,
86                 '%dw' % w
87             )))
88             largest_url = th_url
89         ilustr.attrib['srcset'] = ", ".join(srcset)
90         ilustr.attrib['src'] = largest_url
91
92         f.close()
93
94
95 def transform(wldoc, stylesheet='legacy', options=None, flags=None, css=None, gallery_path='img/', gallery_url='img/', base_url='file://./'):
96     """Transforms the WL document to XHTML.
97
98     If output_filename is None, returns an XML,
99     otherwise returns True if file has been written,False if it hasn't.
100     File won't be written if it has no content.
101     """
102     # Parse XSLT
103     try:
104         style_filename = get_stylesheet(stylesheet)
105         style = etree.parse(style_filename)
106
107         document = copy.deepcopy(wldoc)
108         del wldoc
109         document.swap_endlines()
110
111         if flags:
112             for flag in flags:
113                 document.edoc.getroot().set(flag, 'yes')
114
115         document.clean_ed_note()
116         document.clean_ed_note('abstrakt')
117
118         if not options:
119             options = {}
120
121         try:
122             os.makedirs(gallery_path)
123         except OSError:
124             pass
125
126         add_image_sizes(document.edoc, gallery_path, gallery_url, base_url)
127
128         css = (
129             css
130             or 'https://static.wolnelektury.pl/css/compressed/book_text.css'
131         )
132         css = "'%s'" % css
133         result = document.transform(style, css=css, **options)
134         del document  # no longer needed large object :)
135
136         if html_has_content(result):
137             add_anchors(result.getroot())
138             add_table_of_themes(result.getroot())
139             add_table_of_contents(result.getroot())
140
141             return OutputFile.from_bytes(etree.tostring(
142                 result, method='html', xml_declaration=False,
143                 pretty_print=True, encoding='utf-8'
144             ))
145         else:
146             return None
147     except KeyError:
148         raise ValueError("'%s' is not a valid stylesheet.")
149     except (XMLSyntaxError, XSLTApplyError) as e:
150         raise ParseError(e)
151
152
153 @six.python_2_unicode_compatible
154 class Fragment(object):
155     def __init__(self, id, themes):
156         super(Fragment, self).__init__()
157         self.id = id
158         self.themes = themes
159         self.events = []
160
161     def append(self, event, element):
162         self.events.append((event, element))
163
164     def closed_events(self):
165         stack = []
166         for event, element in self.events:
167             if event == 'start':
168                 stack.append(('end', element))
169             elif event == 'end':
170                 try:
171                     stack.pop()
172                 except IndexError:
173                     print('CLOSED NON-OPEN TAG:', element)
174
175         stack.reverse()
176         return self.events + stack
177
178     def to_string(self):
179         result = []
180         for event, element in self.closed_events():
181             if event == 'start':
182                 result.append(u'<%s %s>' % (
183                     element.tag,
184                     ' '.join(
185                         '%s="%s"' % (k, v)
186                         for k, v in element.attrib.items()
187                     )
188                 ))
189                 if element.text:
190                     result.append(element.text)
191             elif event == 'end':
192                 result.append(u'</%s>' % element.tag)
193                 if element.tail:
194                     result.append(element.tail)
195             else:
196                 result.append(element)
197
198         return ''.join(result)
199
200     def __str__(self):
201         return self.to_string()
202
203
204 def extract_fragments(input_filename):
205     """Extracts theme fragments from input_filename."""
206     open_fragments = {}
207     closed_fragments = {}
208
209     # iterparse would die on a HTML document
210     parser = etree.HTMLParser(encoding='utf-8')
211     buf = six.BytesIO()
212     buf.write(etree.tostring(
213         etree.parse(input_filename, parser).getroot()[0][0],
214         encoding='utf-8'
215     ))
216     buf.seek(0)
217
218     for event, element in etree.iterparse(buf, events=('start', 'end')):
219         # Process begin and end elements
220         if element.get('class', '') in ('theme-begin', 'theme-end'):
221             if not event == 'end':
222                 continue  # Process elements only once, on end event
223
224             # Open new fragment
225             if element.get('class', '') == 'theme-begin':
226                 fragment = Fragment(id=element.get('fid'), themes=element.text)
227
228                 # Append parents
229                 parent = element.getparent()
230                 parents = []
231                 while parent.get('id', None) != 'book-text':
232                     cparent = copy.deepcopy(parent)
233                     cparent.text = None
234                     if 'id' in cparent.attrib:
235                         del cparent.attrib['id']
236                     parents.append(cparent)
237                     parent = parent.getparent()
238
239                 parents.reverse()
240                 for parent in parents:
241                     fragment.append('start', parent)
242
243                 open_fragments[fragment.id] = fragment
244
245             # Close existing fragment
246             else:
247                 try:
248                     fragment = open_fragments[element.get('fid')]
249                 except KeyError:
250                     print('%s:closed not open fragment #%s' % (
251                         input_filename, element.get('fid')
252                     ))
253                 else:
254                     closed_fragments[fragment.id] = fragment
255                     del open_fragments[fragment.id]
256
257             # Append element tail to lost_text
258             # (we don't want to lose any text)
259             if element.tail:
260                 for fragment_id in open_fragments:
261                     open_fragments[fragment_id].append('text', element.tail)
262
263         # Process all elements except begin and end
264         else:
265             # Omit annotation tags
266             if (len(element.get('name', '')) or
267                     element.get('class', '') in ('annotation', 'anchor')):
268                 if event == 'end' and element.tail:
269                     for fragment_id in open_fragments:
270                         open_fragments[fragment_id].append(
271                             'text', element.tail
272                         )
273             else:
274                 for fragment_id in open_fragments:
275                     celem = copy.copy(element)
276                     if 'id' in celem.attrib:
277                         del celem.attrib['id']
278                     open_fragments[fragment_id].append(
279                         event, celem
280                     )
281
282     return closed_fragments, open_fragments
283
284
285 def add_anchor(element, prefix, with_link=True, with_target=True,
286                link_text=None):
287     parent = element.getparent()
288     index = parent.index(element)
289
290     if with_link:
291         if link_text is None:
292             link_text = prefix
293         anchor = etree.Element('a', href='#%s' % prefix)
294         anchor.set('class', 'anchor')
295         anchor.text = six.text_type(link_text)
296         parent.insert(index, anchor)
297
298     if with_target:
299         anchor_target = etree.Element('a', name='%s' % prefix)
300         anchor_target.set('class', 'target')
301         anchor_target.text = u' '
302         parent.insert(index, anchor_target)
303
304
305 def any_ancestor(element, test):
306     for ancestor in element.iterancestors():
307         if test(ancestor):
308             return True
309     return False
310
311
312 def add_anchors(root):
313     counter = 1
314     for element in root.iterdescendants():
315         def f(e):
316             return (
317                 e.get('class') in (
318                     'note', 'motto', 'motto_podpis', 'dedication', 'frame'
319                 )
320                 or e.get('id') == 'nota_red'
321                 or e.tag == 'blockquote'
322             )
323         if any_ancestor(element, f):
324             continue
325
326         if element.tag == 'div' and 'verse' in element.get('class', ''):
327             if counter == 1 or counter % 5 == 0:
328                 add_anchor(element, "f%d" % counter, link_text=counter)
329             counter += 1
330         elif 'paragraph' in element.get('class', ''):
331             add_anchor(element, "f%d" % counter, link_text=counter)
332             counter += 1
333
334
335 def raw_printable_text(element):
336     working = copy.deepcopy(element)
337     for e in working.findall('a'):
338         if e.get('class') in ('annotation', 'theme-begin'):
339             e.text = ''
340     return etree.tostring(working, method='text', encoding='unicode').strip()
341
342
343 def add_table_of_contents(root):
344     sections = []
345     counter = 1
346     for element in root.iterdescendants():
347         if element.tag in ('h2', 'h3'):
348             if any_ancestor(
349                     element,
350                     lambda e: e.get('id') in (
351                         'footnotes', 'nota_red'
352                     ) or e.get('class') in ('person-list',)):
353                 continue
354
355             element_text = raw_printable_text(element)
356             if (element.tag == 'h3' and len(sections)
357                     and sections[-1][1] == 'h2'):
358                 sections[-1][3].append(
359                     (counter, element.tag, element_text, [])
360                 )
361             else:
362                 sections.append((counter, element.tag, element_text, []))
363             add_anchor(element, "s%d" % counter, with_link=False)
364             counter += 1
365
366     toc = etree.Element('div')
367     toc.set('id', 'toc')
368     toc_header = etree.SubElement(toc, 'h2')
369     toc_header.text = u'Spis treści'
370     toc_list = etree.SubElement(toc, 'ol')
371
372     for n, section, text, subsections in sections:
373         section_element = etree.SubElement(toc_list, 'li')
374         add_anchor(section_element, "s%d" % n, with_target=False,
375                    link_text=text)
376
377         if len(subsections):
378             subsection_list = etree.SubElement(section_element, 'ol')
379             for n1, subsection, subtext, _ in subsections:
380                 subsection_element = etree.SubElement(subsection_list, 'li')
381                 add_anchor(subsection_element, "s%d" % n1, with_target=False,
382                            link_text=subtext)
383
384     root.insert(0, toc)
385
386
387 def add_table_of_themes(root):
388     try:
389         from sortify import sortify
390     except ImportError:
391         def sortify(x):
392             return x
393
394     book_themes = {}
395     for fragment in root.findall('.//a[@class="theme-begin"]'):
396         if not fragment.text:
397             continue
398         theme_names = [s.strip() for s in fragment.text.split(',')]
399         for theme_name in theme_names:
400             book_themes.setdefault(theme_name, []).append(fragment.get('name'))
401     book_themes = list(book_themes.items())
402     book_themes.sort(key=lambda s: sortify(s[0]))
403     themes_div = etree.Element('div', id="themes")
404     themes_ol = etree.SubElement(themes_div, 'ol')
405     for theme_name, fragments in book_themes:
406         themes_li = etree.SubElement(themes_ol, 'li')
407         themes_li.text = "%s: " % theme_name
408         for i, fragment in enumerate(fragments):
409             item = etree.SubElement(themes_li, 'a', href="#%s" % fragment)
410             item.text = str(i + 1)
411             item.tail = ' '
412     root.insert(0, themes_div)
413
414
415 def extract_annotations(html_path):
416     """Extracts annotations from HTML for annotations dictionary.
417
418     For each annotation, yields a tuple of:
419     anchor, footnote type, valid qualifiers, text, html.
420
421     """
422     from .fn_qualifiers import FN_QUALIFIERS
423
424     parser = etree.HTMLParser(encoding='utf-8')
425     tree = etree.parse(html_path, parser)
426     footnotes = tree.find('//*[@id="footnotes"]')
427     re_qualifier = re.compile(r'[^\u2014]+\s+\(([^\)]+)\)\s+\u2014')
428     if footnotes is not None:
429         for footnote in footnotes.findall('div'):
430             fn_type = footnote.get('class').split('-')[1]
431             anchor = footnote.find('a[@class="annotation"]').get('href')[1:]
432             del footnote[:2]
433             footnote.text = None
434             if len(footnote) and footnote[-1].tail == '\n':
435                 footnote[-1].tail = None
436             text_str = etree.tostring(footnote, method='text',
437                                       encoding='unicode').strip()
438             html_str = etree.tostring(footnote, method='html',
439                                       encoding='unicode').strip()
440
441             match = re_qualifier.match(text_str)
442             if match:
443                 qualifier_str = match.group(1)
444                 qualifiers = []
445                 for candidate in re.split('[;,]', qualifier_str):
446                     candidate = candidate.strip()
447                     if candidate in FN_QUALIFIERS:
448                         qualifiers.append(candidate)
449                     elif candidate.startswith('z '):
450                         subcandidate = candidate.split()[1]
451                         if subcandidate in FN_QUALIFIERS:
452                             qualifiers.append(subcandidate)
453             else:
454                 qualifiers = []
455
456             yield anchor, fn_type, qualifiers, text_str, html_str