363286cfb3dda9c5764c97faf20deee8340a5c15
[librarian.git] / src / librarian / html.py
1 # -*- coding: utf-8 -*-
2 #
3 # This file is part of Librarian, licensed under GNU Affero GPLv3 or later.
4 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
5 #
6 from __future__ import print_function, unicode_literals
7
8 import os
9 import re
10 import copy
11
12 from lxml import etree
13 from librarian import XHTMLNS, ParseError, OutputFile
14 from librarian import functions
15 from PIL import Image
16
17 from lxml.etree import XMLSyntaxError, XSLTApplyError
18 import six
19
20
21 functions.reg_substitute_entities()
22 functions.reg_person_name()
23
24 STYLESHEETS = {
25     'legacy': 'xslt/book2html.xslt',
26     'full': 'xslt/wl2html_full.xslt',
27     'partial': 'xslt/wl2html_partial.xslt'
28 }
29
30
31 def get_stylesheet(name):
32     return os.path.join(os.path.dirname(__file__), STYLESHEETS[name])
33
34
35 def html_has_content(text):
36     return etree.ETXPath(
37         '//p|//{%(ns)s}p|//h1|//{%(ns)s}h1' % {'ns': str(XHTMLNS)}
38     )(text)
39
40
41 def transform_abstrakt(abstrakt_element):
42     style_filename = get_stylesheet('legacy')
43     style = etree.parse(style_filename)
44     xml = etree.tostring(abstrakt_element, encoding='unicode')
45     document = etree.parse(six.StringIO(
46         xml.replace('abstrakt', 'dlugi_cytat')
47     ))  # HACK
48     result = document.xslt(style)
49     html = re.sub('<a name="sec[0-9]*"/>', '',
50                   etree.tostring(result, encoding='unicode'))
51     return re.sub('</?blockquote[^>]*>', '', html)
52
53
54 def add_image_sizes(tree, gallery_path, gallery_url, base_url):
55     widths = [360, 600, 1200, 1800, 2400]
56
57     for i, ilustr in enumerate(tree.findall('//ilustr')):
58         rel_path = ilustr.attrib['src']
59         img_url = six.moves.urllib.parse.urljoin(base_url, rel_path)
60
61         with six.moves.urllib.request.urlopen(img_url) as f:
62             img = Image.open(f)
63
64         ext = {'GIF': 'gif', 'PNG': 'png'}.get(img.format, 'jpg')
65
66         srcset = []
67         # Needed widths: predefined and original, limited by
68         # whichever is smaller.
69         img_widths = [
70             w for w in
71             sorted(
72                 set(widths + [img.size[0]])
73             )
74             if w <= min(widths[-1], img.size[0])
75         ]
76         largest = None
77         for w in widths:
78             height = round(img.size[1] * w / img.size[0])
79             th = img.resize((w, height))
80             fname = '%d.W%d.%s' % (i, w, ext)
81             th.save(gallery_path + fname)
82             th_url = gallery_url + fname
83             srcset.append(" ".join((
84                 th_url,
85                 '%dw' % w
86             )))
87             largest_url = th_url
88         ilustr.attrib['srcset'] = ", ".join(srcset)
89         ilustr.attrib['src'] = largest_url
90
91
92 def transform(wldoc, stylesheet='legacy', options=None, flags=None, css=None, gallery_path='img/', gallery_url='img/', base_url='file://./'):
93     """Transforms the WL document to XHTML.
94
95     If output_filename is None, returns an XML,
96     otherwise returns True if file has been written,False if it hasn't.
97     File won't be written if it has no content.
98     """
99     # Parse XSLT
100     try:
101         style_filename = get_stylesheet(stylesheet)
102         style = etree.parse(style_filename)
103
104         document = copy.deepcopy(wldoc)
105         del wldoc
106         document.swap_endlines()
107
108         if flags:
109             for flag in flags:
110                 document.edoc.getroot().set(flag, 'yes')
111
112         document.clean_ed_note()
113         document.clean_ed_note('abstrakt')
114
115         if not options:
116             options = {}
117
118         try:
119             os.makedirs(gallery_path)
120         except OSError:
121             pass
122
123         add_image_sizes(document.edoc, gallery_path, gallery_url, base_url)
124
125         css = (
126             css
127             or 'https://static.wolnelektury.pl/css/compressed/book_text.css'
128         )
129         css = "'%s'" % css
130         result = document.transform(style, css=css, **options)
131         del document  # no longer needed large object :)
132
133         if html_has_content(result):
134             add_anchors(result.getroot())
135             add_table_of_themes(result.getroot())
136             add_table_of_contents(result.getroot())
137
138             return OutputFile.from_bytes(etree.tostring(
139                 result, method='html', xml_declaration=False,
140                 pretty_print=True, encoding='utf-8'
141             ))
142         else:
143             return None
144     except KeyError:
145         raise ValueError("'%s' is not a valid stylesheet.")
146     except (XMLSyntaxError, XSLTApplyError) as e:
147         raise ParseError(e)
148
149
150 @six.python_2_unicode_compatible
151 class Fragment(object):
152     def __init__(self, id, themes):
153         super(Fragment, self).__init__()
154         self.id = id
155         self.themes = themes
156         self.events = []
157
158     def append(self, event, element):
159         self.events.append((event, element))
160
161     def closed_events(self):
162         stack = []
163         for event, element in self.events:
164             if event == 'start':
165                 stack.append(('end', element))
166             elif event == 'end':
167                 try:
168                     stack.pop()
169                 except IndexError:
170                     print('CLOSED NON-OPEN TAG:', element)
171
172         stack.reverse()
173         return self.events + stack
174
175     def to_string(self):
176         result = []
177         for event, element in self.closed_events():
178             if event == 'start':
179                 result.append(u'<%s %s>' % (
180                     element.tag,
181                     ' '.join(
182                         '%s="%s"' % (k, v)
183                         for k, v in element.attrib.items()
184                     )
185                 ))
186                 if element.text:
187                     result.append(element.text)
188             elif event == 'end':
189                 result.append(u'</%s>' % element.tag)
190                 if element.tail:
191                     result.append(element.tail)
192             else:
193                 result.append(element)
194
195         return ''.join(result)
196
197     def __str__(self):
198         return self.to_string()
199
200
201 def extract_fragments(input_filename):
202     """Extracts theme fragments from input_filename."""
203     open_fragments = {}
204     closed_fragments = {}
205
206     # iterparse would die on a HTML document
207     parser = etree.HTMLParser(encoding='utf-8')
208     buf = six.BytesIO()
209     buf.write(etree.tostring(
210         etree.parse(input_filename, parser).getroot()[0][0],
211         encoding='utf-8'
212     ))
213     buf.seek(0)
214
215     for event, element in etree.iterparse(buf, events=('start', 'end')):
216         # Process begin and end elements
217         if element.get('class', '') in ('theme-begin', 'theme-end'):
218             if not event == 'end':
219                 continue  # Process elements only once, on end event
220
221             # Open new fragment
222             if element.get('class', '') == 'theme-begin':
223                 fragment = Fragment(id=element.get('fid'), themes=element.text)
224
225                 # Append parents
226                 parent = element.getparent()
227                 parents = []
228                 while parent.get('id', None) != 'book-text':
229                     cparent = copy.deepcopy(parent)
230                     cparent.text = None
231                     if 'id' in cparent.attrib:
232                         del cparent.attrib['id']
233                     parents.append(cparent)
234                     parent = parent.getparent()
235
236                 parents.reverse()
237                 for parent in parents:
238                     fragment.append('start', parent)
239
240                 open_fragments[fragment.id] = fragment
241
242             # Close existing fragment
243             else:
244                 try:
245                     fragment = open_fragments[element.get('fid')]
246                 except KeyError:
247                     print('%s:closed not open fragment #%s' % (
248                         input_filename, element.get('fid')
249                     ))
250                 else:
251                     closed_fragments[fragment.id] = fragment
252                     del open_fragments[fragment.id]
253
254             # Append element tail to lost_text
255             # (we don't want to lose any text)
256             if element.tail:
257                 for fragment_id in open_fragments:
258                     open_fragments[fragment_id].append('text', element.tail)
259
260         # Process all elements except begin and end
261         else:
262             # Omit annotation tags
263             if (len(element.get('name', '')) or
264                     element.get('class', '') in ('annotation', 'anchor')):
265                 if event == 'end' and element.tail:
266                     for fragment_id in open_fragments:
267                         open_fragments[fragment_id].append(
268                             'text', element.tail
269                         )
270             else:
271                 for fragment_id in open_fragments:
272                     celem = copy.copy(element)
273                     if 'id' in celem.attrib:
274                         del celem.attrib['id']
275                     open_fragments[fragment_id].append(
276                         event, celem
277                     )
278
279     return closed_fragments, open_fragments
280
281
282 def add_anchor(element, prefix, with_link=True, with_target=True,
283                link_text=None):
284     parent = element.getparent()
285     index = parent.index(element)
286
287     if with_link:
288         if link_text is None:
289             link_text = prefix
290         anchor = etree.Element('a', href='#%s' % prefix)
291         anchor.set('class', 'anchor')
292         anchor.text = six.text_type(link_text)
293         parent.insert(index, anchor)
294
295     if with_target:
296         anchor_target = etree.Element('a', name='%s' % prefix)
297         anchor_target.set('class', 'target')
298         anchor_target.text = u' '
299         parent.insert(index, anchor_target)
300
301
302 def any_ancestor(element, test):
303     for ancestor in element.iterancestors():
304         if test(ancestor):
305             return True
306     return False
307
308
309 def add_anchors(root):
310     counter = 1
311     for element in root.iterdescendants():
312         def f(e):
313             return (
314                 e.get('class') in (
315                     'note', 'motto', 'motto_podpis', 'dedication', 'frame'
316                 )
317                 or e.get('id') == 'nota_red'
318                 or e.tag == 'blockquote'
319             )
320         if any_ancestor(element, f):
321             continue
322
323         if element.tag == 'div' and 'verse' in element.get('class', ''):
324             if counter == 1 or counter % 5 == 0:
325                 add_anchor(element, "f%d" % counter, link_text=counter)
326             counter += 1
327         elif 'paragraph' in element.get('class', ''):
328             add_anchor(element, "f%d" % counter, link_text=counter)
329             counter += 1
330
331
332 def raw_printable_text(element):
333     working = copy.deepcopy(element)
334     for e in working.findall('a'):
335         if e.get('class') in ('annotation', 'theme-begin'):
336             e.text = ''
337     return etree.tostring(working, method='text', encoding='unicode').strip()
338
339
340 def add_table_of_contents(root):
341     sections = []
342     counter = 1
343     for element in root.iterdescendants():
344         if element.tag in ('h2', 'h3'):
345             if any_ancestor(
346                     element,
347                     lambda e: e.get('id') in (
348                         'footnotes', 'nota_red'
349                     ) or e.get('class') in ('person-list',)):
350                 continue
351
352             element_text = raw_printable_text(element)
353             if (element.tag == 'h3' and len(sections)
354                     and sections[-1][1] == 'h2'):
355                 sections[-1][3].append(
356                     (counter, element.tag, element_text, [])
357                 )
358             else:
359                 sections.append((counter, element.tag, element_text, []))
360             add_anchor(element, "s%d" % counter, with_link=False)
361             counter += 1
362
363     toc = etree.Element('div')
364     toc.set('id', 'toc')
365     toc_header = etree.SubElement(toc, 'h2')
366     toc_header.text = u'Spis treści'
367     toc_list = etree.SubElement(toc, 'ol')
368
369     for n, section, text, subsections in sections:
370         section_element = etree.SubElement(toc_list, 'li')
371         add_anchor(section_element, "s%d" % n, with_target=False,
372                    link_text=text)
373
374         if len(subsections):
375             subsection_list = etree.SubElement(section_element, 'ol')
376             for n1, subsection, subtext, _ in subsections:
377                 subsection_element = etree.SubElement(subsection_list, 'li')
378                 add_anchor(subsection_element, "s%d" % n1, with_target=False,
379                            link_text=subtext)
380
381     root.insert(0, toc)
382
383
384 def add_table_of_themes(root):
385     try:
386         from sortify import sortify
387     except ImportError:
388         def sortify(x):
389             return x
390
391     book_themes = {}
392     for fragment in root.findall('.//a[@class="theme-begin"]'):
393         if not fragment.text:
394             continue
395         theme_names = [s.strip() for s in fragment.text.split(',')]
396         for theme_name in theme_names:
397             book_themes.setdefault(theme_name, []).append(fragment.get('name'))
398     book_themes = list(book_themes.items())
399     book_themes.sort(key=lambda s: sortify(s[0]))
400     themes_div = etree.Element('div', id="themes")
401     themes_ol = etree.SubElement(themes_div, 'ol')
402     for theme_name, fragments in book_themes:
403         themes_li = etree.SubElement(themes_ol, 'li')
404         themes_li.text = "%s: " % theme_name
405         for i, fragment in enumerate(fragments):
406             item = etree.SubElement(themes_li, 'a', href="#%s" % fragment)
407             item.text = str(i + 1)
408             item.tail = ' '
409     root.insert(0, themes_div)
410
411
412 def extract_annotations(html_path):
413     """Extracts annotations from HTML for annotations dictionary.
414
415     For each annotation, yields a tuple of:
416     anchor, footnote type, valid qualifiers, text, html.
417
418     """
419     from .fn_qualifiers import FN_QUALIFIERS
420
421     parser = etree.HTMLParser(encoding='utf-8')
422     tree = etree.parse(html_path, parser)
423     footnotes = tree.find('//*[@id="footnotes"]')
424     re_qualifier = re.compile(r'[^\u2014]+\s+\(([^\)]+)\)\s+\u2014')
425     if footnotes is not None:
426         for footnote in footnotes.findall('div'):
427             fn_type = footnote.get('class').split('-')[1]
428             anchor = footnote.find('a[@class="annotation"]').get('href')[1:]
429             del footnote[:2]
430             footnote.text = None
431             if len(footnote) and footnote[-1].tail == '\n':
432                 footnote[-1].tail = None
433             text_str = etree.tostring(footnote, method='text',
434                                       encoding='unicode').strip()
435             html_str = etree.tostring(footnote, method='html',
436                                       encoding='unicode').strip()
437
438             match = re_qualifier.match(text_str)
439             if match:
440                 qualifier_str = match.group(1)
441                 qualifiers = []
442                 for candidate in re.split('[;,]', qualifier_str):
443                     candidate = candidate.strip()
444                     if candidate in FN_QUALIFIERS:
445                         qualifiers.append(candidate)
446                     elif candidate.startswith('z '):
447                         subcandidate = candidate.split()[1]
448                         if subcandidate in FN_QUALIFIERS:
449                             qualifiers.append(subcandidate)
450             else:
451                 qualifiers = []
452
453             yield anchor, fn_type, qualifiers, text_str, html_str