Some reorg.
[librarian.git] / src / librarian / html.py
1 # -*- coding: utf-8 -*-
2 #
3 # This file is part of Librarian, licensed under GNU Affero GPLv3 or later.
4 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
5 #
6 from __future__ import print_function, unicode_literals
7
8 import os
9 import re
10 import copy
11
12 from lxml import etree
13 from librarian import XHTMLNS, ParseError, OutputFile
14 from librarian import functions
15 from PIL import Image
16
17 from lxml.etree import XMLSyntaxError, XSLTApplyError
18 import six
19
20
21 functions.reg_substitute_entities()
22 functions.reg_person_name()
23
24 STYLESHEETS = {
25     'legacy': 'xslt/book2html.xslt',
26     'full': 'xslt/wl2html_full.xslt',
27     'partial': 'xslt/wl2html_partial.xslt'
28 }
29
30
31 def get_stylesheet(name):
32     return os.path.join(os.path.dirname(__file__), STYLESHEETS[name])
33
34
35 def html_has_content(text):
36     return etree.ETXPath(
37         '//p|//{%(ns)s}p|//h1|//{%(ns)s}h1' % {'ns': str(XHTMLNS)}
38     )(text)
39
40
41 def transform_abstrakt(abstrakt_element):
42     style_filename = get_stylesheet('legacy')
43     style = etree.parse(style_filename)
44     xml = etree.tostring(abstrakt_element, encoding='unicode')
45     document = etree.parse(six.StringIO(
46         xml.replace('abstrakt', 'dlugi_cytat')
47     ))  # HACK
48     result = document.xslt(style)
49     html = re.sub('<a name="sec[0-9]*"/>', '',
50                   etree.tostring(result, encoding='unicode'))
51     return re.sub('</?blockquote[^>]*>', '', html)
52
53
54 def add_image_sizes(tree, gallery_path, gallery_url, base_url):
55     widths = [360, 600, 1200, 1800, 2400]
56
57     for i, ilustr in enumerate(tree.findall('//ilustr')):
58         rel_path = ilustr.attrib['src']
59         img_url = six.moves.urllib.parse.urljoin(base_url, rel_path)
60
61         f = six.moves.urllib.request.urlopen(img_url)
62         img = Image.open(f)
63         ext = {'GIF': 'gif', 'PNG': 'png'}.get(img.format, 'jpg')
64
65         srcset = []
66         # Needed widths: predefined and original, limited by
67         # whichever is smaller.
68         img_widths = [
69             w for w in
70             sorted(
71                 set(widths + [img.size[0]])
72             )
73             if w <= min(widths[-1], img.size[0])
74         ]
75         largest = None
76         for w in widths:
77             fname = '%d.W%d.%s' % (i, w, ext)
78             fpath = gallery_path + fname
79             if not os.path.exists(fpath):
80                 height = round(img.size[1] * w / img.size[0])
81                 th = img.resize((w, height))
82                 th.save(fpath)
83             th_url = gallery_url + fname
84             srcset.append(" ".join((
85                 th_url,
86                 '%dw' % w
87             )))
88             largest_url = th_url
89         ilustr.attrib['srcset'] = ", ".join(srcset)
90         ilustr.attrib['src'] = largest_url
91
92         f.close()
93
94
95 def transform(wldoc, stylesheet='legacy', options=None, flags=None, css=None, gallery_path='img/', gallery_url='img/', base_url='file://./'):
96     """Transforms the WL document to XHTML.
97
98     If output_filename is None, returns an XML,
99     otherwise returns True if file has been written,False if it hasn't.
100     File won't be written if it has no content.
101     """
102     # Parse XSLT
103     try:
104         style_filename = get_stylesheet(stylesheet)
105         style = etree.parse(style_filename)
106
107         document = copy.deepcopy(wldoc)
108         del wldoc
109         document.swap_endlines()
110
111         if flags:
112             for flag in flags:
113                 document.edoc.getroot().set(flag, 'yes')
114
115         document.clean_ed_note()
116         document.clean_ed_note('abstrakt')
117         document.fix_pa_akap()
118         
119         if not options:
120             options = {}
121
122         try:
123             os.makedirs(gallery_path)
124         except OSError:
125             pass
126
127         add_image_sizes(document.edoc, gallery_path, gallery_url, base_url)
128
129         css = (
130             css
131             or 'https://static.wolnelektury.pl/css/compressed/book_text.css'
132         )
133         css = "'%s'" % css
134         result = document.transform(style, css=css, **options)
135         del document  # no longer needed large object :)
136
137         if html_has_content(result):
138             add_anchors(result.getroot())
139             add_table_of_themes(result.getroot())
140             add_table_of_contents(result.getroot())
141
142             return OutputFile.from_bytes(etree.tostring(
143                 result, method='html', xml_declaration=False,
144                 pretty_print=True, encoding='utf-8'
145             ))
146         else:
147             return None
148     except KeyError:
149         raise ValueError("'%s' is not a valid stylesheet.")
150     except (XMLSyntaxError, XSLTApplyError) as e:
151         raise ParseError(e)
152
153
154 @six.python_2_unicode_compatible
155 class Fragment(object):
156     def __init__(self, id, themes):
157         super(Fragment, self).__init__()
158         self.id = id
159         self.themes = themes
160         self.events = []
161
162     def append(self, event, element):
163         self.events.append((event, element))
164
165     def closed_events(self):
166         stack = []
167         for event, element in self.events:
168             if event == 'start':
169                 stack.append(('end', element))
170             elif event == 'end':
171                 try:
172                     stack.pop()
173                 except IndexError:
174                     print('CLOSED NON-OPEN TAG:', element)
175
176         stack.reverse()
177         return self.events + stack
178
179     def to_string(self):
180         result = []
181         for event, element in self.closed_events():
182             if event == 'start':
183                 result.append(u'<%s %s>' % (
184                     element.tag,
185                     ' '.join(
186                         '%s="%s"' % (k, v)
187                         for k, v in element.attrib.items()
188                     )
189                 ))
190                 if element.text:
191                     result.append(element.text)
192             elif event == 'end':
193                 result.append(u'</%s>' % element.tag)
194                 if element.tail:
195                     result.append(element.tail)
196             else:
197                 result.append(element)
198
199         return ''.join(result)
200
201     def __str__(self):
202         return self.to_string()
203
204
205 def extract_fragments(input_filename):
206     """Extracts theme fragments from input_filename."""
207     open_fragments = {}
208     closed_fragments = {}
209
210     # iterparse would die on a HTML document
211     parser = etree.HTMLParser(encoding='utf-8')
212     buf = six.BytesIO()
213     buf.write(etree.tostring(
214         etree.parse(input_filename, parser).getroot()[0][0],
215         encoding='utf-8'
216     ))
217     buf.seek(0)
218
219     for event, element in etree.iterparse(buf, events=('start', 'end')):
220         # Process begin and end elements
221         if element.get('class', '') in ('theme-begin', 'theme-end'):
222             if not event == 'end':
223                 continue  # Process elements only once, on end event
224
225             # Open new fragment
226             if element.get('class', '') == 'theme-begin':
227                 fragment = Fragment(id=element.get('fid'), themes=element.text)
228
229                 # Append parents
230                 parent = element.getparent()
231                 parents = []
232                 while parent.get('id', None) != 'book-text':
233                     cparent = copy.deepcopy(parent)
234                     cparent.text = None
235                     if 'id' in cparent.attrib:
236                         del cparent.attrib['id']
237                     parents.append(cparent)
238                     parent = parent.getparent()
239
240                 parents.reverse()
241                 for parent in parents:
242                     fragment.append('start', parent)
243
244                 open_fragments[fragment.id] = fragment
245
246             # Close existing fragment
247             else:
248                 try:
249                     fragment = open_fragments[element.get('fid')]
250                 except KeyError:
251                     print('%s:closed not open fragment #%s' % (
252                         input_filename, element.get('fid')
253                     ))
254                 else:
255                     closed_fragments[fragment.id] = fragment
256                     del open_fragments[fragment.id]
257
258             # Append element tail to lost_text
259             # (we don't want to lose any text)
260             if element.tail:
261                 for fragment_id in open_fragments:
262                     open_fragments[fragment_id].append('text', element.tail)
263
264         # Process all elements except begin and end
265         else:
266             # Omit annotation tags
267             if (len(element.get('name', '')) or
268                     element.get('class', '') in ('annotation', 'anchor')):
269                 if event == 'end' and element.tail:
270                     for fragment_id in open_fragments:
271                         open_fragments[fragment_id].append(
272                             'text', element.tail
273                         )
274             else:
275                 for fragment_id in open_fragments:
276                     celem = copy.copy(element)
277                     if 'id' in celem.attrib:
278                         del celem.attrib['id']
279                     open_fragments[fragment_id].append(
280                         event, celem
281                     )
282
283     return closed_fragments, open_fragments
284
285
286 def add_anchor(element, prefix, with_link=True, with_target=True,
287                link_text=None):
288     parent = element.getparent()
289     index = parent.index(element)
290
291     if with_link:
292         if link_text is None:
293             link_text = prefix
294         anchor = etree.Element('a', href='#%s' % prefix)
295         anchor.set('class', 'anchor')
296         anchor.text = six.text_type(link_text)
297         parent.insert(index, anchor)
298
299     if with_target:
300         anchor_target = etree.Element('a', name='%s' % prefix)
301         anchor_target.set('class', 'target')
302         anchor_target.text = u' '
303         parent.insert(index, anchor_target)
304
305
306 def any_ancestor(element, test):
307     for ancestor in element.iterancestors():
308         if test(ancestor):
309             return True
310     return False
311
312
313 def add_anchors(root):
314     counter = 1
315     visible_counter = 1
316     for element in root.iterdescendants():
317         def f(e):
318             return (
319                 e.get('class') in (
320                     'note', 'motto', 'motto_podpis', 'dedication', 'frame'
321                 )
322                 or e.get('id') == 'nota_red'
323                 or e.tag == 'blockquote'
324                 or e.get('id') == 'footnotes'
325             )
326
327         if element.get('class') == 'numeracja':
328             try:
329                 visible_counter = int(element.get('data-start'))
330             except ValueError:
331                 visible_counter = 1
332
333         if any_ancestor(element, f):
334             continue
335
336         if element.tag == 'div' and 'verse' in element.get('class', ''):
337             if visible_counter == 1 or visible_counter % 5 == 0:
338                 add_anchor(element, "f%d" % counter, link_text=visible_counter)
339             counter += 1
340             visible_counter += 1
341         elif 'paragraph' in element.get('class', ''):
342             add_anchor(element, "f%d" % counter, link_text=visible_counter)
343             counter += 1
344             visible_counter += 1
345
346
347 def raw_printable_text(element):
348     working = copy.deepcopy(element)
349     for e in working.findall('a'):
350         if e.get('class') in ('annotation', 'theme-begin'):
351             e.text = ''
352     return etree.tostring(working, method='text', encoding='unicode').strip()
353
354
355 def add_table_of_contents(root):
356     sections = []
357     counter = 1
358     for element in root.iterdescendants():
359         if element.tag in ('h2', 'h3'):
360             if any_ancestor(
361                     element,
362                     lambda e: e.get('id') in (
363                         'footnotes', 'nota_red'
364                     ) or e.get('class') in ('person-list',)):
365                 continue
366
367             element_text = raw_printable_text(element)
368             if (element.tag == 'h3' and len(sections)
369                     and sections[-1][1] == 'h2'):
370                 sections[-1][3].append(
371                     (counter, element.tag, element_text, [])
372                 )
373             else:
374                 sections.append((counter, element.tag, element_text, []))
375             add_anchor(element, "s%d" % counter, with_link=False)
376             counter += 1
377
378     toc = etree.Element('div')
379     toc.set('id', 'toc')
380     toc_header = etree.SubElement(toc, 'h2')
381     toc_header.text = u'Spis treści'
382     toc_list = etree.SubElement(toc, 'ol')
383
384     for n, section, text, subsections in sections:
385         section_element = etree.SubElement(toc_list, 'li')
386         add_anchor(section_element, "s%d" % n, with_target=False,
387                    link_text=text)
388
389         if len(subsections):
390             subsection_list = etree.SubElement(section_element, 'ol')
391             for n1, subsection, subtext, _ in subsections:
392                 subsection_element = etree.SubElement(subsection_list, 'li')
393                 add_anchor(subsection_element, "s%d" % n1, with_target=False,
394                            link_text=subtext)
395
396     root.insert(0, toc)
397
398
399 def add_table_of_themes(root):
400     try:
401         from sortify import sortify
402     except ImportError:
403         def sortify(x):
404             return x
405
406     book_themes = {}
407     for fragment in root.findall('.//a[@class="theme-begin"]'):
408         if not fragment.text:
409             continue
410         theme_names = [s.strip() for s in fragment.text.split(',')]
411         for theme_name in theme_names:
412             book_themes.setdefault(theme_name, []).append(fragment.get('name'))
413     book_themes = list(book_themes.items())
414     book_themes.sort(key=lambda s: sortify(s[0]))
415     themes_div = etree.Element('div', id="themes")
416     themes_ol = etree.SubElement(themes_div, 'ol')
417     for theme_name, fragments in book_themes:
418         themes_li = etree.SubElement(themes_ol, 'li')
419         themes_li.text = "%s: " % theme_name
420         for i, fragment in enumerate(fragments):
421             item = etree.SubElement(themes_li, 'a', href="#%s" % fragment)
422             item.text = str(i + 1)
423             item.tail = ' '
424     root.insert(0, themes_div)
425
426
427 def extract_annotations(html_path):
428     """Extracts annotations from HTML for annotations dictionary.
429
430     For each annotation, yields a tuple of:
431     anchor, footnote type, valid qualifiers, text, html.
432
433     """
434     from .fn_qualifiers import FN_QUALIFIERS
435
436     parser = etree.HTMLParser(encoding='utf-8')
437     tree = etree.parse(html_path, parser)
438     footnotes = tree.find('//*[@id="footnotes"]')
439     re_qualifier = re.compile(r'[^\u2014]+\s+\(([^\)]+)\)\s+\u2014')
440     if footnotes is not None:
441         for footnote in footnotes.findall('div'):
442             fn_type = footnote.get('class').split('-')[1]
443             anchor = footnote.find('a[@class="annotation"]').get('href')[1:]
444             del footnote[:2]
445             footnote.text = None
446             if len(footnote) and footnote[-1].tail == '\n':
447                 footnote[-1].tail = None
448             text_str = etree.tostring(footnote, method='text',
449                                       encoding='unicode').strip()
450             html_str = etree.tostring(footnote, method='html',
451                                       encoding='unicode').strip()
452
453             match = re_qualifier.match(text_str)
454             if match:
455                 qualifier_str = match.group(1)
456                 qualifiers = []
457                 for candidate in re.split('[;,]', qualifier_str):
458                     candidate = candidate.strip()
459                     if candidate in FN_QUALIFIERS:
460                         qualifiers.append(candidate)
461                     elif candidate.startswith('z '):
462                         subcandidate = candidate.split()[1]
463                         if subcandidate in FN_QUALIFIERS:
464                             qualifiers.append(subcandidate)
465             else:
466                 qualifiers = []
467
468             yield anchor, fn_type, qualifiers, text_str, html_str