Support for ref.
[librarian.git] / src / librarian / html.py
1 # -*- coding: utf-8 -*-
2 #
3 # This file is part of Librarian, licensed under GNU Affero GPLv3 or later.
4 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
5 #
6 from __future__ import print_function, unicode_literals
7
8 import os
9 import re
10 import copy
11
12 from lxml import etree
13 from librarian import XHTMLNS, ParseError, OutputFile
14 from librarian import functions
15
16 from lxml.etree import XMLSyntaxError, XSLTApplyError
17 import six
18
19
20 functions.reg_substitute_entities()
21 functions.reg_person_name()
22
23 STYLESHEETS = {
24     'legacy': 'xslt/book2html.xslt',
25     'full': 'xslt/wl2html_full.xslt',
26     'partial': 'xslt/wl2html_partial.xslt'
27 }
28
29
30 def get_stylesheet(name):
31     return os.path.join(os.path.dirname(__file__), STYLESHEETS[name])
32
33
34 def html_has_content(text):
35     return etree.ETXPath(
36         '//p|//{%(ns)s}p|//h1|//{%(ns)s}h1' % {'ns': str(XHTMLNS)}
37     )(text)
38
39
40 def transform_abstrakt(abstrakt_element):
41     style_filename = get_stylesheet('legacy')
42     style = etree.parse(style_filename)
43     xml = etree.tostring(abstrakt_element, encoding='unicode')
44     document = etree.parse(six.StringIO(
45         xml.replace('abstrakt', 'dlugi_cytat')
46     ))  # HACK
47     result = document.xslt(style)
48     html = re.sub('<a name="sec[0-9]*"/>', '',
49                   etree.tostring(result, encoding='unicode'))
50     return re.sub('</?blockquote[^>]*>', '', html)
51
52
53 def transform(wldoc, stylesheet='legacy', options=None, flags=None, css=None):
54     """Transforms the WL document to XHTML.
55
56     If output_filename is None, returns an XML,
57     otherwise returns True if file has been written,False if it hasn't.
58     File won't be written if it has no content.
59     """
60     # Parse XSLT
61     try:
62         style_filename = get_stylesheet(stylesheet)
63         style = etree.parse(style_filename)
64
65         document = copy.deepcopy(wldoc)
66         del wldoc
67         document.swap_endlines()
68
69         if flags:
70             for flag in flags:
71                 document.edoc.getroot().set(flag, 'yes')
72
73         document.clean_ed_note()
74         document.clean_ed_note('abstrakt')
75
76         if not options:
77             options = {}
78         options.setdefault('gallery', "''")
79
80         css = (
81             css
82             or 'https://static.wolnelektury.pl/css/compressed/book_text.css'
83         )
84         css = "'%s'" % css
85         result = document.transform(style, css=css, **options)
86         del document  # no longer needed large object :)
87
88         if html_has_content(result):
89             add_anchors(result.getroot())
90             add_table_of_themes(result.getroot())
91             add_table_of_contents(result.getroot())
92
93             return OutputFile.from_bytes(etree.tostring(
94                 result, method='html', xml_declaration=False,
95                 pretty_print=True, encoding='utf-8'
96             ))
97         else:
98             return None
99     except KeyError:
100         raise ValueError("'%s' is not a valid stylesheet.")
101     except (XMLSyntaxError, XSLTApplyError) as e:
102         raise ParseError(e)
103
104
105 @six.python_2_unicode_compatible
106 class Fragment(object):
107     def __init__(self, id, themes):
108         super(Fragment, self).__init__()
109         self.id = id
110         self.themes = themes
111         self.events = []
112
113     def append(self, event, element):
114         self.events.append((event, element))
115
116     def closed_events(self):
117         stack = []
118         for event, element in self.events:
119             if event == 'start':
120                 stack.append(('end', element))
121             elif event == 'end':
122                 try:
123                     stack.pop()
124                 except IndexError:
125                     print('CLOSED NON-OPEN TAG:', element)
126
127         stack.reverse()
128         return self.events + stack
129
130     def to_string(self):
131         result = []
132         for event, element in self.closed_events():
133             if event == 'start':
134                 result.append(u'<%s %s>' % (
135                     element.tag,
136                     ' '.join(
137                         '%s="%s"' % (k, v)
138                         for k, v in element.attrib.items()
139                     )
140                 ))
141                 if element.text:
142                     result.append(element.text)
143             elif event == 'end':
144                 result.append(u'</%s>' % element.tag)
145                 if element.tail:
146                     result.append(element.tail)
147             else:
148                 result.append(element)
149
150         return ''.join(result)
151
152     def __str__(self):
153         return self.to_string()
154
155
156 def extract_fragments(input_filename):
157     """Extracts theme fragments from input_filename."""
158     open_fragments = {}
159     closed_fragments = {}
160
161     # iterparse would die on a HTML document
162     parser = etree.HTMLParser(encoding='utf-8')
163     buf = six.BytesIO()
164     buf.write(etree.tostring(
165         etree.parse(input_filename, parser).getroot()[0][0],
166         encoding='utf-8'
167     ))
168     buf.seek(0)
169
170     for event, element in etree.iterparse(buf, events=('start', 'end')):
171         # Process begin and end elements
172         if element.get('class', '') in ('theme-begin', 'theme-end'):
173             if not event == 'end':
174                 continue  # Process elements only once, on end event
175
176             # Open new fragment
177             if element.get('class', '') == 'theme-begin':
178                 fragment = Fragment(id=element.get('fid'), themes=element.text)
179
180                 # Append parents
181                 parent = element.getparent()
182                 parents = []
183                 while parent.get('id', None) != 'book-text':
184                     cparent = copy.deepcopy(parent)
185                     cparent.text = None
186                     if 'id' in cparent.attrib:
187                         del cparent.attrib['id']
188                     parents.append(cparent)
189                     parent = parent.getparent()
190
191                 parents.reverse()
192                 for parent in parents:
193                     fragment.append('start', parent)
194
195                 open_fragments[fragment.id] = fragment
196
197             # Close existing fragment
198             else:
199                 try:
200                     fragment = open_fragments[element.get('fid')]
201                 except KeyError:
202                     print('%s:closed not open fragment #%s' % (
203                         input_filename, element.get('fid')
204                     ))
205                 else:
206                     closed_fragments[fragment.id] = fragment
207                     del open_fragments[fragment.id]
208
209             # Append element tail to lost_text
210             # (we don't want to lose any text)
211             if element.tail:
212                 for fragment_id in open_fragments:
213                     open_fragments[fragment_id].append('text', element.tail)
214
215         # Process all elements except begin and end
216         else:
217             # Omit annotation tags
218             if (len(element.get('name', '')) or
219                     element.get('class', '') in ('annotation', 'anchor')):
220                 if event == 'end' and element.tail:
221                     for fragment_id in open_fragments:
222                         open_fragments[fragment_id].append(
223                             'text', element.tail
224                         )
225             else:
226                 for fragment_id in open_fragments:
227                     celem = copy.copy(element)
228                     if 'id' in celem.attrib:
229                         del celem.attrib['id']
230                     open_fragments[fragment_id].append(
231                         event, celem
232                     )
233
234     return closed_fragments, open_fragments
235
236
237 def add_anchor(element, prefix, with_link=True, with_target=True,
238                link_text=None):
239     parent = element.getparent()
240     index = parent.index(element)
241
242     if with_link:
243         if link_text is None:
244             link_text = prefix
245         anchor = etree.Element('a', href='#%s' % prefix)
246         anchor.set('class', 'anchor')
247         anchor.text = six.text_type(link_text)
248         parent.insert(index, anchor)
249
250     if with_target:
251         anchor_target = etree.Element('a', name='%s' % prefix)
252         anchor_target.set('class', 'target')
253         anchor_target.text = u' '
254         parent.insert(index, anchor_target)
255
256
257 def any_ancestor(element, test):
258     for ancestor in element.iterancestors():
259         if test(ancestor):
260             return True
261     return False
262
263
264 def add_anchors(root):
265     counter = 1
266     for element in root.iterdescendants():
267         def f(e):
268             return (
269                 e.get('class') in (
270                     'note', 'motto', 'motto_podpis', 'dedication', 'frame'
271                 )
272                 or e.get('id') == 'nota_red'
273                 or e.tag == 'blockquote'
274             )
275         if any_ancestor(element, f):
276             continue
277
278         if element.tag == 'div' and 'verse' in element.get('class', ''):
279             if counter == 1 or counter % 5 == 0:
280                 add_anchor(element, "f%d" % counter, link_text=counter)
281             counter += 1
282         elif 'paragraph' in element.get('class', ''):
283             add_anchor(element, "f%d" % counter, link_text=counter)
284             counter += 1
285
286
287 def raw_printable_text(element):
288     working = copy.deepcopy(element)
289     for e in working.findall('a'):
290         if e.get('class') in ('annotation', 'theme-begin'):
291             e.text = ''
292     return etree.tostring(working, method='text', encoding='unicode').strip()
293
294
295 def add_table_of_contents(root):
296     sections = []
297     counter = 1
298     for element in root.iterdescendants():
299         if element.tag in ('h2', 'h3'):
300             if any_ancestor(
301                     element,
302                     lambda e: e.get('id') in (
303                         'footnotes', 'nota_red'
304                     ) or e.get('class') in ('person-list',)):
305                 continue
306
307             element_text = raw_printable_text(element)
308             if (element.tag == 'h3' and len(sections)
309                     and sections[-1][1] == 'h2'):
310                 sections[-1][3].append(
311                     (counter, element.tag, element_text, [])
312                 )
313             else:
314                 sections.append((counter, element.tag, element_text, []))
315             add_anchor(element, "s%d" % counter, with_link=False)
316             counter += 1
317
318     toc = etree.Element('div')
319     toc.set('id', 'toc')
320     toc_header = etree.SubElement(toc, 'h2')
321     toc_header.text = u'Spis treści'
322     toc_list = etree.SubElement(toc, 'ol')
323
324     for n, section, text, subsections in sections:
325         section_element = etree.SubElement(toc_list, 'li')
326         add_anchor(section_element, "s%d" % n, with_target=False,
327                    link_text=text)
328
329         if len(subsections):
330             subsection_list = etree.SubElement(section_element, 'ol')
331             for n1, subsection, subtext, _ in subsections:
332                 subsection_element = etree.SubElement(subsection_list, 'li')
333                 add_anchor(subsection_element, "s%d" % n1, with_target=False,
334                            link_text=subtext)
335
336     root.insert(0, toc)
337
338
339 def add_table_of_themes(root):
340     try:
341         from sortify import sortify
342     except ImportError:
343         def sortify(x):
344             return x
345
346     book_themes = {}
347     for fragment in root.findall('.//a[@class="theme-begin"]'):
348         if not fragment.text:
349             continue
350         theme_names = [s.strip() for s in fragment.text.split(',')]
351         for theme_name in theme_names:
352             book_themes.setdefault(theme_name, []).append(fragment.get('name'))
353     book_themes = list(book_themes.items())
354     book_themes.sort(key=lambda s: sortify(s[0]))
355     themes_div = etree.Element('div', id="themes")
356     themes_ol = etree.SubElement(themes_div, 'ol')
357     for theme_name, fragments in book_themes:
358         themes_li = etree.SubElement(themes_ol, 'li')
359         themes_li.text = "%s: " % theme_name
360         for i, fragment in enumerate(fragments):
361             item = etree.SubElement(themes_li, 'a', href="#%s" % fragment)
362             item.text = str(i + 1)
363             item.tail = ' '
364     root.insert(0, themes_div)
365
366
367 def extract_annotations(html_path):
368     """Extracts annotations from HTML for annotations dictionary.
369
370     For each annotation, yields a tuple of:
371     anchor, footnote type, valid qualifiers, text, html.
372
373     """
374     from .fn_qualifiers import FN_QUALIFIERS
375
376     parser = etree.HTMLParser(encoding='utf-8')
377     tree = etree.parse(html_path, parser)
378     footnotes = tree.find('//*[@id="footnotes"]')
379     re_qualifier = re.compile(r'[^\u2014]+\s+\(([^\)]+)\)\s+\u2014')
380     if footnotes is not None:
381         for footnote in footnotes.findall('div'):
382             fn_type = footnote.get('class').split('-')[1]
383             anchor = footnote.find('a[@class="annotation"]').get('href')[1:]
384             del footnote[:2]
385             footnote.text = None
386             if len(footnote) and footnote[-1].tail == '\n':
387                 footnote[-1].tail = None
388             text_str = etree.tostring(footnote, method='text',
389                                       encoding='unicode').strip()
390             html_str = etree.tostring(footnote, method='html',
391                                       encoding='unicode').strip()
392
393             match = re_qualifier.match(text_str)
394             if match:
395                 qualifier_str = match.group(1)
396                 qualifiers = []
397                 for candidate in re.split('[;,]', qualifier_str):
398                     candidate = candidate.strip()
399                     if candidate in FN_QUALIFIERS:
400                         qualifiers.append(candidate)
401                     elif candidate.startswith('z '):
402                         subcandidate = candidate.split()[1]
403                         if subcandidate in FN_QUALIFIERS:
404                             qualifiers.append(subcandidate)
405             else:
406                 qualifiers = []
407
408             yield anchor, fn_type, qualifiers, text_str, html_str