src/librarian/html.py

   1 # -*- coding: utf-8 -*-
   2 #
   3 # This file is part of Librarian, licensed under GNU Affero GPLv3 or later.
   4 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
   5 #
   6 from __future__ import print_function, unicode_literals
   7
   8 import os
   9 import re
  10 import copy
  11
  12 from lxml import etree
  13 from librarian import XHTMLNS, ParseError, OutputFile
  14 from librarian import functions
  15 from PIL import Image
  16
  17 from lxml.etree import XMLSyntaxError, XSLTApplyError
  18 import six
  19
  20
  21 functions.reg_substitute_entities()
  22 functions.reg_person_name()
  23
  24 STYLESHEETS = {
  25     'legacy': 'xslt/book2html.xslt',
  26     'full': 'xslt/wl2html_full.xslt',
  27     'partial': 'xslt/wl2html_partial.xslt'
  28 }
  29
  30
  31 def get_stylesheet(name):
  32     return os.path.join(os.path.dirname(__file__), STYLESHEETS[name])
  33
  34
  35 def html_has_content(text):
  36     return etree.ETXPath(
  37         '//p|//{%(ns)s}p|//h1|//{%(ns)s}h1' % {'ns': str(XHTMLNS)}
  38     )(text)
  39
  40
  41 def transform_abstrakt(abstrakt_element):
  42     style_filename = get_stylesheet('legacy')
  43     style = etree.parse(style_filename)
  44     xml = etree.tostring(abstrakt_element, encoding='unicode')
  45     document = etree.parse(six.StringIO(
  46         xml.replace('abstrakt', 'dlugi_cytat')
  47     ))  # HACK
  48     result = document.xslt(style)
  49     html = re.sub('<a name="sec[0-9]*"/>', '',
  50                   etree.tostring(result, encoding='unicode'))
  51     return re.sub('</?blockquote[^>]*>', '', html)
  52
  53
  54 def add_image_sizes(tree, gallery_path, gallery_url, base_url):
  55     widths = [360, 600, 1200, 1800, 2400]
  56
  57     for i, ilustr in enumerate(tree.findall('//ilustr')):
  58         rel_path = ilustr.attrib['src']
  59         img_url = six.moves.urllib.parse.urljoin(base_url, rel_path)
  60
  61         f = six.moves.urllib.request.urlopen(img_url)
  62         img = Image.open(f)
  63         ext = {'GIF': 'gif', 'PNG': 'png'}.get(img.format, 'jpg')
  64
  65         srcset = []
  66         # Needed widths: predefined and original, limited by
  67         # whichever is smaller.
  68         img_widths = [
  69             w for w in
  70             sorted(
  71                 set(widths + [img.size[0]])
  72             )
  73             if w <= min(widths[-1], img.size[0])
  74         ]
  75         largest = None
  76         for w in widths:
  77             fname = '%d.W%d.%s' % (i, w, ext)
  78             fpath = gallery_path + fname
  79             if not os.path.exists(fpath):
  80                 height = round(img.size[1] * w / img.size[0])
  81                 th = img.resize((w, height))
  82                 th.save(fpath)
  83             th_url = gallery_url + fname
  84             srcset.append(" ".join((
  85                 th_url,
  86                 '%dw' % w
  87             )))
  88             largest_url = th_url
  89         ilustr.attrib['srcset'] = ", ".join(srcset)
  90         ilustr.attrib['src'] = largest_url
  91
  92         f.close()
  93
  94
  95 def transform(wldoc, stylesheet='legacy', options=None, flags=None, css=None, gallery_path='img/', gallery_url='img/', base_url='file://./'):
  96     """Transforms the WL document to XHTML.
  97
  98     If output_filename is None, returns an XML,
  99     otherwise returns True if file has been written,False if it hasn't.
 100     File won't be written if it has no content.
 101     """
 102     # Parse XSLT
 103     try:
 104         style_filename = get_stylesheet(stylesheet)
 105         style = etree.parse(style_filename)
 106
 107         document = copy.deepcopy(wldoc)
 108         del wldoc
 109         document.swap_endlines()
 110
 111         if flags:
 112             for flag in flags:
 113                 document.edoc.getroot().set(flag, 'yes')
 114
 115         document.clean_ed_note()
 116         document.clean_ed_note('abstrakt')
 117         document.fix_pa_akap()
 118
 119         if not options:
 120             options = {}
 121
 122         try:
 123             os.makedirs(gallery_path)
 124         except OSError:
 125             pass
 126
 127         add_image_sizes(document.edoc, gallery_path, gallery_url, base_url)
 128
 129         css = (
 130             css
 131             or 'https://static.wolnelektury.pl/css/compressed/book_text.css'
 132         )
 133         css = "'%s'" % css
 134         result = document.transform(style, css=css, **options)
 135         del document  # no longer needed large object :)
 136
 137         if html_has_content(result):
 138             add_anchors(result.getroot())
 139             add_table_of_themes(result.getroot())
 140             add_table_of_contents(result.getroot())
 141
 142             return OutputFile.from_bytes(etree.tostring(
 143                 result, method='html', xml_declaration=False,
 144                 pretty_print=True, encoding='utf-8'
 145             ))
 146         else:
 147             return None
 148     except KeyError:
 149         raise ValueError("'%s' is not a valid stylesheet.")
 150     except (XMLSyntaxError, XSLTApplyError) as e:
 151         raise ParseError(e)
 152
 153
 154 @six.python_2_unicode_compatible
 155 class Fragment(object):
 156     def __init__(self, id, themes):
 157         super(Fragment, self).__init__()
 158         self.id = id
 159         self.themes = themes
 160         self.events = []
 161
 162     def append(self, event, element):
 163         self.events.append((event, element))
 164
 165     def closed_events(self):
 166         stack = []
 167         for event, element in self.events:
 168             if event == 'start':
 169                 stack.append(('end', element))
 170             elif event == 'end':
 171                 try:
 172                     stack.pop()
 173                 except IndexError:
 174                     print('CLOSED NON-OPEN TAG:', element)
 175
 176         stack.reverse()
 177         return self.events + stack
 178
 179     def to_string(self):
 180         result = []
 181         for event, element in self.closed_events():
 182             if event == 'start':
 183                 result.append(u'<%s %s>' % (
 184                     element.tag,
 185                     ' '.join(
 186                         '%s="%s"' % (k, v)
 187                         for k, v in element.attrib.items()
 188                     )
 189                 ))
 190                 if element.text:
 191                     result.append(element.text)
 192             elif event == 'end':
 193                 result.append(u'</%s>' % element.tag)
 194                 if element.tail:
 195                     result.append(element.tail)
 196             else:
 197                 result.append(element)
 198
 199         return ''.join(result)
 200
 201     def __str__(self):
 202         return self.to_string()
 203
 204
 205 def extract_fragments(input_filename):
 206     """Extracts theme fragments from input_filename."""
 207     open_fragments = {}
 208     closed_fragments = {}
 209
 210     # iterparse would die on a HTML document
 211     parser = etree.HTMLParser(encoding='utf-8')
 212     buf = six.BytesIO()
 213     buf.write(etree.tostring(
 214         etree.parse(input_filename, parser).getroot()[0][0],
 215         encoding='utf-8'
 216     ))
 217     buf.seek(0)
 218
 219     for event, element in etree.iterparse(buf, events=('start', 'end')):
 220         # Process begin and end elements
 221         if element.get('class', '') in ('theme-begin', 'theme-end'):
 222             if not event == 'end':
 223                 continue  # Process elements only once, on end event
 224
 225             # Open new fragment
 226             if element.get('class', '') == 'theme-begin':
 227                 fragment = Fragment(id=element.get('fid'), themes=element.text)
 228
 229                 # Append parents
 230                 parent = element.getparent()
 231                 parents = []
 232                 while parent.get('id', None) != 'book-text':
 233                     cparent = copy.deepcopy(parent)
 234                     cparent.text = None
 235                     if 'id' in cparent.attrib:
 236                         del cparent.attrib['id']
 237                     parents.append(cparent)
 238                     parent = parent.getparent()
 239
 240                 parents.reverse()
 241                 for parent in parents:
 242                     fragment.append('start', parent)
 243
 244                 open_fragments[fragment.id] = fragment
 245
 246             # Close existing fragment
 247             else:
 248                 try:
 249                     fragment = open_fragments[element.get('fid')]
 250                 except KeyError:
 251                     print('%s:closed not open fragment #%s' % (
 252                         input_filename, element.get('fid')
 253                     ))
 254                 else:
 255                     closed_fragments[fragment.id] = fragment
 256                     del open_fragments[fragment.id]
 257
 258             # Append element tail to lost_text
 259             # (we don't want to lose any text)
 260             if element.tail:
 261                 for fragment_id in open_fragments:
 262                     open_fragments[fragment_id].append('text', element.tail)
 263
 264         # Process all elements except begin and end
 265         else:
 266             # Omit annotation tags
 267             if (len(element.get('name', '')) or
 268                     element.get('class', '') in ('annotation', 'anchor')):
 269                 if event == 'end' and element.tail:
 270                     for fragment_id in open_fragments:
 271                         open_fragments[fragment_id].append(
 272                             'text', element.tail
 273                         )
 274             else:
 275                 for fragment_id in open_fragments:
 276                     celem = copy.copy(element)
 277                     if 'id' in celem.attrib:
 278                         del celem.attrib['id']
 279                     open_fragments[fragment_id].append(
 280                         event, celem
 281                     )
 282
 283     return closed_fragments, open_fragments
 284
 285
 286 def add_anchor(element, prefix, with_link=True, with_target=True,
 287                link_text=None):
 288     parent = element.getparent()
 289     index = parent.index(element)
 290
 291     if with_link:
 292         if link_text is None:
 293             link_text = prefix
 294         anchor = etree.Element('a', href='#%s' % prefix)
 295         anchor.set('class', 'anchor')
 296         anchor.text = six.text_type(link_text)
 297         parent.insert(index, anchor)
 298
 299     if with_target:
 300         anchor_target = etree.Element('a', name='%s' % prefix)
 301         anchor_target.set('class', 'target')
 302         anchor_target.text = u' '
 303         parent.insert(index, anchor_target)
 304
 305
 306 def any_ancestor(element, test):
 307     for ancestor in element.iterancestors():
 308         if test(ancestor):
 309             return True
 310     return False
 311
 312
 313 def add_anchors(root):
 314     counter = 1
 315     for element in root.iterdescendants():
 316         def f(e):
 317             return (
 318                 e.get('class') in (
 319                     'note', 'motto', 'motto_podpis', 'dedication', 'frame'
 320                 )
 321                 or e.get('id') == 'nota_red'
 322                 or e.tag == 'blockquote'
 323                 or e.get('id') == 'footnotes'
 324             )
 325         if any_ancestor(element, f):
 326             continue
 327
 328         if element.tag == 'div' and 'verse' in element.get('class', ''):
 329             if counter == 1 or counter % 5 == 0:
 330                 add_anchor(element, "f%d" % counter, link_text=counter)
 331             counter += 1
 332         elif 'paragraph' in element.get('class', ''):
 333             add_anchor(element, "f%d" % counter, link_text=counter)
 334             counter += 1
 335
 336
 337 def raw_printable_text(element):
 338     working = copy.deepcopy(element)
 339     for e in working.findall('a'):
 340         if e.get('class') in ('annotation', 'theme-begin'):
 341             e.text = ''
 342     return etree.tostring(working, method='text', encoding='unicode').strip()
 343
 344
 345 def add_table_of_contents(root):
 346     sections = []
 347     counter = 1
 348     for element in root.iterdescendants():
 349         if element.tag in ('h2', 'h3'):
 350             if any_ancestor(
 351                     element,
 352                     lambda e: e.get('id') in (
 353                         'footnotes', 'nota_red'
 354                     ) or e.get('class') in ('person-list',)):
 355                 continue
 356
 357             element_text = raw_printable_text(element)
 358             if (element.tag == 'h3' and len(sections)
 359                     and sections[-1][1] == 'h2'):
 360                 sections[-1][3].append(
 361                     (counter, element.tag, element_text, [])
 362                 )
 363             else:
 364                 sections.append((counter, element.tag, element_text, []))
 365             add_anchor(element, "s%d" % counter, with_link=False)
 366             counter += 1
 367
 368     toc = etree.Element('div')
 369     toc.set('id', 'toc')
 370     toc_header = etree.SubElement(toc, 'h2')
 371     toc_header.text = u'Spis treści'
 372     toc_list = etree.SubElement(toc, 'ol')
 373
 374     for n, section, text, subsections in sections:
 375         section_element = etree.SubElement(toc_list, 'li')
 376         add_anchor(section_element, "s%d" % n, with_target=False,
 377                    link_text=text)
 378
 379         if len(subsections):
 380             subsection_list = etree.SubElement(section_element, 'ol')
 381             for n1, subsection, subtext, _ in subsections:
 382                 subsection_element = etree.SubElement(subsection_list, 'li')
 383                 add_anchor(subsection_element, "s%d" % n1, with_target=False,
 384                            link_text=subtext)
 385
 386     root.insert(0, toc)
 387
 388
 389 def add_table_of_themes(root):
 390     try:
 391         from sortify import sortify
 392     except ImportError:
 393         def sortify(x):
 394             return x
 395
 396     book_themes = {}
 397     for fragment in root.findall('.//a[@class="theme-begin"]'):
 398         if not fragment.text:
 399             continue
 400         theme_names = [s.strip() for s in fragment.text.split(',')]
 401         for theme_name in theme_names:
 402             book_themes.setdefault(theme_name, []).append(fragment.get('name'))
 403     book_themes = list(book_themes.items())
 404     book_themes.sort(key=lambda s: sortify(s[0]))
 405     themes_div = etree.Element('div', id="themes")
 406     themes_ol = etree.SubElement(themes_div, 'ol')
 407     for theme_name, fragments in book_themes:
 408         themes_li = etree.SubElement(themes_ol, 'li')
 409         themes_li.text = "%s: " % theme_name
 410         for i, fragment in enumerate(fragments):
 411             item = etree.SubElement(themes_li, 'a', href="#%s" % fragment)
 412             item.text = str(i + 1)
 413             item.tail = ' '
 414     root.insert(0, themes_div)
 415
 416
 417 def extract_annotations(html_path):
 418     """Extracts annotations from HTML for annotations dictionary.
 419
 420     For each annotation, yields a tuple of:
 421     anchor, footnote type, valid qualifiers, text, html.
 422
 423     """
 424     from .fn_qualifiers import FN_QUALIFIERS
 425
 426     parser = etree.HTMLParser(encoding='utf-8')
 427     tree = etree.parse(html_path, parser)
 428     footnotes = tree.find('//*[@id="footnotes"]')
 429     re_qualifier = re.compile(r'[^\u2014]+\s+\(([^\)]+)\)\s+\u2014')
 430     if footnotes is not None:
 431         for footnote in footnotes.findall('div'):
 432             fn_type = footnote.get('class').split('-')[1]
 433             anchor = footnote.find('a[@class="annotation"]').get('href')[1:]
 434             del footnote[:2]
 435             footnote.text = None
 436             if len(footnote) and footnote[-1].tail == '\n':
 437                 footnote[-1].tail = None
 438             text_str = etree.tostring(footnote, method='text',
 439                                       encoding='unicode').strip()
 440             html_str = etree.tostring(footnote, method='html',
 441                                       encoding='unicode').strip()
 442
 443             match = re_qualifier.match(text_str)
 444             if match:
 445                 qualifier_str = match.group(1)
 446                 qualifiers = []
 447                 for candidate in re.split('[;,]', qualifier_str):
 448                     candidate = candidate.strip()
 449                     if candidate in FN_QUALIFIERS:
 450                         qualifiers.append(candidate)
 451                     elif candidate.startswith('z '):
 452                         subcandidate = candidate.split()[1]
 453                         if subcandidate in FN_QUALIFIERS:
 454                             qualifiers.append(subcandidate)
 455             else:
 456                 qualifiers = []
 457
 458             yield anchor, fn_type, qualifiers, text_str, html_str