src/librarian/html.py

   1 # -*- coding: utf-8 -*-
   2 #
   3 # This file is part of Librarian, licensed under GNU Affero GPLv3 or later.
   4 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
   5 #
   6 from __future__ import print_function, unicode_literals
   7
   8 import os
   9 import re
  10 import copy
  11
  12 from lxml import etree
  13 from librarian import XHTMLNS, ParseError, OutputFile
  14 from librarian import functions
  15 from PIL import Image
  16
  17 from lxml.etree import XMLSyntaxError, XSLTApplyError
  18 import six
  19
  20
  21 functions.reg_substitute_entities()
  22 functions.reg_person_name()
  23
  24 STYLESHEETS = {
  25     'legacy': 'xslt/book2html.xslt',
  26     'full': 'xslt/wl2html_full.xslt',
  27     'partial': 'xslt/wl2html_partial.xslt'
  28 }
  29
  30
  31 def get_stylesheet(name):
  32     return os.path.join(os.path.dirname(__file__), STYLESHEETS[name])
  33
  34
  35 def html_has_content(text):
  36     return etree.ETXPath(
  37         '//p|//{%(ns)s}p|//h1|//{%(ns)s}h1' % {'ns': str(XHTMLNS)}
  38     )(text)
  39
  40
  41 def transform_abstrakt(abstrakt_element):
  42     style_filename = get_stylesheet('legacy')
  43     style = etree.parse(style_filename)
  44     xml = etree.tostring(abstrakt_element, encoding='unicode')
  45     document = etree.parse(six.StringIO(
  46         xml.replace('abstrakt', 'dlugi_cytat')
  47     ))  # HACK
  48     result = document.xslt(style)
  49     html = re.sub('<a name="sec[0-9]*"/>', '',
  50                   etree.tostring(result, encoding='unicode'))
  51     return re.sub('</?blockquote[^>]*>', '', html)
  52
  53
  54 def add_image_sizes(tree, gallery_path, gallery_url, base_url):
  55     widths = [360, 600, 1200, 1800, 2400]
  56
  57     for i, ilustr in enumerate(tree.findall('//ilustr')):
  58         rel_path = ilustr.attrib['src']
  59         img_url = six.moves.urllib.parse.urljoin(base_url, rel_path)
  60
  61         f = six.moves.urllib.request.urlopen(img_url)
  62         img = Image.open(f)
  63         ext = {'GIF': 'gif', 'PNG': 'png'}.get(img.format, 'jpg')
  64
  65         srcset = []
  66         # Needed widths: predefined and original, limited by
  67         # whichever is smaller.
  68         img_widths = [
  69             w for w in
  70             sorted(
  71                 set(widths + [img.size[0]])
  72             )
  73             if w <= min(widths[-1], img.size[0])
  74         ]
  75         largest = None
  76         for w in widths:
  77             fname = '%d.W%d.%s' % (i, w, ext)
  78             fpath = gallery_path + fname
  79             if not os.path.exists(fpath):
  80                 height = round(img.size[1] * w / img.size[0])
  81                 th = img.resize((w, height))
  82                 th.save(fpath)
  83             th_url = gallery_url + fname
  84             srcset.append(" ".join((
  85                 th_url,
  86                 '%dw' % w
  87             )))
  88             largest_url = th_url
  89         ilustr.attrib['srcset'] = ", ".join(srcset)
  90         ilustr.attrib['src'] = largest_url
  91
  92         f.close()
  93
  94
  95 def transform(wldoc, stylesheet='legacy', options=None, flags=None, css=None, gallery_path='img/', gallery_url='img/', base_url='file://./'):
  96     """Transforms the WL document to XHTML.
  97
  98     If output_filename is None, returns an XML,
  99     otherwise returns True if file has been written,False if it hasn't.
 100     File won't be written if it has no content.
 101     """
 102     # Parse XSLT
 103     try:
 104         style_filename = get_stylesheet(stylesheet)
 105         style = etree.parse(style_filename)
 106
 107         document = copy.deepcopy(wldoc)
 108         del wldoc
 109         document.swap_endlines()
 110
 111         if flags:
 112             for flag in flags:
 113                 document.edoc.getroot().set(flag, 'yes')
 114
 115         document.clean_ed_note()
 116         document.clean_ed_note('abstrakt')
 117         document.fix_pa_akap()
 118
 119         if not options:
 120             options = {}
 121
 122         try:
 123             os.makedirs(gallery_path)
 124         except OSError:
 125             pass
 126
 127         add_image_sizes(document.edoc, gallery_path, gallery_url, base_url)
 128
 129         css = (
 130             css
 131             or 'https://static.wolnelektury.pl/css/compressed/book_text.css'
 132         )
 133         css = "'%s'" % css
 134         result = document.transform(style, css=css, **options)
 135         del document  # no longer needed large object :)
 136
 137         if html_has_content(result):
 138             add_anchors(result.getroot())
 139             add_table_of_themes(result.getroot())
 140             add_table_of_contents(result.getroot())
 141
 142             return OutputFile.from_bytes(etree.tostring(
 143                 result, method='html', xml_declaration=False,
 144                 pretty_print=True, encoding='utf-8'
 145             ))
 146         else:
 147             return None
 148     except KeyError:
 149         raise ValueError("'%s' is not a valid stylesheet.")
 150     except (XMLSyntaxError, XSLTApplyError) as e:
 151         raise ParseError(e)
 152
 153
 154 @six.python_2_unicode_compatible
 155 class Fragment(object):
 156     def __init__(self, id, themes):
 157         super(Fragment, self).__init__()
 158         self.id = id
 159         self.themes = themes
 160         self.events = []
 161
 162     def append(self, event, element):
 163         self.events.append((event, element))
 164
 165     def closed_events(self):
 166         stack = []
 167         for event, element in self.events:
 168             if event == 'start':
 169                 stack.append(('end', element))
 170             elif event == 'end':
 171                 try:
 172                     stack.pop()
 173                 except IndexError:
 174                     print('CLOSED NON-OPEN TAG:', element)
 175
 176         stack.reverse()
 177         return self.events + stack
 178
 179     def to_string(self):
 180         result = []
 181         for event, element in self.closed_events():
 182             if event == 'start':
 183                 result.append(u'<%s %s>' % (
 184                     element.tag,
 185                     ' '.join(
 186                         '%s="%s"' % (k, v)
 187                         for k, v in element.attrib.items()
 188                     )
 189                 ))
 190                 if element.text:
 191                     result.append(element.text)
 192             elif event == 'end':
 193                 result.append(u'</%s>' % element.tag)
 194                 if element.tail:
 195                     result.append(element.tail)
 196             else:
 197                 result.append(element)
 198
 199         return ''.join(result)
 200
 201     def __str__(self):
 202         return self.to_string()
 203
 204
 205 def extract_fragments(input_filename):
 206     """Extracts theme fragments from input_filename."""
 207     open_fragments = {}
 208     closed_fragments = {}
 209
 210     # iterparse would die on a HTML document
 211     parser = etree.HTMLParser(encoding='utf-8')
 212     buf = six.BytesIO()
 213     buf.write(etree.tostring(
 214         etree.parse(input_filename, parser).getroot()[0][0],
 215         encoding='utf-8'
 216     ))
 217     buf.seek(0)
 218
 219     for event, element in etree.iterparse(buf, events=('start', 'end')):
 220         # Process begin and end elements
 221         if element.get('class', '') in ('theme-begin', 'theme-end'):
 222             if not event == 'end':
 223                 continue  # Process elements only once, on end event
 224
 225             # Open new fragment
 226             if element.get('class', '') == 'theme-begin':
 227                 fragment = Fragment(id=element.get('fid'), themes=element.text)
 228
 229                 # Append parents
 230                 parent = element.getparent()
 231                 parents = []
 232                 while parent.get('id', None) != 'book-text':
 233                     cparent = copy.deepcopy(parent)
 234                     cparent.text = None
 235                     if 'id' in cparent.attrib:
 236                         del cparent.attrib['id']
 237                     parents.append(cparent)
 238                     parent = parent.getparent()
 239
 240                 parents.reverse()
 241                 for parent in parents:
 242                     fragment.append('start', parent)
 243
 244                 open_fragments[fragment.id] = fragment
 245
 246             # Close existing fragment
 247             else:
 248                 try:
 249                     fragment = open_fragments[element.get('fid')]
 250                 except KeyError:
 251                     print('%s:closed not open fragment #%s' % (
 252                         input_filename, element.get('fid')
 253                     ))
 254                 else:
 255                     closed_fragments[fragment.id] = fragment
 256                     del open_fragments[fragment.id]
 257
 258             # Append element tail to lost_text
 259             # (we don't want to lose any text)
 260             if element.tail:
 261                 for fragment_id in open_fragments:
 262                     open_fragments[fragment_id].append('text', element.tail)
 263
 264         # Process all elements except begin and end
 265         else:
 266             # Omit annotation tags
 267             if (len(element.get('name', '')) or
 268                     element.get('class', '') in ('annotation', 'anchor')):
 269                 if event == 'end' and element.tail:
 270                     for fragment_id in open_fragments:
 271                         open_fragments[fragment_id].append(
 272                             'text', element.tail
 273                         )
 274             else:
 275                 for fragment_id in open_fragments:
 276                     celem = copy.copy(element)
 277                     if 'id' in celem.attrib:
 278                         del celem.attrib['id']
 279                     open_fragments[fragment_id].append(
 280                         event, celem
 281                     )
 282
 283     return closed_fragments, open_fragments
 284
 285
 286 def add_anchor(element, prefix, with_link=True, with_target=True,
 287                link_text=None):
 288     parent = element.getparent()
 289     index = parent.index(element)
 290
 291     if with_link:
 292         if link_text is None:
 293             link_text = prefix
 294         anchor = etree.Element('a', href='#%s' % prefix)
 295         anchor.set('class', 'anchor')
 296         anchor.text = six.text_type(link_text)
 297         parent.insert(index, anchor)
 298
 299     if with_target:
 300         anchor_target = etree.Element('a', name='%s' % prefix)
 301         anchor_target.set('class', 'target')
 302         anchor_target.text = u' '
 303         parent.insert(index, anchor_target)
 304
 305
 306 def any_ancestor(element, test):
 307     for ancestor in element.iterancestors():
 308         if test(ancestor):
 309             return True
 310     return False
 311
 312
 313 def add_anchors(root):
 314     counter = 1
 315     visible_counter = 1
 316     for element in root.iterdescendants():
 317         def f(e):
 318             return (
 319                 e.get('class') in (
 320                     'note', 'motto', 'motto_podpis', 'dedication', 'frame'
 321                 )
 322                 or e.get('id') == 'nota_red'
 323                 or e.tag == 'blockquote'
 324                 or e.get('id') == 'footnotes'
 325             )
 326
 327         if element.get('class') == 'numeracja':
 328             try:
 329                 visible_counter = int(element.get('data-start'))
 330             except ValueError:
 331                 visible_counter = 1
 332
 333         if any_ancestor(element, f):
 334             continue
 335
 336         if element.tag == 'div' and 'verse' in element.get('class', ''):
 337             if visible_counter == 1 or visible_counter % 5 == 0:
 338                 add_anchor(element, "f%d" % counter, link_text=visible_counter)
 339             counter += 1
 340             visible_counter += 1
 341         elif 'paragraph' in element.get('class', ''):
 342             add_anchor(element, "f%d" % counter, link_text=visible_counter)
 343             counter += 1
 344             visible_counter += 1
 345
 346
 347 def raw_printable_text(element):
 348     working = copy.deepcopy(element)
 349     for e in working.findall('a'):
 350         if e.get('class') in ('annotation', 'theme-begin'):
 351             e.text = ''
 352     return etree.tostring(working, method='text', encoding='unicode').strip()
 353
 354
 355 def add_table_of_contents(root):
 356     sections = []
 357     counter = 1
 358     for element in root.iterdescendants():
 359         if element.tag in ('h2', 'h3'):
 360             if any_ancestor(
 361                     element,
 362                     lambda e: e.get('id') in (
 363                         'footnotes', 'nota_red'
 364                     ) or e.get('class') in ('person-list',)):
 365                 continue
 366
 367             element_text = raw_printable_text(element)
 368             if (element.tag == 'h3' and len(sections)
 369                     and sections[-1][1] == 'h2'):
 370                 sections[-1][3].append(
 371                     (counter, element.tag, element_text, [])
 372                 )
 373             else:
 374                 sections.append((counter, element.tag, element_text, []))
 375             add_anchor(element, "s%d" % counter, with_link=False)
 376             counter += 1
 377
 378     toc = etree.Element('div')
 379     toc.set('id', 'toc')
 380     toc_header = etree.SubElement(toc, 'h2')
 381     toc_header.text = u'Spis treści'
 382     toc_list = etree.SubElement(toc, 'ol')
 383
 384     for n, section, text, subsections in sections:
 385         section_element = etree.SubElement(toc_list, 'li')
 386         add_anchor(section_element, "s%d" % n, with_target=False,
 387                    link_text=text)
 388
 389         if len(subsections):
 390             subsection_list = etree.SubElement(section_element, 'ol')
 391             for n1, subsection, subtext, _ in subsections:
 392                 subsection_element = etree.SubElement(subsection_list, 'li')
 393                 add_anchor(subsection_element, "s%d" % n1, with_target=False,
 394                            link_text=subtext)
 395
 396     root.insert(0, toc)
 397
 398
 399 def add_table_of_themes(root):
 400     try:
 401         from sortify import sortify
 402     except ImportError:
 403         def sortify(x):
 404             return x
 405
 406     book_themes = {}
 407     for fragment in root.findall('.//a[@class="theme-begin"]'):
 408         if not fragment.text:
 409             continue
 410         theme_names = [s.strip() for s in fragment.text.split(',')]
 411         for theme_name in theme_names:
 412             book_themes.setdefault(theme_name, []).append(fragment.get('name'))
 413     book_themes = list(book_themes.items())
 414     book_themes.sort(key=lambda s: sortify(s[0]))
 415     themes_div = etree.Element('div', id="themes")
 416     themes_ol = etree.SubElement(themes_div, 'ol')
 417     for theme_name, fragments in book_themes:
 418         themes_li = etree.SubElement(themes_ol, 'li')
 419         themes_li.text = "%s: " % theme_name
 420         for i, fragment in enumerate(fragments):
 421             item = etree.SubElement(themes_li, 'a', href="#%s" % fragment)
 422             item.text = str(i + 1)
 423             item.tail = ' '
 424     root.insert(0, themes_div)
 425
 426
 427 def extract_annotations(html_path):
 428     """Extracts annotations from HTML for annotations dictionary.
 429
 430     For each annotation, yields a tuple of:
 431     anchor, footnote type, valid qualifiers, text, html.
 432
 433     """
 434     from .fn_qualifiers import FN_QUALIFIERS
 435
 436     parser = etree.HTMLParser(encoding='utf-8')
 437     tree = etree.parse(html_path, parser)
 438     footnotes = tree.find('//*[@id="footnotes"]')
 439     re_qualifier = re.compile(r'[^\u2014]+\s+\(([^\)]+)\)\s+\u2014')
 440     if footnotes is not None:
 441         for footnote in footnotes.findall('div'):
 442             fn_type = footnote.get('class').split('-')[1]
 443             anchor = footnote.find('a[@class="annotation"]').get('href')[1:]
 444             del footnote[:2]
 445             footnote.text = None
 446             if len(footnote) and footnote[-1].tail == '\n':
 447                 footnote[-1].tail = None
 448             text_str = etree.tostring(footnote, method='text',
 449                                       encoding='unicode').strip()
 450             html_str = etree.tostring(footnote, method='html',
 451                                       encoding='unicode').strip()
 452
 453             match = re_qualifier.match(text_str)
 454             if match:
 455                 qualifier_str = match.group(1)
 456                 qualifiers = []
 457                 for candidate in re.split('[;,]', qualifier_str):
 458                     candidate = candidate.strip()
 459                     if candidate in FN_QUALIFIERS:
 460                         qualifiers.append(candidate)
 461                     elif candidate.startswith('z '):
 462                         subcandidate = candidate.split()[1]
 463                         if subcandidate in FN_QUALIFIERS:
 464                             qualifiers.append(subcandidate)
 465             else:
 466                 qualifiers = []
 467
 468             yield anchor, fn_type, qualifiers, text_str, html_str