src/librarian/html.py

   1 # This file is part of Librarian, licensed under GNU Affero GPLv3 or later.
   2 # Copyright © Fundacja Wolne Lektury. See NOTICE for more information.
   3 #
   4 import io
   5 import os
   6 import re
   7 import copy
   8 import urllib.parse
   9 import urllib.request
  10
  11 from lxml import etree
  12 from librarian import functions
  13 from PIL import Image
  14
  15
  16 def add_image_sizes(tree, gallery_path, gallery_url, base_url):
  17     widths = [360, 600, 1200, 1800, 2400]
  18     try:
  19         os.makedirs(gallery_path)
  20     except:
  21         pass
  22
  23     for i, ilustr in enumerate(tree.findall('//ilustr')):
  24         rel_path = ilustr.attrib['src']
  25         img_url = urllib.parse.urljoin(base_url, rel_path)
  26
  27         f = urllib.request.urlopen(img_url)
  28         img = Image.open(f)
  29         ext = {'GIF': 'gif', 'PNG': 'png'}.get(img.format, 'jpg')
  30
  31         srcset = []
  32         # Needed widths: predefined and original, limited by
  33         # whichever is smaller.
  34         img_widths = [
  35             w for w in
  36             sorted(
  37                 set(widths + [img.size[0]])
  38             )
  39             if w <= min(widths[-1], img.size[0])
  40         ]
  41         largest = None
  42         for w in widths:
  43             fname = '%d.W%d.%s' % (i, w, ext)
  44             fpath = gallery_path + fname
  45             if not os.path.exists(fpath):
  46                 height = round(img.size[1] * w / img.size[0])
  47                 th = img.resize((w, height))
  48                 th.save(fpath)
  49             th_url = gallery_url + fname
  50             srcset.append(" ".join((
  51                 th_url,
  52                 '%dw' % w
  53             )))
  54             largest_url = th_url
  55         ilustr.attrib['srcset'] = ", ".join(srcset)
  56         ilustr.attrib['src'] = largest_url
  57
  58         f.close()
  59
  60
  61 class Fragment:
  62     def __init__(self, id, themes):
  63         super(Fragment, self).__init__()
  64         self.id = id
  65         self.themes = themes
  66         self.events = []
  67
  68     def append(self, event, element):
  69         self.events.append((event, element))
  70
  71     def closed_events(self):
  72         stack = []
  73         for event, element in self.events:
  74             if event == 'start':
  75                 stack.append(('end', element))
  76             elif event == 'end':
  77                 try:
  78                     stack.pop()
  79                 except IndexError:
  80                     print('CLOSED NON-OPEN TAG:', element)
  81
  82         stack.reverse()
  83         return self.events + stack
  84
  85     def to_string(self):
  86         result = []
  87         for event, element in self.closed_events():
  88             if event == 'start':
  89                 result.append('<%s %s>' % (
  90                     element.tag,
  91                     ' '.join(
  92                         '%s="%s"' % (k, v)
  93                         for k, v in element.attrib.items()
  94                     )
  95                 ))
  96                 if element.text:
  97                     result.append(element.text)
  98             elif event == 'end':
  99                 result.append('</%s>' % element.tag)
 100                 if element.tail:
 101                     result.append(element.tail)
 102             else:
 103                 result.append(element)
 104
 105         return ''.join(result)
 106
 107     def __str__(self):
 108         return self.to_string()
 109
 110
 111 def extract_fragments(input_filename):
 112     """Extracts theme fragments from input_filename."""
 113     open_fragments = {}
 114     closed_fragments = {}
 115
 116     # iterparse would die on a HTML document
 117     parser = etree.HTMLParser(encoding='utf-8')
 118     buf = io.BytesIO()
 119     buf.write(etree.tostring(
 120         etree.parse(input_filename, parser).getroot()[0][0],
 121         encoding='utf-8'
 122     ))
 123     buf.seek(0)
 124
 125     for event, element in etree.iterparse(buf, events=('start', 'end')):
 126         # Process begin and end elements
 127         if element.get('class', '') in ('theme-begin', 'theme-end'):
 128             if not event == 'end':
 129                 continue  # Process elements only once, on end event
 130
 131             # Open new fragment
 132             if element.get('class', '') == 'theme-begin':
 133                 fragment = Fragment(id=element.get('fid'), themes=element.text)
 134
 135                 # Append parents
 136                 parent = element.getparent()
 137                 parents = []
 138                 while parent.get('id', None) != 'book-text':
 139                     cparent = copy.deepcopy(parent)
 140                     cparent.text = None
 141                     if 'id' in cparent.attrib:
 142                         del cparent.attrib['id']
 143                     parents.append(cparent)
 144                     parent = parent.getparent()
 145
 146                 parents.reverse()
 147                 for parent in parents:
 148                     fragment.append('start', parent)
 149
 150                 if fragment.id not in open_fragments:
 151                     open_fragments[fragment.id] = fragment
 152
 153             # Close existing fragment
 154             else:
 155                 try:
 156                     fragment = open_fragments[element.get('fid')]
 157                 except KeyError:
 158                     print('%s:closed not open fragment #%s' % (
 159                         input_filename, element.get('fid')
 160                     ))
 161                 else:
 162                     closed_fragments[fragment.id] = fragment
 163                     del open_fragments[fragment.id]
 164
 165             # Append element tail to lost_text
 166             # (we don't want to lose any text)
 167             if element.tail:
 168                 for fragment_id in open_fragments:
 169                     open_fragments[fragment_id].append('text', element.tail)
 170
 171         # Process all elements except begin and end
 172         else:
 173             # Omit annotation tags
 174             if (len(element.get('name', '')) or
 175                     element.get('class', '') in ('annotation-anchor', 'anchor', 'wl-num', 'reference')):
 176                 if event == 'end' and element.tail:
 177                     for fragment_id in open_fragments:
 178                         open_fragments[fragment_id].append(
 179                             'text', element.tail
 180                         )
 181             else:
 182                 for fragment_id in open_fragments:
 183                     celem = copy.copy(element)
 184                     if 'id' in celem.attrib:
 185                         del celem.attrib['id']
 186                     open_fragments[fragment_id].append(
 187                         event, celem
 188                     )
 189
 190     return closed_fragments, open_fragments
 191
 192
 193 def add_anchor(element, prefix, link_text=None):
 194     parent = element.getparent()
 195     index = parent.index(element)
 196
 197     if link_text is None:
 198         link_text = prefix
 199     anchor = etree.Element('a', href='#%s' % prefix)
 200     anchor.set('class', 'anchor')
 201     anchor.text = str(link_text)
 202     parent.insert(index, anchor)
 203
 204
 205 def any_ancestor(element, test):
 206     for ancestor in element.iterancestors():
 207         if test(ancestor):
 208             return True
 209     return False
 210
 211
 212 def raw_printable_text(element):
 213     working = copy.deepcopy(element)
 214     for e in working.findall('.//a'):
 215         if e.get('class') in ('annotation', 'theme-begin'):
 216             e.text = ''
 217     return etree.tostring(working, method='text', encoding='unicode').strip()
 218
 219
 220 def add_table_of_contents(root):
 221     sections = []
 222     for element in root.iterdescendants():
 223         if element.tag in ('h2', 'h3'):
 224             if any_ancestor(
 225                     element,
 226                     lambda e: e.get('id') in (
 227                         'footnotes', 'nota_red'
 228                     ) or e.get('class') in ('person-list',)):
 229                 continue
 230
 231             element_text = raw_printable_text(element)
 232             if (element.tag == 'h3' and len(sections)
 233                     and sections[-1][1] == 'h2'):
 234                 sections[-1][3].append(
 235                     (element.attrib['id'], element.tag, element_text, [])
 236                 )
 237             else:
 238                 sections.append((element.attrib['id'], element.tag, element_text, []))
 239
 240     if not sections:
 241         return
 242
 243     toc = etree.Element('div')
 244     toc.set('id', 'toc')
 245     toc_header = etree.SubElement(toc, 'h2')
 246     toc_header.text = 'Spis treści'
 247     toc_list = etree.SubElement(toc, 'ol')
 248
 249     for n, section, text, subsections in sections:
 250         section_element = etree.SubElement(toc_list, 'li')
 251         add_anchor(section_element, n,
 252                    link_text=text)
 253
 254         if len(subsections):
 255             subsection_list = etree.SubElement(section_element, 'ol')
 256             for n1, subsection, subtext, _ in subsections:
 257                 subsection_element = etree.SubElement(subsection_list, 'li')
 258                 add_anchor(subsection_element, n1,
 259                            link_text=subtext)
 260
 261     root.insert(0, toc)
 262
 263
 264 def add_table_of_themes(root):
 265     try:
 266         from sortify import sortify
 267     except ImportError:
 268         def sortify(x):
 269             return x
 270
 271     book_themes = {}
 272     for fragment in root.findall('.//a[@class="theme-begin"]'):
 273         if not fragment.text:
 274             continue
 275         theme_names = [s.strip() for s in fragment.text.split(',')]
 276         for theme_name in theme_names:
 277             book_themes.setdefault(theme_name, []).append(fragment.get('name'))
 278     book_themes = list(book_themes.items())
 279     book_themes.sort(key=lambda s: sortify(s[0]))
 280     themes_div = etree.Element('div', id="themes")
 281     themes_ol = etree.SubElement(themes_div, 'ol')
 282     for theme_name, fragments in book_themes:
 283         themes_li = etree.SubElement(themes_ol, 'li')
 284         themes_li.text = "%s: " % theme_name
 285         for i, fragment in enumerate(fragments):
 286             item = etree.SubElement(themes_li, 'a', href="#%s" % fragment)
 287             item.text = str(i + 1)
 288             item.tail = ' '
 289
 290     if not len(themes_ol):
 291         return
 292
 293     root.insert(0, themes_div)
 294     themes_div.tail = root.text
 295     root.text = None
 296
 297
 298 def extract_annotations(html_path):
 299     """Extracts annotations from HTML for annotations dictionary.
 300
 301     For each annotation, yields a tuple of:
 302     anchor, footnote type, valid qualifiers, text, html.
 303
 304     """
 305     from .fn_qualifiers import FN_QUALIFIERS
 306
 307     parser = etree.HTMLParser(encoding='utf-8')
 308     tree = etree.parse(html_path, parser)
 309     footnotes = tree.find('//*[@id="footnotes"]')
 310     re_qualifier = re.compile(r'[^\u2014]+\s+\(([^\)]+)\)\s+\u2014')
 311     if footnotes is not None:
 312         for footnote in footnotes.findall('div'):
 313             fn_type = footnote.get('class').split('-')[1]
 314             anchor = footnote.find('a[@class="annotation"]').get('href')[1:]
 315             del footnote[:2]
 316             footnote.text = None
 317             if len(footnote) and footnote[-1].tail == '\n':
 318                 footnote[-1].tail = None
 319             text_str = etree.tostring(footnote, method='text',
 320                                       encoding='unicode').strip()
 321             html_str = etree.tostring(footnote, method='html',
 322                                       encoding='unicode').strip()
 323
 324             match = re_qualifier.match(text_str)
 325             if match:
 326                 qualifier_str = match.group(1)
 327                 qualifiers = []
 328                 for candidate in re.split('[;,]', qualifier_str):
 329                     candidate = candidate.strip()
 330                     if candidate in FN_QUALIFIERS:
 331                         qualifiers.append(candidate)
 332                     elif candidate.startswith('z\u00A0'):
 333                         subcandidate = candidate.split('\u00A0')[1].split()[0]
 334                         if subcandidate in FN_QUALIFIERS:
 335                             qualifiers.append(subcandidate)
 336             else:
 337                 qualifiers = []
 338
 339             yield anchor, fn_type, qualifiers, text_str, html_str