X-Git-Url: https://git.mdrn.pl/librarian.git/blobdiff_plain/fefdce4e24f9e397df5538fe6e7f54b5ece4d841..011b98a4661b754a0789ae78e145437b5b86e5fe:/src/librarian/html.py diff --git a/src/librarian/html.py b/src/librarian/html.py index f8e4a83..a96e975 100644 --- a/src/librarian/html.py +++ b/src/librarian/html.py @@ -1,99 +1,64 @@ -# -*- coding: utf-8 -*- -# # This file is part of Librarian, licensed under GNU Affero GPLv3 or later. -# Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information. +# Copyright © Fundacja Wolne Lektury. See NOTICE for more information. # -from __future__ import print_function, unicode_literals - +import io import os import re import copy +import urllib.parse +import urllib.request from lxml import etree -from librarian import XHTMLNS, ParseError, OutputFile from librarian import functions +from PIL import Image -from lxml.etree import XMLSyntaxError, XSLTApplyError -import six - - -functions.reg_substitute_entities() -functions.reg_person_name() - -STYLESHEETS = { - 'legacy': 'xslt/book2html.xslt', - 'full': 'xslt/wl2html_full.xslt', - 'partial': 'xslt/wl2html_partial.xslt' -} - - -def get_stylesheet(name): - return os.path.join(os.path.dirname(__file__), STYLESHEETS[name]) - - -def html_has_content(text): - return etree.ETXPath('//p|//{%(ns)s}p|//h1|//{%(ns)s}h1' % {'ns': str(XHTMLNS)})(text) - -def transform_abstrakt(abstrakt_element): - style_filename = get_stylesheet('legacy') - style = etree.parse(style_filename) - xml = etree.tostring(abstrakt_element, encoding='unicode') - document = etree.parse(six.StringIO(xml.replace('abstrakt', 'dlugi_cytat'))) # HACK - result = document.xslt(style) - html = re.sub('', '', etree.tostring(result, encoding='unicode')) - return re.sub(']*>', '', html) - - -def transform(wldoc, stylesheet='legacy', options=None, flags=None, css=None): - """Transforms the WL document to XHTML. - - If output_filename is None, returns an XML, - otherwise returns True if file has been written,False if it hasn't. - File won't be written if it has no content. - """ - # Parse XSLT +def add_image_sizes(tree, gallery_path, gallery_url, base_url): + widths = [360, 600, 1200, 1800, 2400] try: - style_filename = get_stylesheet(stylesheet) - style = etree.parse(style_filename) - - document = copy.deepcopy(wldoc) - del wldoc - document.swap_endlines() - - if flags: - for flag in flags: - document.edoc.getroot().set(flag, 'yes') - - document.clean_ed_note() - document.clean_ed_note('abstrakt') - - if not options: - options = {} - options.setdefault('gallery', "''") - - css = css or 'https://static.wolnelektury.pl/css/compressed/book_text.css' - css = "'%s'" % css - result = document.transform(style, css=css, **options) - del document # no longer needed large object :) - - if html_has_content(result): - add_anchors(result.getroot()) - add_table_of_themes(result.getroot()) - add_table_of_contents(result.getroot()) - - return OutputFile.from_bytes(etree.tostring( - result, method='html', xml_declaration=False, pretty_print=True, encoding='utf-8')) - else: - return None - except KeyError: - raise ValueError("'%s' is not a valid stylesheet.") - except (XMLSyntaxError, XSLTApplyError) as e: - raise ParseError(e) - - -@six.python_2_unicode_compatible -class Fragment(object): + os.makedirs(gallery_path) + except: + pass + + for i, ilustr in enumerate(tree.findall('//ilustr')): + rel_path = ilustr.attrib['src'] + img_url = urllib.parse.urljoin(base_url, rel_path) + + f = urllib.request.urlopen(img_url) + img = Image.open(f) + ext = {'GIF': 'gif', 'PNG': 'png'}.get(img.format, 'jpg') + + srcset = [] + # Needed widths: predefined and original, limited by + # whichever is smaller. + img_widths = [ + w for w in + sorted( + set(widths + [img.size[0]]) + ) + if w <= min(widths[-1], img.size[0]) + ] + largest = None + for w in widths: + fname = '%d.W%d.%s' % (i, w, ext) + fpath = gallery_path + fname + if not os.path.exists(fpath): + height = round(img.size[1] * w / img.size[0]) + th = img.resize((w, height)) + th.save(fpath) + th_url = gallery_url + fname + srcset.append(" ".join(( + th_url, + '%dw' % w + ))) + largest_url = th_url + ilustr.attrib['srcset'] = ", ".join(srcset) + ilustr.attrib['src'] = largest_url + + f.close() + + +class Fragment: def __init__(self, id, themes): super(Fragment, self).__init__() self.id = id @@ -121,12 +86,17 @@ class Fragment(object): result = [] for event, element in self.closed_events(): if event == 'start': - result.append(u'<%s %s>' % ( - element.tag, ' '.join('%s="%s"' % (k, v) for k, v in element.attrib.items()))) + result.append('<%s %s>' % ( + element.tag, + ' '.join( + '%s="%s"' % (k, v) + for k, v in element.attrib.items() + ) + )) if element.text: result.append(element.text) elif event == 'end': - result.append(u'' % element.tag) + result.append('' % element.tag) if element.tail: result.append(element.tail) else: @@ -145,8 +115,11 @@ def extract_fragments(input_filename): # iterparse would die on a HTML document parser = etree.HTMLParser(encoding='utf-8') - buf = six.BytesIO() - buf.write(etree.tostring(etree.parse(input_filename, parser).getroot()[0][0], encoding='utf-8')) + buf = io.BytesIO() + buf.write(etree.tostring( + etree.parse(input_filename, parser).getroot()[0][0], + encoding='utf-8' + )) buf.seek(0) for event, element in etree.iterparse(buf, events=('start', 'end')): @@ -165,6 +138,8 @@ def extract_fragments(input_filename): while parent.get('id', None) != 'book-text': cparent = copy.deepcopy(parent) cparent.text = None + if 'id' in cparent.attrib: + del cparent.attrib['id'] parents.append(cparent) parent = parent.getparent() @@ -172,19 +147,23 @@ def extract_fragments(input_filename): for parent in parents: fragment.append('start', parent) - open_fragments[fragment.id] = fragment + if fragment.id not in open_fragments: + open_fragments[fragment.id] = fragment # Close existing fragment else: try: fragment = open_fragments[element.get('fid')] except KeyError: - print('%s:closed not open fragment #%s' % (input_filename, element.get('fid'))) + print('%s:closed not open fragment #%s' % ( + input_filename, element.get('fid') + )) else: closed_fragments[fragment.id] = fragment del open_fragments[fragment.id] - # Append element tail to lost_text (we don't want to lose any text) + # Append element tail to lost_text + # (we don't want to lose any text) if element.tail: for fragment_id in open_fragments: open_fragments[fragment_id].append('text', element.tail) @@ -192,35 +171,35 @@ def extract_fragments(input_filename): # Process all elements except begin and end else: # Omit annotation tags - if (len(element.get('name', '')) or - element.get('class', '') in ('annotation', 'anchor')): + if (len(element.get('name', '')) or + element.get('class', '') in ('annotation-anchor', 'anchor', 'wl-num', 'reference')): if event == 'end' and element.tail: for fragment_id in open_fragments: - open_fragments[fragment_id].append('text', element.tail) + open_fragments[fragment_id].append( + 'text', element.tail + ) else: for fragment_id in open_fragments: - open_fragments[fragment_id].append(event, copy.copy(element)) + celem = copy.copy(element) + if 'id' in celem.attrib: + del celem.attrib['id'] + open_fragments[fragment_id].append( + event, celem + ) return closed_fragments, open_fragments -def add_anchor(element, prefix, with_link=True, with_target=True, link_text=None): +def add_anchor(element, prefix, link_text=None): parent = element.getparent() index = parent.index(element) - if with_link: - if link_text is None: - link_text = prefix - anchor = etree.Element('a', href='#%s' % prefix) - anchor.set('class', 'anchor') - anchor.text = six.text_type(link_text) - parent.insert(index, anchor) - - if with_target: - anchor_target = etree.Element('a', name='%s' % prefix) - anchor_target.set('class', 'target') - anchor_target.text = u' ' - parent.insert(index, anchor_target) + if link_text is None: + link_text = prefix + anchor = etree.Element('a', href='#%s' % prefix) + anchor.set('class', 'anchor') + anchor.text = str(link_text) + parent.insert(index, anchor) def any_ancestor(element, test): @@ -230,27 +209,9 @@ def any_ancestor(element, test): return False -def add_anchors(root): - counter = 1 - for element in root.iterdescendants(): - def f(e): - return e.get('class') in ('note', 'motto', 'motto_podpis', 'dedication', 'frame') or \ - e.get('id') == 'nota_red' or e.tag == 'blockquote' - if any_ancestor(element, f): - continue - - if element.tag == 'div' and 'verse' in element.get('class', ''): - if counter == 1 or counter % 5 == 0: - add_anchor(element, "f%d" % counter, link_text=counter) - counter += 1 - elif 'paragraph' in element.get('class', ''): - add_anchor(element, "f%d" % counter, link_text=counter) - counter += 1 - - def raw_printable_text(element): working = copy.deepcopy(element) - for e in working.findall('a'): + for e in working.findall('.//a'): if e.get('class') in ('annotation', 'theme-begin'): e.text = '' return etree.tostring(working, method='text', encoding='unicode').strip() @@ -258,40 +219,48 @@ def raw_printable_text(element): def add_table_of_contents(root): sections = [] - counter = 1 for element in root.iterdescendants(): if element.tag in ('h2', 'h3'): - if any_ancestor(element, - lambda e: e.get('id') in ('footnotes', 'nota_red') or e.get('class') in ('person-list',)): + if any_ancestor( + element, + lambda e: e.get('id') in ( + 'footnotes', 'nota_red' + ) or e.get('class') in ('person-list',)): continue element_text = raw_printable_text(element) - if element.tag == 'h3' and len(sections) and sections[-1][1] == 'h2': - sections[-1][3].append((counter, element.tag, element_text, [])) + if (element.tag == 'h3' and len(sections) + and sections[-1][1] == 'h2'): + sections[-1][3].append( + (element.attrib['id'], element.tag, element_text, []) + ) else: - sections.append((counter, element.tag, element_text, [])) - add_anchor(element, "s%d" % counter, with_link=False) - counter += 1 + sections.append((element.attrib['id'], element.tag, element_text, [])) + + if not sections: + return toc = etree.Element('div') toc.set('id', 'toc') toc_header = etree.SubElement(toc, 'h2') - toc_header.text = u'Spis treści' + toc_header.text = 'Spis treści' toc_list = etree.SubElement(toc, 'ol') for n, section, text, subsections in sections: section_element = etree.SubElement(toc_list, 'li') - add_anchor(section_element, "s%d" % n, with_target=False, link_text=text) + add_anchor(section_element, n, + link_text=text) if len(subsections): subsection_list = etree.SubElement(section_element, 'ol') for n1, subsection, subtext, _ in subsections: subsection_element = etree.SubElement(subsection_list, 'li') - add_anchor(subsection_element, "s%d" % n1, with_target=False, link_text=subtext) + add_anchor(subsection_element, n1, + link_text=subtext) root.insert(0, toc) - + def add_table_of_themes(root): try: from sortify import sortify @@ -317,7 +286,13 @@ def add_table_of_themes(root): item = etree.SubElement(themes_li, 'a', href="#%s" % fragment) item.text = str(i + 1) item.tail = ' ' + + if not len(themes_ol): + return + root.insert(0, themes_div) + themes_div.tail = root.text + root.text = None def extract_annotations(html_path): @@ -341,8 +316,10 @@ def extract_annotations(html_path): footnote.text = None if len(footnote) and footnote[-1].tail == '\n': footnote[-1].tail = None - text_str = etree.tostring(footnote, method='text', encoding='unicode').strip() - html_str = etree.tostring(footnote, method='html', encoding='unicode').strip() + text_str = etree.tostring(footnote, method='text', + encoding='unicode').strip() + html_str = etree.tostring(footnote, method='html', + encoding='unicode').strip() match = re_qualifier.match(text_str) if match: @@ -352,8 +329,8 @@ def extract_annotations(html_path): candidate = candidate.strip() if candidate in FN_QUALIFIERS: qualifiers.append(candidate) - elif candidate.startswith('z '): - subcandidate = candidate.split()[1] + elif candidate.startswith('z\u00A0'): + subcandidate = candidate.split('\u00A0')[1].split()[0] if subcandidate in FN_QUALIFIERS: qualifiers.append(subcandidate) else: