src/librarian/parser.py

   1 # This file is part of Librarian, licensed under GNU Affero GPLv3 or later.
   2 # Copyright © Fundacja Wolne Lektury. See NOTICE for more information.
   3 #
   4 from collections import Counter
   5 import io
   6 import os
   7 import re
   8 from xml.parsers.expat import ExpatError
   9 from lxml import etree
  10 from lxml.etree import XMLSyntaxError, XSLTApplyError
  11 from librarian import ValidationError, NoDublinCore,  ParseError, NoProvider
  12 from librarian import RDFNS
  13 from librarian.cover import make_cover
  14 from librarian import dcparser
  15 from .elements import WL_ELEMENTS
  16
  17
  18 class WLElementLookup(etree.CustomElementClassLookup):
  19     def lookup(self, node_type, document, namespace, name):
  20         if node_type != 'element':
  21             return
  22         if namespace:
  23             return
  24         try:
  25             return WL_ELEMENTS[name]
  26         except KeyError:
  27             return
  28
  29
  30 parser = etree.XMLParser()
  31 parser.set_element_class_lookup(
  32     WLElementLookup()
  33 )
  34
  35
  36
  37 class WLDocument:
  38     """Legacy class, to be replaced with documents.WLDocument."""
  39     LINE_SWAP_EXPR = re.compile(r'/\s', re.MULTILINE | re.UNICODE)
  40     provider = None
  41
  42     def __init__(self, edoc, parse_dublincore=True, provider=None,
  43                  strict=False, meta_fallbacks=None):
  44         self.edoc = edoc
  45         self.provider = provider
  46
  47         root_elem = edoc.getroot()
  48
  49         dc_path = './/' + RDFNS('RDF')
  50
  51         if root_elem.tag != 'utwor':
  52             raise ValidationError(
  53                 "Invalid root element. Found '%s', should be 'utwor'"
  54                 % root_elem.tag
  55             )
  56
  57         if parse_dublincore:
  58             self.rdf_elem = root_elem.find(dc_path)
  59
  60             if self.rdf_elem is None:
  61                 raise NoDublinCore(
  62                     "Document must have a '%s' element." % RDFNS('RDF')
  63                 )
  64
  65             self.book_info = dcparser.BookInfo.from_element(
  66                 self.rdf_elem, fallbacks=meta_fallbacks, strict=strict)
  67         else:
  68             self.book_info = None
  69
  70     def get_statistics(self):
  71         def count_text(text, counter, in_fn=False, stanza=False):
  72             if text:
  73                 text = re.sub(r'\s+', ' ', text)
  74
  75                 chars = len(text) if text.strip() else 0
  76                 words = len(text.split()) if text.strip() else 0
  77
  78                 counter['chars_with_fn'] += chars
  79                 counter['words_with_fn'] += words
  80                 if not in_fn:
  81                     counter['chars'] += chars
  82                     counter['words'] += words
  83                 if not stanza:
  84                     counter['chars_out_verse_with_fn'] += chars
  85                     if not in_fn:
  86                         counter['chars_out_verse'] += chars
  87
  88         def count(elem, counter, in_fn=False, stanza=False):
  89             if elem.tag in (RDFNS('RDF'), 'nota_red', 'abstrakt', 'uwaga', 'ekstra'):
  90                 return
  91             if not in_fn and elem.tag in ('pa', 'pe', 'pr', 'pt', 'motyw'):
  92                 in_fn = True
  93             if elem.tag == 'strofa':
  94                 # count verses now
  95                 verses = len(elem.findall('.//br')) + 1
  96                 counter['verses_with_fn'] += verses
  97                 if not in_fn:
  98                     counter['verses'] += verses
  99                 stanza = True
 100             count_text(elem.text, counter, in_fn=in_fn, stanza=stanza)
 101             for child in elem:
 102                 count(child, counter, in_fn=in_fn, stanza=stanza)
 103                 count_text(child.tail, counter, in_fn=in_fn, stanza=stanza)
 104
 105         self.swap_endlines()
 106
 107         data = {
 108             "self": Counter(),
 109             "parts": [],
 110             "total": {
 111             }
 112         }
 113
 114         count(self.edoc.getroot(), data['self'])
 115         for k, v in data['self'].items():
 116             data['total'][k] = v
 117
 118         for part in self.parts(pass_part_errors=True):
 119             if isinstance(part, Exception):
 120                 data['parts'].append((None, {}))
 121             else:
 122                 data['parts'].append((part, part.get_statistics()))
 123                 for k, v in data['parts'][-1][1]['total'].items():
 124                     data['total'][k] = data['total'].get(k, 0) + v
 125
 126         return data
 127
 128     @classmethod
 129     def from_bytes(cls, xml, *args, **kwargs):
 130         return cls.from_file(io.BytesIO(xml), *args, **kwargs)
 131
 132     @classmethod
 133     def from_file(cls, xmlfile, *args, **kwargs):
 134
 135         # first, prepare for parsing
 136         if isinstance(xmlfile, str):
 137             file = open(xmlfile, 'rb')
 138             try:
 139                 data = file.read()
 140             finally:
 141                 file.close()
 142         else:
 143             data = xmlfile.read()
 144
 145         if not isinstance(data, str):
 146             data = data.decode('utf-8')
 147
 148         data = data.replace('\ufeff', '')
 149
 150         try:
 151             parser = etree.XMLParser(remove_blank_text=False)
 152             tree = etree.parse(io.BytesIO(data.encode('utf-8')), parser)
 153
 154             return cls(tree, *args, **kwargs)
 155         except (ExpatError, XMLSyntaxError, XSLTApplyError) as e:
 156             raise ParseError(e)
 157
 158     def swap_endlines(self):
 159         """Converts line breaks in stanzas into <br/> tags."""
 160         # only swap inside stanzas
 161         for elem in self.edoc.iter('strofa'):
 162             for child in list(elem):
 163                 if child.tail:
 164                     chunks = self.LINE_SWAP_EXPR.split(child.tail)
 165                     ins_index = elem.index(child) + 1
 166                     while len(chunks) > 1:
 167                         ins = etree.Element('br')
 168                         ins.tail = chunks.pop()
 169                         elem.insert(ins_index, ins)
 170                     child.tail = chunks.pop(0)
 171             if elem.text:
 172                 chunks = self.LINE_SWAP_EXPR.split(elem.text)
 173                 while len(chunks) > 1:
 174                     ins = etree.Element('br')
 175                     ins.tail = chunks.pop()
 176                     elem.insert(0, ins)
 177                 elem.text = chunks.pop(0)
 178
 179     def parts(self, pass_part_errors=False):
 180         if self.provider is None:
 181             raise NoProvider('No document provider supplied.')
 182         if self.book_info is None:
 183             raise NoDublinCore('No Dublin Core in document.')
 184         for part_uri in self.book_info.parts:
 185             try:
 186                 with self.provider.by_slug(part_uri.slug) as f:
 187                     yield self.from_file(f, provider=self.provider)
 188             except Exception as e:
 189                 if pass_part_errors:
 190                     yield e
 191                 else:
 192                     raise
 193
 194     def chunk(self, path):
 195         # convert the path to XPath
 196         expr = self.path_to_xpath(path)
 197         elems = self.edoc.xpath(expr)
 198
 199         if len(elems) == 0:
 200             return None
 201         else:
 202             return elems[0]
 203
 204     def path_to_xpath(self, path):
 205         parts = []
 206
 207         for part in path.split('/'):
 208             match = re.match(r'([^\[]+)\[(\d+)\]', part)
 209             if not match:
 210                 parts.append(part)
 211             else:
 212                 tag, n = match.groups()
 213                 parts.append("*[%d][name() = '%s']" % (int(n)+1, tag))
 214
 215         if parts[0] == '.':
 216             parts[0] = ''
 217
 218         return '/'.join(parts)
 219
 220     def transform(self, stylesheet, **options):
 221         return self.edoc.xslt(stylesheet, **options)
 222
 223     def update_dc(self):
 224         if self.book_info:
 225             parent = self.rdf_elem.getparent()
 226             parent.replace(self.rdf_elem, self.book_info.to_etree(parent))
 227
 228     def serialize(self):
 229         self.update_dc()
 230         return etree.tostring(self.edoc, encoding='unicode', pretty_print=True)
 231
 232     def merge_chunks(self, chunk_dict):
 233         unmerged = []
 234
 235         for key, data in chunk_dict.iteritems():
 236             try:
 237                 xpath = self.path_to_xpath(key)
 238                 node = self.edoc.xpath(xpath)[0]
 239                 repl = etree.fromstring(
 240                     "<%s>%s</%s>" % (node.tag, data, node.tag)
 241                 )
 242                 node.getparent().replace(node, repl)
 243             except Exception as e:
 244                 unmerged.append(repr((key, xpath, e)))
 245
 246         return unmerged
 247
 248     def clean_ed_note(self, note_tag='nota_red'):
 249         """ deletes forbidden tags from nota_red """
 250
 251         for node in self.edoc.xpath('|'.join(
 252                 '//%s//%s' % (note_tag, tag) for tag in
 253                 ('pa', 'pe', 'pr', 'pt', 'begin', 'end', 'motyw'))):
 254             tail = node.tail
 255             node.clear()
 256             node.tag = 'span'
 257             node.tail = tail
 258
 259     def fix_pa_akap(self):
 260         for pa in ('pa','pe','pr','pt'):
 261             for akap in self.edoc.findall(f'//{pa}/akap'):
 262                 akap.getparent().set('blocks', 'true')
 263                 if not akap.getparent().index(akap):
 264                     akap.set('inline', 'true')
 265
 266     def editors(self):
 267         """Returns a set of all editors for book and its children.
 268
 269         :returns: set of dcparser.Person objects
 270         """
 271         if self.book_info is None:
 272             raise NoDublinCore('No Dublin Core in document.')
 273         persons = set(self.book_info.editors
 274                       + self.book_info.technical_editors)
 275         for child in self.parts():
 276             persons.update(child.editors())
 277         if None in persons:
 278             persons.remove(None)
 279         return persons
 280
 281     # Converters
 282
 283     def as_html(self, *args, **kwargs):
 284         from librarian import html
 285         return html.transform(self, *args, **kwargs)
 286
 287     def as_text(self, *args, **kwargs):
 288         from librarian import text
 289         return text.transform(self, *args, **kwargs)
 290
 291     def as_pdf(self, *args, **kwargs):
 292         from librarian import pdf
 293         return pdf.transform(self, *args, **kwargs)
 294
 295     def as_fb2(self, *args, **kwargs):
 296         from librarian import fb2
 297         return fb2.transform(self, *args, **kwargs)
 298
 299     def as_cover(self, cover_class=None, *args, **kwargs):
 300         if cover_class is None:
 301             cover_class = make_cover
 302         return cover_class(self.book_info, *args, **kwargs).output_file()
 303
 304     # for debugging only
 305     def latex_dir(self, *args, **kwargs):
 306         kwargs['latex_dir'] = True
 307         from librarian import pdf
 308         return pdf.transform(self, *args, **kwargs)
 309
 310     def save_output_file(self, output_file, output_path=None,
 311                          output_dir_path=None, make_author_dir=False,
 312                          ext=None):
 313         if output_dir_path:
 314             save_path = output_dir_path
 315             if make_author_dir:
 316                 save_path = os.path.join(
 317                     save_path,
 318                     str(self.book_info.author).encode('utf-8')
 319                 )
 320             save_path = os.path.join(save_path, self.book_info.url.slug)
 321             if ext:
 322                 save_path += '.%s' % ext
 323         else:
 324             save_path = output_path
 325
 326         output_file.save_as(save_path)