tabs and centered verses
[librarian.git] / src / librarian / parser.py
1 # -*- coding: utf-8 -*-
2 #
3 # This file is part of Librarian, licensed under GNU Affero GPLv3 or later.
4 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
5 #
6 from __future__ import unicode_literals
7
8 from collections import Counter
9
10 from librarian import ValidationError, NoDublinCore,  ParseError, NoProvider
11 from librarian import RDFNS
12 from librarian.cover import make_cover
13 from librarian import dcparser
14
15 from xml.parsers.expat import ExpatError
16 from lxml import etree
17 from lxml.etree import XMLSyntaxError, XSLTApplyError
18
19 import os
20 import re
21 import six
22
23
24 from .elements import WL_ELEMENTS
25
26
27 class WLElementLookup(etree.CustomElementClassLookup):
28     def lookup(self, node_type, document, namespace, name):
29         if node_type != 'element':
30             return
31         if namespace:
32             return
33         try:
34             return WL_ELEMENTS[name]
35         except KeyError:
36             return
37
38
39 parser = etree.XMLParser()
40 parser.set_element_class_lookup(
41     WLElementLookup()
42 )
43
44
45
46 class WLDocument(object):
47     """Legacy class, to be replaced with documents.WLDocument."""
48     LINE_SWAP_EXPR = re.compile(r'/\s', re.MULTILINE | re.UNICODE)
49     provider = None
50
51     def __init__(self, edoc, parse_dublincore=True, provider=None,
52                  strict=False, meta_fallbacks=None):
53         self.edoc = edoc
54         self.provider = provider
55
56         root_elem = edoc.getroot()
57
58         dc_path = './/' + RDFNS('RDF')
59
60         if root_elem.tag != 'utwor':
61             raise ValidationError(
62                 "Invalid root element. Found '%s', should be 'utwor'"
63                 % root_elem.tag
64             )
65
66         if parse_dublincore:
67             self.rdf_elem = root_elem.find(dc_path)
68
69             if self.rdf_elem is None:
70                 raise NoDublinCore(
71                     "Document must have a '%s' element." % RDFNS('RDF')
72                 )
73
74             self.book_info = dcparser.BookInfo.from_element(
75                 self.rdf_elem, fallbacks=meta_fallbacks, strict=strict)
76         else:
77             self.book_info = None
78
79     def get_statistics(self):
80         def count_text(text, counter, in_fn=False, stanza=False):
81             if text:
82                 text = re.sub(r'\s+', ' ', text)
83
84                 chars = len(text) if text.strip() else 0
85                 words = len(text.split()) if text.strip() else 0
86                 
87                 counter['chars_with_fn'] += chars
88                 counter['words_with_fn'] += words
89                 if not in_fn:
90                     counter['chars'] += chars
91                     counter['words'] += words
92                 if not stanza:
93                     counter['chars_out_verse_with_fn'] += chars
94                     if not in_fn:
95                         counter['chars_out_verse'] += chars
96                 
97         def count(elem, counter, in_fn=False, stanza=False):
98             if elem.tag in (RDFNS('RDF'), 'nota_red', 'abstrakt', 'uwaga', 'ekstra'):
99                 return
100             if not in_fn and elem.tag in ('pa', 'pe', 'pr', 'pt', 'motyw'):
101                 in_fn = True
102             if elem.tag == 'strofa':
103                 # count verses now
104                 verses = len(elem.findall('.//br')) + 1
105                 counter['verses_with_fn'] += verses
106                 if not in_fn:
107                     counter['verses'] += verses
108                 stanza = True
109             count_text(elem.text, counter, in_fn=in_fn, stanza=stanza)
110             for child in elem:
111                 count(child, counter, in_fn=in_fn, stanza=stanza)
112                 count_text(child.tail, counter, in_fn=in_fn, stanza=stanza)
113
114         self.swap_endlines()
115
116         data = {
117             "self": Counter(),
118             "parts": [],
119             "total": {
120             }
121         }
122
123         count(self.edoc.getroot(), data['self'])
124         for k, v in data['self'].items():
125             data['total'][k] = v
126         
127         for part in self.parts(pass_part_errors=True):
128             if isinstance(part, Exception):
129                 data['parts'].append((None, {}))
130             else:
131                 data['parts'].append((part, part.get_statistics()))
132                 for k, v in data['parts'][-1][1]['total'].items():
133                     data['total'][k] = data['total'].get(k, 0) + v
134             
135         return data
136
137     @classmethod
138     def from_bytes(cls, xml, *args, **kwargs):
139         return cls.from_file(six.BytesIO(xml), *args, **kwargs)
140
141     @classmethod
142     def from_file(cls, xmlfile, *args, **kwargs):
143
144         # first, prepare for parsing
145         if isinstance(xmlfile, six.text_type):
146             file = open(xmlfile, 'rb')
147             try:
148                 data = file.read()
149             finally:
150                 file.close()
151         else:
152             data = xmlfile.read()
153
154         if not isinstance(data, six.text_type):
155             data = data.decode('utf-8')
156
157         data = data.replace(u'\ufeff', '')
158
159         try:
160             parser = etree.XMLParser(remove_blank_text=False)
161             tree = etree.parse(six.BytesIO(data.encode('utf-8')), parser)
162
163             return cls(tree, *args, **kwargs)
164         except (ExpatError, XMLSyntaxError, XSLTApplyError) as e:
165             raise ParseError(e)
166
167     def swap_endlines(self):
168         """Converts line breaks in stanzas into <br/> tags."""
169         # only swap inside stanzas
170         for elem in self.edoc.iter('strofa'):
171             for child in list(elem):
172                 if child.tail:
173                     chunks = self.LINE_SWAP_EXPR.split(child.tail)
174                     ins_index = elem.index(child) + 1
175                     while len(chunks) > 1:
176                         ins = etree.Element('br')
177                         ins.tail = chunks.pop()
178                         elem.insert(ins_index, ins)
179                     child.tail = chunks.pop(0)
180             if elem.text:
181                 chunks = self.LINE_SWAP_EXPR.split(elem.text)
182                 while len(chunks) > 1:
183                     ins = etree.Element('br')
184                     ins.tail = chunks.pop()
185                     elem.insert(0, ins)
186                 elem.text = chunks.pop(0)
187
188     def parts(self, pass_part_errors=False):
189         if self.provider is None:
190             raise NoProvider('No document provider supplied.')
191         if self.book_info is None:
192             raise NoDublinCore('No Dublin Core in document.')
193         for part_uri in self.book_info.parts:
194             try:
195                 yield self.from_file(
196                     self.provider.by_slug(part_uri.slug), provider=self.provider
197                 )
198             except Exception as e:
199                 if pass_part_errors:
200                     yield e
201                 else:
202                     raise
203
204     def chunk(self, path):
205         # convert the path to XPath
206         expr = self.path_to_xpath(path)
207         elems = self.edoc.xpath(expr)
208
209         if len(elems) == 0:
210             return None
211         else:
212             return elems[0]
213
214     def path_to_xpath(self, path):
215         parts = []
216
217         for part in path.split('/'):
218             match = re.match(r'([^\[]+)\[(\d+)\]', part)
219             if not match:
220                 parts.append(part)
221             else:
222                 tag, n = match.groups()
223                 parts.append("*[%d][name() = '%s']" % (int(n)+1, tag))
224
225         if parts[0] == '.':
226             parts[0] = ''
227
228         return '/'.join(parts)
229
230     def transform(self, stylesheet, **options):
231         return self.edoc.xslt(stylesheet, **options)
232
233     def update_dc(self):
234         if self.book_info:
235             parent = self.rdf_elem.getparent()
236             parent.replace(self.rdf_elem, self.book_info.to_etree(parent))
237
238     def serialize(self):
239         self.update_dc()
240         return etree.tostring(self.edoc, encoding='unicode', pretty_print=True)
241
242     def merge_chunks(self, chunk_dict):
243         unmerged = []
244
245         for key, data in chunk_dict.iteritems():
246             try:
247                 xpath = self.path_to_xpath(key)
248                 node = self.edoc.xpath(xpath)[0]
249                 repl = etree.fromstring(
250                     "<%s>%s</%s>" % (node.tag, data, node.tag)
251                 )
252                 node.getparent().replace(node, repl)
253             except Exception as e:
254                 unmerged.append(repr((key, xpath, e)))
255
256         return unmerged
257
258     def clean_ed_note(self, note_tag='nota_red'):
259         """ deletes forbidden tags from nota_red """
260
261         for node in self.edoc.xpath('|'.join(
262                 '//%s//%s' % (note_tag, tag) for tag in
263                 ('pa', 'pe', 'pr', 'pt', 'begin', 'end', 'motyw'))):
264             tail = node.tail
265             node.clear()
266             node.tag = 'span'
267             node.tail = tail
268
269     def fix_pa_akap(self):
270         for pa in ('pa','pe','pr','pt'):
271             for akap in self.edoc.findall(f'//{pa}/akap'):
272                 akap.getparent().set('blocks', 'true')
273                 if not akap.getparent().index(akap):
274                     akap.set('inline', 'true')
275             
276     def editors(self):
277         """Returns a set of all editors for book and its children.
278
279         :returns: set of dcparser.Person objects
280         """
281         if self.book_info is None:
282             raise NoDublinCore('No Dublin Core in document.')
283         persons = set(self.book_info.editors
284                       + self.book_info.technical_editors)
285         for child in self.parts():
286             persons.update(child.editors())
287         if None in persons:
288             persons.remove(None)
289         return persons
290
291     # Converters
292
293     def as_html(self, *args, **kwargs):
294         from librarian import html
295         return html.transform(self, *args, **kwargs)
296
297     def as_text(self, *args, **kwargs):
298         from librarian import text
299         return text.transform(self, *args, **kwargs)
300
301     def as_epub(self, *args, **kwargs):
302         from librarian import epub
303         return epub.transform(self, *args, **kwargs)
304
305     def as_pdf(self, *args, **kwargs):
306         from librarian import pdf
307         return pdf.transform(self, *args, **kwargs)
308
309     def as_mobi(self, *args, **kwargs):
310         from librarian import mobi
311         return mobi.transform(self, *args, **kwargs)
312
313     def as_fb2(self, *args, **kwargs):
314         from librarian import fb2
315         return fb2.transform(self, *args, **kwargs)
316
317     def as_cover(self, cover_class=None, *args, **kwargs):
318         if cover_class is None:
319             cover_class = make_cover
320         return cover_class(self.book_info, *args, **kwargs).output_file()
321
322     # for debugging only
323     def latex_dir(self, *args, **kwargs):
324         kwargs['latex_dir'] = True
325         from librarian import pdf
326         return pdf.transform(self, *args, **kwargs)
327
328     def save_output_file(self, output_file, output_path=None,
329                          output_dir_path=None, make_author_dir=False,
330                          ext=None):
331         if output_dir_path:
332             save_path = output_dir_path
333             if make_author_dir:
334                 save_path = os.path.join(
335                     save_path,
336                     six.text_type(self.book_info.author).encode('utf-8')
337                 )
338             save_path = os.path.join(save_path, self.book_info.url.slug)
339             if ext:
340                 save_path += '.%s' % ext
341         else:
342             save_path = output_path
343
344         output_file.save_as(save_path)