8adde33c11b4c96c4ee442ccd8dbbf8af20f2682
[librarian.git] / src / librarian / parser.py
1 # -*- coding: utf-8 -*-
2 #
3 # This file is part of Librarian, licensed under GNU Affero GPLv3 or later.
4 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
5 #
6 from __future__ import unicode_literals
7
8 from collections import Counter
9
10 from librarian import ValidationError, NoDublinCore,  ParseError, NoProvider
11 from librarian import RDFNS
12 from librarian.cover import make_cover
13 from librarian import dcparser
14
15 from xml.parsers.expat import ExpatError
16 from lxml import etree
17 from lxml.etree import XMLSyntaxError, XSLTApplyError
18
19 import os
20 import re
21 import six
22
23
24 from .elements import WL_ELEMENTS
25
26
27 class WLElementLookup(etree.CustomElementClassLookup):
28     def lookup(self, node_type, document, namespace, name):
29         if node_type != 'element':
30             return
31         if namespace:
32             return
33         return WL_ELEMENTS[name]
34
35
36 parser = etree.XMLParser()
37 parser.set_element_class_lookup(
38     WLElementLookup()
39 )
40
41
42
43 class WLDocument(object):
44     """Legacy class, to be replaced with documents.WLDocument."""
45     LINE_SWAP_EXPR = re.compile(r'/\s', re.MULTILINE | re.UNICODE)
46     provider = None
47
48     def __init__(self, edoc, parse_dublincore=True, provider=None,
49                  strict=False, meta_fallbacks=None):
50         self.edoc = edoc
51         self.provider = provider
52
53         root_elem = edoc.getroot()
54
55         dc_path = './/' + RDFNS('RDF')
56
57         if root_elem.tag != 'utwor':
58             raise ValidationError(
59                 "Invalid root element. Found '%s', should be 'utwor'"
60                 % root_elem.tag
61             )
62
63         if parse_dublincore:
64             self.rdf_elem = root_elem.find(dc_path)
65
66             if self.rdf_elem is None:
67                 raise NoDublinCore(
68                     "Document must have a '%s' element." % RDFNS('RDF')
69                 )
70
71             self.book_info = dcparser.BookInfo.from_element(
72                 self.rdf_elem, fallbacks=meta_fallbacks, strict=strict)
73         else:
74             self.book_info = None
75
76     def get_statistics(self):
77         def count_text(text, counter, in_fn=False):
78             if text:
79                 text = re.sub(r'\s+', ' ', text)
80
81                 chars = len(text) if text.strip() else 0
82                 words = len(text.split()) if text.strip() else 0
83                 
84                 counter['chars'] += chars
85                 counter['words'] += words
86                 if not in_fn:
87                     counter['chars_with_fn'] += chars
88                     counter['words_with_fn'] += words
89                 
90         def count(elem, counter, in_fn=False):
91             if elem.tag in (RDFNS('RDF'), 'nota_red', 'abstrakt', 'uwaga', 'ekstra'):
92                 return
93             if not in_fn and elem.tag in ('pa', 'pe', 'pr', 'pt', 'motyw'):
94                 in_fn = True
95             count_text(elem.text, counter, in_fn=in_fn)
96             for child in elem:
97                 count(child, counter, in_fn=in_fn)
98                 count_text(child.tail, counter, in_fn=in_fn)
99             
100             
101         data = {
102             "self": Counter(),
103             "parts": [],
104             "total": {
105             }
106         }
107
108         count(self.edoc.getroot(), data['self'])
109         for k, v in data['self'].items():
110             data['total'][k] = v
111         
112         for part in self.parts(pass_part_errors=True):
113             if isinstance(part, Exception):
114                 data['parts'].append((None, {}))
115             else:
116                 data['parts'].append((part, part.get_statistics()))
117                 for k, v in data['parts'][-1][1]['total'].items():
118                     data['total'][k] += v
119             
120         return data
121
122     @classmethod
123     def from_bytes(cls, xml, *args, **kwargs):
124         return cls.from_file(six.BytesIO(xml), *args, **kwargs)
125
126     @classmethod
127     def from_file(cls, xmlfile, *args, **kwargs):
128
129         # first, prepare for parsing
130         if isinstance(xmlfile, six.text_type):
131             file = open(xmlfile, 'rb')
132             try:
133                 data = file.read()
134             finally:
135                 file.close()
136         else:
137             data = xmlfile.read()
138
139         if not isinstance(data, six.text_type):
140             data = data.decode('utf-8')
141
142         data = data.replace(u'\ufeff', '')
143
144         try:
145             parser = etree.XMLParser(remove_blank_text=False)
146             tree = etree.parse(six.BytesIO(data.encode('utf-8')), parser)
147
148             return cls(tree, *args, **kwargs)
149         except (ExpatError, XMLSyntaxError, XSLTApplyError) as e:
150             raise ParseError(e)
151
152     def swap_endlines(self):
153         """Converts line breaks in stanzas into <br/> tags."""
154         # only swap inside stanzas
155         for elem in self.edoc.iter('strofa'):
156             for child in list(elem):
157                 if child.tail:
158                     chunks = self.LINE_SWAP_EXPR.split(child.tail)
159                     ins_index = elem.index(child) + 1
160                     while len(chunks) > 1:
161                         ins = etree.Element('br')
162                         ins.tail = chunks.pop()
163                         elem.insert(ins_index, ins)
164                     child.tail = chunks.pop(0)
165             if elem.text:
166                 chunks = self.LINE_SWAP_EXPR.split(elem.text)
167                 while len(chunks) > 1:
168                     ins = etree.Element('br')
169                     ins.tail = chunks.pop()
170                     elem.insert(0, ins)
171                 elem.text = chunks.pop(0)
172
173     def parts(self, pass_part_errors=False):
174         if self.provider is None:
175             raise NoProvider('No document provider supplied.')
176         if self.book_info is None:
177             raise NoDublinCore('No Dublin Core in document.')
178         for part_uri in self.book_info.parts:
179             try:
180                 yield self.from_file(
181                     self.provider.by_uri(part_uri), provider=self.provider
182                 )
183             except Exception as e:
184                 if pass_part_errors:
185                     yield e
186                 else:
187                     raise
188
189     def chunk(self, path):
190         # convert the path to XPath
191         expr = self.path_to_xpath(path)
192         elems = self.edoc.xpath(expr)
193
194         if len(elems) == 0:
195             return None
196         else:
197             return elems[0]
198
199     def path_to_xpath(self, path):
200         parts = []
201
202         for part in path.split('/'):
203             match = re.match(r'([^\[]+)\[(\d+)\]', part)
204             if not match:
205                 parts.append(part)
206             else:
207                 tag, n = match.groups()
208                 parts.append("*[%d][name() = '%s']" % (int(n)+1, tag))
209
210         if parts[0] == '.':
211             parts[0] = ''
212
213         return '/'.join(parts)
214
215     def transform(self, stylesheet, **options):
216         return self.edoc.xslt(stylesheet, **options)
217
218     def update_dc(self):
219         if self.book_info:
220             parent = self.rdf_elem.getparent()
221             parent.replace(self.rdf_elem, self.book_info.to_etree(parent))
222
223     def serialize(self):
224         self.update_dc()
225         return etree.tostring(self.edoc, encoding='unicode', pretty_print=True)
226
227     def merge_chunks(self, chunk_dict):
228         unmerged = []
229
230         for key, data in chunk_dict.iteritems():
231             try:
232                 xpath = self.path_to_xpath(key)
233                 node = self.edoc.xpath(xpath)[0]
234                 repl = etree.fromstring(
235                     "<%s>%s</%s>" % (node.tag, data, node.tag)
236                 )
237                 node.getparent().replace(node, repl)
238             except Exception as e:
239                 unmerged.append(repr((key, xpath, e)))
240
241         return unmerged
242
243     def clean_ed_note(self, note_tag='nota_red'):
244         """ deletes forbidden tags from nota_red """
245
246         for node in self.edoc.xpath('|'.join(
247                 '//%s//%s' % (note_tag, tag) for tag in
248                 ('pa', 'pe', 'pr', 'pt', 'begin', 'end', 'motyw'))):
249             tail = node.tail
250             node.clear()
251             node.tag = 'span'
252             node.tail = tail
253
254     def fix_pa_akap(self):
255         for pa in ('pa','pe','pr','pt'):
256             for akap in self.edoc.findall(f'//{pa}/akap'):
257                 akap.getparent().set('blocks', 'true')
258                 if not akap.getparent().index(akap):
259                     akap.set('inline', 'true')
260             
261     def editors(self):
262         """Returns a set of all editors for book and its children.
263
264         :returns: set of dcparser.Person objects
265         """
266         if self.book_info is None:
267             raise NoDublinCore('No Dublin Core in document.')
268         persons = set(self.book_info.editors
269                       + self.book_info.technical_editors)
270         for child in self.parts():
271             persons.update(child.editors())
272         if None in persons:
273             persons.remove(None)
274         return persons
275
276     # Converters
277
278     def as_html(self, *args, **kwargs):
279         from librarian import html
280         return html.transform(self, *args, **kwargs)
281
282     def as_text(self, *args, **kwargs):
283         from librarian import text
284         return text.transform(self, *args, **kwargs)
285
286     def as_epub(self, *args, **kwargs):
287         from librarian import epub
288         return epub.transform(self, *args, **kwargs)
289
290     def as_pdf(self, *args, **kwargs):
291         from librarian import pdf
292         return pdf.transform(self, *args, **kwargs)
293
294     def as_mobi(self, *args, **kwargs):
295         from librarian import mobi
296         return mobi.transform(self, *args, **kwargs)
297
298     def as_fb2(self, *args, **kwargs):
299         from librarian import fb2
300         return fb2.transform(self, *args, **kwargs)
301
302     def as_cover(self, cover_class=None, *args, **kwargs):
303         if cover_class is None:
304             cover_class = make_cover
305         return cover_class(self.book_info, *args, **kwargs).output_file()
306
307     # for debugging only
308     def latex_dir(self, *args, **kwargs):
309         kwargs['latex_dir'] = True
310         from librarian import pdf
311         return pdf.transform(self, *args, **kwargs)
312
313     def save_output_file(self, output_file, output_path=None,
314                          output_dir_path=None, make_author_dir=False,
315                          ext=None):
316         if output_dir_path:
317             save_path = output_dir_path
318             if make_author_dir:
319                 save_path = os.path.join(
320                     save_path,
321                     six.text_type(self.book_info.author).encode('utf-8')
322                 )
323             save_path = os.path.join(save_path, self.book_info.url.slug)
324             if ext:
325                 save_path += '.%s' % ext
326         else:
327             save_path = output_path
328
329         output_file.save_as(save_path)