d61abb443f1c698d498220917418d8b372e39bb9
[librarian.git] / src / librarian / document.py
1 # This file is part of Librarian, licensed under GNU Affero GPLv3 or later.
2 # Copyright © Fundacja Wolne Lektury. See NOTICE for more information.
3 #
4 from collections import Counter
5 import gettext
6 import os
7 import re
8 import urllib.request
9 from lxml import etree
10 from .parser import parser
11 from . import dcparser, DCNS, RDFNS, DirDocProvider
12 from .functions import lang_code_3to2
13
14
15 class WLDocument:
16     def __init__(self, filename=None, url=None, provider=None):
17         source = filename or urllib.request.urlopen(url)
18         tree = etree.parse(source, parser=parser)
19         self.tree = tree
20         tree.getroot().document = self
21
22         self.preprocess()
23
24         self.base_meta = dcparser.BookInfo({}, {
25             DCNS('language'): ["pol"],
26         }, validate_required=False)
27
28         self.provider = provider if provider is not None else DirDocProvider('.')
29
30         self.tree.getroot().validate()
31
32     @property
33     def meta(self):
34         # Allow metadata of the master element as document meta.
35         #master = self.tree.getroot()[-1]
36         return self.tree.getroot().meta
37         return master.meta
38
39     def preprocess(self):
40         # Change slash-verses into real verses.
41         for _e, elem in etree.iterwalk(self.tree, ('start',), 'strofa'):
42             elem.preprocess()
43
44     @property
45     def children(self):
46         for part_uri in self.meta.parts or []:
47             with self.provider.by_slug(part_uri.slug) as f:
48                 try:
49                     yield type(self)(filename=f, provider=self.provider)
50                 except Exception as e:
51
52                     yield e
53
54     def build(self, builder, base_url=None, **kwargs):
55         return builder(base_url=base_url).build(self, **kwargs)
56
57     def editors(self):
58         persons = set(self.meta.editors
59                       + self.meta.technical_editors)
60         for child in self.children:
61             persons.update(child.editors())
62         if None in persons:
63             persons.remove(None)
64         return persons
65
66     def references(self):
67         return self.tree.findall('.//ref')
68
69     def get_statistics(self):
70         def count_text(text, counter, in_fn=False, stanza=False):
71             if text:
72                 text = re.sub(r'\s+', ' ', text)
73
74                 chars = len(text) if text.strip() else 0
75                 words = len(text.split()) if text.strip() else 0
76
77                 counter['chars_with_fn'] += chars
78                 counter['words_with_fn'] += words
79                 if not in_fn:
80                     counter['chars'] += chars
81                     counter['words'] += words
82                 if not stanza:
83                     counter['chars_out_verse_with_fn'] += chars
84                     if not in_fn:
85                         counter['chars_out_verse'] += chars
86
87         def count(elem, counter, in_fn=False, stanza=False):
88             if elem.tag in (RDFNS('RDF'), 'nota_red', 'abstrakt', 'uwaga', 'ekstra'):
89                 return
90             if not in_fn and elem.tag in ('pa', 'pe', 'pr', 'pt', 'motyw'):
91                 in_fn = True
92             if elem.tag == 'strofa':
93                 # count verses now
94                 #verses = len(elem.findall('.//br')) + 1
95                 verses = list(elem.get_verses())
96                 counter['verses_with_fn'] += len(verses)
97                 if not in_fn:
98                     counter['verses'] += len(verses)
99                 stanza = True
100
101                 for child in verses:
102                     count(child, counter, in_fn=in_fn, stanza=True)
103             else:
104                 count_text(elem.text, counter, in_fn=in_fn, stanza=stanza)
105                 for child in elem:
106                     count(child, counter, in_fn=in_fn, stanza=stanza)
107                     count_text(child.tail, counter, in_fn=in_fn, stanza=stanza)
108
109         data = {
110             "self": Counter(),
111             "parts": [],
112             "total": {
113             }
114         }
115
116         count(self.tree.getroot(), data['self'])
117         for k, v in data['self'].items():
118             data['total'][k] = v
119
120         for part in self.children:
121             if isinstance(part, Exception):
122                 data['parts'].append((None, {'error': part}))
123             else:
124                 data['parts'].append((part, part.get_statistics()))
125                 for k, v in data['parts'][-1][1]['total'].items():
126                     data['total'][k] = data['total'].get(k, 0) + v
127
128         return data