2e9a4a5ff8fa938f5185d66c34499ce095e8df9f
[librarian.git] / src / librarian / document.py
1 # This file is part of Librarian, licensed under GNU Affero GPLv3 or later.
2 # Copyright © Fundacja Wolne Lektury. See NOTICE for more information.
3 #
4 from collections import Counter
5 import gettext
6 import os
7 import re
8 import urllib.request
9 from lxml import etree
10 from .parser import parser
11 from . import dcparser, DCNS, RDFNS, DirDocProvider
12 from .functions import lang_code_3to2
13
14
15 class WLDocument:
16     def __init__(self, filename=None, url=None, provider=None):
17         source = filename or urllib.request.urlopen(url)
18         tree = etree.parse(source, parser=parser)
19         self.tree = tree
20         tree.getroot().document = self
21         self.base_meta = dcparser.BookInfo({}, {
22             DCNS('language'): ["pol"],
23         }, validate_required=False)
24
25         self.provider = provider if provider is not None else DirDocProvider('.')
26
27         self.tree.getroot().validate()
28
29     @property
30     def meta(self):
31         # Allow metadata of the master element as document meta.
32         #master = self.tree.getroot()[-1]
33         return self.tree.getroot().meta
34         return master.meta
35
36     @property
37     def children(self):
38         for part_uri in self.meta.parts or []:
39             with self.provider.by_slug(part_uri.slug) as f:
40                 try:
41                     yield type(self)(filename=f, provider=self.provider)
42                 except Exception as e:
43
44                     yield e
45
46     def build(self, builder, base_url=None, **kwargs):
47         return builder(base_url=base_url).build(self, **kwargs)
48
49     def assign_ids(self, existing=None):
50         # Find all existing IDs.
51         existing = existing or set()
52         que = [self.tree.getroot()]
53         while que:
54             item = que.pop(0)
55             try:
56                 item.normalize_insides()
57             except AttributeError:
58                 pass
59             existing.add(item.attrib.get('id'))
60             que.extend(item)
61
62         i = 1
63         que = [self.tree.getroot()]
64         while que:
65             item = que.pop(0)
66             que.extend(item)
67             if item.attrib.get('id'):
68                 continue
69             if not getattr(item, 'SHOULD_HAVE_ID', False):
70                 continue
71             while f'e{i}' in existing:
72                 i += 1
73             item.attrib['id'] = f'e{i}'
74             i += 1
75
76     def _compat_assign_ordered_ids(self):
77         """
78         Compatibility: ids in document order, to be roughly compatible with legacy
79         footnote ids. Just for testing consistency, change to some sane identifiers
80         at convenience.
81         """
82         EXPR = re.compile(r'/\s', re.MULTILINE | re.UNICODE)
83         def _compat_assign_ordered_ids_in_elem(elem, i):
84             elem.attrib['_compat_ordered_id'] = str(i)
85             i += 1
86             if getattr(elem, 'HTML_CLASS', None) == 'stanza':
87                 if elem.text:
88                     i += len(EXPR.split(elem.text)) - 1
89                 for sub in elem:
90                     i = _compat_assign_ordered_ids_in_elem(sub, i)
91                     if sub.tail:
92                         i += len(EXPR.split(sub.tail)) - 1
93             else:
94                 if elem.tag in ('uwaga', 'extra'):
95                     return i
96                 for sub in elem:
97                     i = _compat_assign_ordered_ids_in_elem(sub, i)
98             return i
99
100         _compat_assign_ordered_ids_in_elem(self.tree.getroot(), 4)
101
102     def _compat_assign_section_ids(self):
103         """
104         Ids in master-section order. These need to be compatible with the
105         #secN anchors used by WL search results page to link to fragments.
106         """
107         def _compat_assigns_section_ids_in_elem(elem, prefix='sec'):
108             for i, child in enumerate(elem):
109                 idfier = '{}{}'.format(prefix, i + 1)
110                 try:
111                     child.attrib['_compat_section_id'] = idfier
112                 except:
113                     pass
114                 _compat_assigns_section_ids_in_elem(child, idfier + '-')
115         _compat_assigns_section_ids_in_elem(self.tree.getroot().master)
116
117
118     def editors(self):
119         persons = set(self.meta.editors
120                       + self.meta.technical_editors)
121         for child in self.children:
122             persons.update(child.editors())
123         if None in persons:
124             persons.remove(None)
125         return persons
126
127     def references(self):
128         return self.tree.findall('.//ref')
129
130     def get_statistics(self):
131         def count_text(text, counter, in_fn=False, stanza=False):
132             if text:
133                 text = re.sub(r'\s+', ' ', text)
134
135                 chars = len(text) if text.strip() else 0
136                 words = len(text.split()) if text.strip() else 0
137
138                 counter['chars_with_fn'] += chars
139                 counter['words_with_fn'] += words
140                 if not in_fn:
141                     counter['chars'] += chars
142                     counter['words'] += words
143                 if not stanza:
144                     counter['chars_out_verse_with_fn'] += chars
145                     if not in_fn:
146                         counter['chars_out_verse'] += chars
147
148         def count(elem, counter, in_fn=False, stanza=False):
149             if elem.tag in (RDFNS('RDF'), 'nota_red', 'abstrakt', 'uwaga', 'ekstra'):
150                 return
151             if not in_fn and elem.tag in ('pa', 'pe', 'pr', 'pt', 'motyw'):
152                 in_fn = True
153             if elem.tag == 'strofa':
154                 # count verses now
155                 #verses = len(elem.findall('.//br')) + 1
156                 verses = list(elem.get_verses())
157                 counter['verses_with_fn'] += len(verses)
158                 if not in_fn:
159                     counter['verses'] += len(verses)
160                 stanza = True
161
162                 for child in verses:
163                     count(child, counter, in_fn=in_fn, stanza=True)
164             else:
165                 count_text(elem.text, counter, in_fn=in_fn, stanza=stanza)
166                 for child in elem:
167                     count(child, counter, in_fn=in_fn, stanza=stanza)
168                     count_text(child.tail, counter, in_fn=in_fn, stanza=stanza)
169
170         data = {
171             "self": Counter(),
172             "parts": [],
173             "total": {
174             }
175         }
176
177         count(self.tree.getroot(), data['self'])
178         for k, v in data['self'].items():
179             data['total'][k] = v
180
181         for part in self.children:
182             if isinstance(part, Exception):
183                 data['parts'].append((None, {'error': part}))
184             else:
185                 data['parts'].append((part, part.get_statistics()))
186                 for k, v in data['parts'][-1][1]['total'].items():
187                     data['total'][k] = data['total'].get(k, 0) + v
188
189         return data