1 # This file is part of Librarian, licensed under GNU Affero GPLv3 or later.
2 # Copyright © Fundacja Wolne Lektury. See NOTICE for more information.
4 from collections import Counter
10 from .parser import parser
11 from . import dcparser, DCNS, RDFNS, DirDocProvider
12 from .functions import lang_code_3to2
16 def __init__(self, filename=None, url=None, provider=None):
17 source = filename or urllib.request.urlopen(url)
18 tree = etree.parse(source, parser=parser)
20 tree.getroot().document = self
21 self.base_meta = dcparser.BookInfo({}, {
22 DCNS('language'): ["pol"],
23 }, validate_required=False)
25 self.provider = provider if provider is not None else DirDocProvider('.')
27 self.tree.getroot().validate()
31 # Allow metadata of the master element as document meta.
32 #master = self.tree.getroot()[-1]
33 return self.tree.getroot().meta
38 for part_uri in self.meta.parts or []:
39 with self.provider.by_slug(part_uri.slug) as f:
41 yield type(self)(filename=f, provider=self.provider)
42 except Exception as e:
46 def build(self, builder, base_url=None, **kwargs):
47 return builder(base_url=base_url).build(self, **kwargs)
49 def assign_ids(self, existing=None):
50 # Find all existing IDs.
51 existing = existing or set()
52 que = [self.tree.getroot()]
56 item.normalize_insides()
57 except AttributeError:
59 existing.add(item.attrib.get('id'))
63 que = [self.tree.getroot()]
67 if item.attrib.get('id'):
69 if not getattr(item, 'SHOULD_HAVE_ID', False):
71 while f'e{i}' in existing:
73 item.attrib['id'] = f'e{i}'
76 def _compat_assign_ordered_ids(self):
78 Compatibility: ids in document order, to be roughly compatible with legacy
79 footnote ids. Just for testing consistency, change to some sane identifiers
82 EXPR = re.compile(r'/\s', re.MULTILINE | re.UNICODE)
83 def _compat_assign_ordered_ids_in_elem(elem, i):
84 elem.attrib['_compat_ordered_id'] = str(i)
86 if getattr(elem, 'HTML_CLASS', None) == 'stanza':
88 i += len(EXPR.split(elem.text)) - 1
90 i = _compat_assign_ordered_ids_in_elem(sub, i)
92 i += len(EXPR.split(sub.tail)) - 1
94 if elem.tag in ('uwaga', 'extra'):
97 i = _compat_assign_ordered_ids_in_elem(sub, i)
100 _compat_assign_ordered_ids_in_elem(self.tree.getroot(), 4)
102 def _compat_assign_section_ids(self):
104 Ids in master-section order. These need to be compatible with the
105 #secN anchors used by WL search results page to link to fragments.
107 def _compat_assigns_section_ids_in_elem(elem, prefix='sec'):
108 for i, child in enumerate(elem):
109 idfier = '{}{}'.format(prefix, i + 1)
111 child.attrib['_compat_section_id'] = idfier
114 _compat_assigns_section_ids_in_elem(child, idfier + '-')
115 _compat_assigns_section_ids_in_elem(self.tree.getroot().master)
119 persons = set(self.meta.editors
120 + self.meta.technical_editors)
121 for child in self.children:
122 persons.update(child.editors())
127 def references(self):
128 return self.tree.findall('.//ref')
130 def get_statistics(self):
131 def count_text(text, counter, in_fn=False, stanza=False):
133 text = re.sub(r'\s+', ' ', text)
135 chars = len(text) if text.strip() else 0
136 words = len(text.split()) if text.strip() else 0
138 counter['chars_with_fn'] += chars
139 counter['words_with_fn'] += words
141 counter['chars'] += chars
142 counter['words'] += words
144 counter['chars_out_verse_with_fn'] += chars
146 counter['chars_out_verse'] += chars
148 def count(elem, counter, in_fn=False, stanza=False):
149 if elem.tag in (RDFNS('RDF'), 'nota_red', 'abstrakt', 'uwaga', 'ekstra'):
151 if not in_fn and elem.tag in ('pa', 'pe', 'pr', 'pt', 'motyw'):
153 if elem.tag == 'strofa':
155 #verses = len(elem.findall('.//br')) + 1
156 verses = list(elem.get_verses())
157 counter['verses_with_fn'] += len(verses)
159 counter['verses'] += len(verses)
163 count(child, counter, in_fn=in_fn, stanza=True)
165 count_text(elem.text, counter, in_fn=in_fn, stanza=stanza)
167 count(child, counter, in_fn=in_fn, stanza=stanza)
168 count_text(child.tail, counter, in_fn=in_fn, stanza=stanza)
177 count(self.tree.getroot(), data['self'])
178 for k, v in data['self'].items():
181 for part in self.children:
182 if isinstance(part, Exception):
183 data['parts'].append((None, {'error': part}))
185 data['parts'].append((part, part.get_statistics()))
186 for k, v in data['parts'][-1][1]['total'].items():
187 data['total'][k] = data['total'].get(k, 0) + v