src/librarian/document.py

   1 # This file is part of Librarian, licensed under GNU Affero GPLv3 or later.
   2 # Copyright © Fundacja Wolne Lektury. See NOTICE for more information.
   3 #
   4 from collections import Counter
   5 import gettext
   6 import os
   7 import re
   8 import urllib.request
   9 from lxml import etree
  10 from .parser import parser
  11 from . import dcparser, DCNS, RDFNS, DirDocProvider
  12 from .functions import lang_code_3to2
  13
  14
  15 class WLDocument:
  16     def __init__(self, filename=None, url=None, provider=None):
  17         source = filename or urllib.request.urlopen(url)
  18         tree = etree.parse(source, parser=parser)
  19         self.tree = tree
  20         tree.getroot().document = self
  21         self.base_meta = dcparser.BookInfo({}, {
  22             DCNS('language'): ["pol"],
  23         }, validate_required=False)
  24
  25         self.provider = provider if provider is not None else DirDocProvider('.')
  26
  27         self.tree.getroot().validate()
  28
  29     @property
  30     def meta(self):
  31         # Allow metadata of the master element as document meta.
  32         #master = self.tree.getroot()[-1]
  33         return self.tree.getroot().meta
  34         return master.meta
  35
  36     @property
  37     def children(self):
  38         for part_uri in self.meta.parts or []:
  39             with self.provider.by_slug(part_uri.slug) as f:
  40                 try:
  41                     yield type(self)(filename=f, provider=self.provider)
  42                 except Exception as e:
  43
  44                     yield e
  45
  46     def build(self, builder, base_url=None, **kwargs):
  47         return builder(base_url=base_url).build(self, **kwargs)
  48
  49     def assign_ids(self, existing=None):
  50         # Find all existing IDs.
  51         existing = existing or set()
  52         que = [self.tree.getroot()]
  53         while que:
  54             item = que.pop(0)
  55             try:
  56                 item.normalize_insides()
  57             except AttributeError:
  58                 pass
  59             existing.add(item.attrib.get('id'))
  60             que.extend(item)
  61
  62         i = 1
  63         que = [self.tree.getroot()]
  64         while que:
  65             item = que.pop(0)
  66             que.extend(item)
  67             if item.attrib.get('id'):
  68                 continue
  69             if not getattr(item, 'SHOULD_HAVE_ID', False):
  70                 continue
  71             while f'e{i}' in existing:
  72                 i += 1
  73             item.attrib['id'] = f'e{i}'
  74             i += 1
  75
  76     def _compat_assign_ordered_ids(self):
  77         """
  78         Compatibility: ids in document order, to be roughly compatible with legacy
  79         footnote ids. Just for testing consistency, change to some sane identifiers
  80         at convenience.
  81         """
  82         EXPR = re.compile(r'/\s', re.MULTILINE | re.UNICODE)
  83         def _compat_assign_ordered_ids_in_elem(elem, i):
  84             if isinstance(elem, etree._Comment): return i
  85             elem.attrib['_compat_ordered_id'] = str(i)
  86             i += 1
  87             if getattr(elem, 'HTML_CLASS', None) == 'stanza':
  88                 if elem.text:
  89                     i += len(EXPR.split(elem.text)) - 1
  90                 for sub in elem:
  91                     i = _compat_assign_ordered_ids_in_elem(sub, i)
  92                     if sub.tail:
  93                         i += len(EXPR.split(sub.tail)) - 1
  94             else:
  95                 if elem.tag in ('uwaga', 'extra'):
  96                     return i
  97                 for sub in elem:
  98                     i = _compat_assign_ordered_ids_in_elem(sub, i)
  99             return i
 100
 101         _compat_assign_ordered_ids_in_elem(self.tree.getroot(), 4)
 102
 103     def _compat_assign_section_ids(self):
 104         """
 105         Ids in master-section order. These need to be compatible with the
 106         #secN anchors used by WL search results page to link to fragments.
 107         """
 108         def _compat_assigns_section_ids_in_elem(elem, prefix='sec'):
 109             for i, child in enumerate(elem):
 110                 idfier = '{}{}'.format(prefix, i + 1)
 111                 try:
 112                     child.attrib['_compat_section_id'] = idfier
 113                 except:
 114                     pass
 115                 _compat_assigns_section_ids_in_elem(child, idfier + '-')
 116         _compat_assigns_section_ids_in_elem(self.tree.getroot().master)
 117
 118
 119     def editors(self):
 120         persons = set(self.meta.editors
 121                       + self.meta.technical_editors)
 122         for child in self.children:
 123             persons.update(child.editors())
 124         if None in persons:
 125             persons.remove(None)
 126         return persons
 127
 128     def references(self):
 129         return self.tree.findall('.//ref')
 130
 131     def get_statistics(self):
 132         def count_text(text, counter, in_fn=False, stanza=False):
 133             if text:
 134                 text = re.sub(r'\s+', ' ', text)
 135
 136                 chars = len(text) if text.strip() else 0
 137                 words = len(text.split()) if text.strip() else 0
 138
 139                 counter['chars_with_fn'] += chars
 140                 counter['words_with_fn'] += words
 141                 if not in_fn:
 142                     counter['chars'] += chars
 143                     counter['words'] += words
 144                 if not stanza:
 145                     counter['chars_out_verse_with_fn'] += chars
 146                     if not in_fn:
 147                         counter['chars_out_verse'] += chars
 148
 149         def count(elem, counter, in_fn=False, stanza=False):
 150             if elem.tag in (RDFNS('RDF'), 'nota_red', 'abstrakt', 'uwaga', 'ekstra'):
 151                 return
 152             if not in_fn and elem.tag in ('pa', 'pe', 'pr', 'pt', 'motyw'):
 153                 in_fn = True
 154             if elem.tag == 'strofa':
 155                 # count verses now
 156                 #verses = len(elem.findall('.//br')) + 1
 157                 verses = list(elem.get_verses())
 158                 counter['verses_with_fn'] += len(verses)
 159                 if not in_fn:
 160                     counter['verses'] += len(verses)
 161                 stanza = True
 162
 163                 for child in verses:
 164                     count(child, counter, in_fn=in_fn, stanza=True)
 165             else:
 166                 count_text(elem.text, counter, in_fn=in_fn, stanza=stanza)
 167                 for child in elem:
 168                     count(child, counter, in_fn=in_fn, stanza=stanza)
 169                     count_text(child.tail, counter, in_fn=in_fn, stanza=stanza)
 170
 171         data = {
 172             "self": Counter(),
 173             "parts": [],
 174             "total": {
 175             }
 176         }
 177
 178         count(self.tree.getroot(), data['self'])
 179         for k, v in data['self'].items():
 180             data['total'][k] = v
 181
 182         for part in self.children:
 183             if isinstance(part, Exception):
 184                 data['parts'].append((None, {'error': part}))
 185             else:
 186                 data['parts'].append((part, part.get_statistics()))
 187                 for k, v in data['parts'][-1][1]['total'].items():
 188                     data['total'][k] = data['total'].get(k, 0) + v
 189
 190         return data