src/librarian/document.py

   1 # This file is part of Librarian, licensed under GNU Affero GPLv3 or later.
   2 # Copyright © Fundacja Wolne Lektury. See NOTICE for more information.
   3 #
   4 from collections import defaultdict, Counter
   5 import gettext
   6 import os
   7 import re
   8 import urllib.request
   9 from lxml import etree
  10 from .parser import parser
  11 from . import dcparser, DCNS, RDFNS, DirDocProvider
  12 from .functions import lang_code_3to2
  13
  14
  15 class WLDocument:
  16     def __init__(self, filename=None, url=None, provider=None):
  17         source = filename or urllib.request.urlopen(url)
  18         tree = etree.parse(source, parser=parser)
  19         self.tree = tree
  20         self.counters = defaultdict(lambda: 1)
  21         tree.getroot().document = self
  22
  23         self.preprocess()
  24
  25         self.base_meta = dcparser.BookInfo({}, {
  26             DCNS('language'): ["pol"],
  27         }, validate_required=False)
  28
  29         self.provider = provider if provider is not None else DirDocProvider('.')
  30
  31         self.tree.getroot().validate()
  32
  33     @property
  34     def meta(self):
  35         # Allow metadata of the master element as document meta.
  36         #master = self.tree.getroot()[-1]
  37         return self.tree.getroot().meta
  38         return master.meta
  39
  40     def preprocess(self):
  41         # Change slash-verses into real verses.
  42         for _e, elem in etree.iterwalk(self.tree, ('start',), 'strofa'):
  43             elem.preprocess()
  44
  45     def assign_ids(self):
  46         # Assign IDs depth-first, to account for any <numeracja> inside.
  47         for _e, elem in etree.iterwalk(self.tree, events=('end',)):
  48             if getattr(elem, 'NUMBERING', None):
  49                 elem.assign_id(self)
  50
  51     @property
  52     def children(self):
  53         for part_uri in self.meta.parts or []:
  54             with self.provider.by_slug(part_uri.slug) as f:
  55                 try:
  56                     yield type(self)(filename=f, provider=self.provider)
  57                 except Exception as e:
  58
  59                     yield e
  60
  61     def build(self, builder, base_url=None, **kwargs):
  62         return builder(base_url=base_url).build(self, **kwargs)
  63
  64     def editors(self):
  65         persons = set(self.meta.editors
  66                       + self.meta.technical_editors)
  67         for child in self.children:
  68             persons.update(child.editors())
  69         if None in persons:
  70             persons.remove(None)
  71         return persons
  72
  73     def references(self):
  74         return self.tree.findall('.//ref')
  75
  76     def get_statistics(self):
  77         def count_text(text, counter, in_fn=False, stanza=False):
  78             if text:
  79                 text = re.sub(r'\s+', ' ', text)
  80
  81                 chars = len(text) if text.strip() else 0
  82                 words = len(text.split()) if text.strip() else 0
  83
  84                 counter['chars_with_fn'] += chars
  85                 counter['words_with_fn'] += words
  86                 if not in_fn:
  87                     counter['chars'] += chars
  88                     counter['words'] += words
  89                 if not stanza:
  90                     counter['chars_out_verse_with_fn'] += chars
  91                     if not in_fn:
  92                         counter['chars_out_verse'] += chars
  93
  94         def count(elem, counter, in_fn=False, stanza=False):
  95             if elem.tag in (RDFNS('RDF'), 'nota_red', 'abstrakt', 'uwaga', 'ekstra'):
  96                 return
  97             if not in_fn and elem.tag in ('pa', 'pe', 'pr', 'pt', 'motyw'):
  98                 in_fn = True
  99             if elem.tag == 'strofa':
 100                 # count verses now
 101                 #verses = len(elem.findall('.//br')) + 1
 102                 verses = list(elem.get_verses())
 103                 counter['verses_with_fn'] += len(verses)
 104                 if not in_fn:
 105                     counter['verses'] += len(verses)
 106                 stanza = True
 107
 108                 for child in verses:
 109                     count(child, counter, in_fn=in_fn, stanza=True)
 110             else:
 111                 count_text(elem.text, counter, in_fn=in_fn, stanza=stanza)
 112                 for child in elem:
 113                     count(child, counter, in_fn=in_fn, stanza=stanza)
 114                     count_text(child.tail, counter, in_fn=in_fn, stanza=stanza)
 115
 116         data = {
 117             "self": Counter(),
 118             "parts": [],
 119             "total": {
 120             }
 121         }
 122
 123         count(self.tree.getroot(), data['self'])
 124         for k, v in data['self'].items():
 125             data['total'][k] = v
 126
 127         for part in self.children:
 128             if isinstance(part, Exception):
 129                 data['parts'].append((None, {'error': part}))
 130             else:
 131                 data['parts'].append((part, part.get_statistics()))
 132                 for k, v in data['parts'][-1][1]['total'].items():
 133                     data['total'][k] = data['total'].get(k, 0) + v
 134
 135         return data