src/librarian/document.py

   1 # This file is part of Librarian, licensed under GNU Affero GPLv3 or later.
   2 # Copyright © Fundacja Wolne Lektury. See NOTICE for more information.
   3 #
   4 from collections import Counter
   5 import gettext
   6 import os
   7 import re
   8 import urllib.request
   9 from lxml import etree
  10 from .parser import parser
  11 from . import dcparser, DCNS, RDFNS, DirDocProvider
  12 from .functions import lang_code_3to2
  13
  14
  15 class WLDocument:
  16     def __init__(self, filename=None, url=None, provider=None):
  17         source = filename or urllib.request.urlopen(url)
  18         tree = etree.parse(source, parser=parser)
  19         self.tree = tree
  20         tree.getroot().document = self
  21
  22         self.preprocess()
  23
  24         self.base_meta = dcparser.BookInfo({}, {
  25             DCNS('language'): ["pol"],
  26         }, validate_required=False)
  27
  28         self.provider = provider if provider is not None else DirDocProvider('.')
  29
  30         self.tree.getroot().validate()
  31
  32     @property
  33     def meta(self):
  34         # Allow metadata of the master element as document meta.
  35         #master = self.tree.getroot()[-1]
  36         return self.tree.getroot().meta
  37         return master.meta
  38
  39     def preprocess(self):
  40         # Change slash-verses into real verses.
  41         for _e, elem in etree.iterwalk(self.tree, ('start',), 'strofa'):
  42             elem.preprocess()
  43
  44     @property
  45     def children(self):
  46         for part_uri in self.meta.parts or []:
  47             with self.provider.by_slug(part_uri.slug) as f:
  48                 try:
  49                     yield type(self)(filename=f, provider=self.provider)
  50                 except Exception as e:
  51
  52                     yield e
  53
  54     def build(self, builder, base_url=None, **kwargs):
  55         return builder(base_url=base_url).build(self, **kwargs)
  56
  57     def editors(self):
  58         persons = set(self.meta.editors
  59                       + self.meta.technical_editors)
  60         for child in self.children:
  61             persons.update(child.editors())
  62         if None in persons:
  63             persons.remove(None)
  64         return persons
  65
  66     def references(self):
  67         return self.tree.findall('.//ref')
  68
  69     def get_statistics(self):
  70         def count_text(text, counter, in_fn=False, stanza=False):
  71             if text:
  72                 text = re.sub(r'\s+', ' ', text)
  73
  74                 chars = len(text) if text.strip() else 0
  75                 words = len(text.split()) if text.strip() else 0
  76
  77                 counter['chars_with_fn'] += chars
  78                 counter['words_with_fn'] += words
  79                 if not in_fn:
  80                     counter['chars'] += chars
  81                     counter['words'] += words
  82                 if not stanza:
  83                     counter['chars_out_verse_with_fn'] += chars
  84                     if not in_fn:
  85                         counter['chars_out_verse'] += chars
  86
  87         def count(elem, counter, in_fn=False, stanza=False):
  88             if elem.tag in (RDFNS('RDF'), 'nota_red', 'abstrakt', 'uwaga', 'ekstra'):
  89                 return
  90             if not in_fn and elem.tag in ('pa', 'pe', 'pr', 'pt', 'motyw'):
  91                 in_fn = True
  92             if elem.tag == 'strofa':
  93                 # count verses now
  94                 #verses = len(elem.findall('.//br')) + 1
  95                 verses = list(elem.get_verses())
  96                 counter['verses_with_fn'] += len(verses)
  97                 if not in_fn:
  98                     counter['verses'] += len(verses)
  99                 stanza = True
 100
 101                 for child in verses:
 102                     count(child, counter, in_fn=in_fn, stanza=True)
 103             else:
 104                 count_text(elem.text, counter, in_fn=in_fn, stanza=stanza)
 105                 for child in elem:
 106                     count(child, counter, in_fn=in_fn, stanza=stanza)
 107                     count_text(child.tail, counter, in_fn=in_fn, stanza=stanza)
 108
 109         data = {
 110             "self": Counter(),
 111             "parts": [],
 112             "total": {
 113             }
 114         }
 115
 116         count(self.tree.getroot(), data['self'])
 117         for k, v in data['self'].items():
 118             data['total'][k] = v
 119
 120         for part in self.children:
 121             if isinstance(part, Exception):
 122                 data['parts'].append((None, {'error': part}))
 123             else:
 124                 data['parts'].append((part, part.get_statistics()))
 125                 for k, v in data['parts'][-1][1]['total'].items():
 126                     data['total'][k] = data['total'].get(k, 0) + v
 127
 128         return data