-from functools import wraps
+# -*- coding: utf-8 -*-
+from copy import deepcopy
import re
+from django.utils.encoding import force_str
from lxml import etree
from catalogue.constants import TRIM_BEGIN, TRIM_END, MASTERS
pass
-def obj_memoized(f):
- """
- A decorator that caches return value of object methods.
- The cache is kept with the object, in a _obj_memoized property.
- """
- @wraps(f)
- def wrapper(self, *args, **kwargs):
- if not hasattr(self, '_obj_memoized'):
- self._obj_memoized = {}
- key = (f.__name__,) + args + tuple(sorted(kwargs.iteritems()))
- try:
- return self._obj_memoized[key]
- except TypeError:
- return f(self, *args, **kwargs)
- except KeyError:
- self._obj_memoized[key] = f(self, *args, **kwargs)
- return self._obj_memoized[key]
- return wrapper
-
-
-class GradedText(object):
- _edoc = None
-
- ROOT = 'utwor'
- RDF = '{http://www.w3.org/1999/02/22-rdf-syntax-ns#}RDF'
-
- def __init__(self, text):
- self._text = text
-
- @obj_memoized
- def is_xml(self):
- """
- Determines if it's a well-formed XML.
-
- >>> GradedText("<a/>").is_xml()
- True
- >>> GradedText("<a>").is_xml()
- False
- """
- try:
- self._edoc = etree.fromstring(self._text)
- except etree.XMLSyntaxError:
- return False
- return True
-
- @obj_memoized
- def is_wl(self):
- """
- Determines if it's an XML with a <utwor> and a master tag.
-
- >>> GradedText("<utwor><powiesc></powiesc></utwor>").is_wl()
- True
- >>> GradedText("<a></a>").is_wl()
- False
- """
- if self.is_xml():
- e = self._edoc
- # FIXME: there could be comments
- ret = e.tag == self.ROOT and (
- len(e) == 1 and e[0].tag in MASTERS or
- len(e) == 2 and e[0].tag == self.RDF
- and e[1].tag in MASTERS)
- if ret:
- self._master = e[-1].tag
- del self._edoc
- return ret
- else:
- return False
-
- @obj_memoized
- def is_broken_wl(self):
- """
- Determines if it at least looks like broken WL file
- and not just some untagged text.
-
- >>> GradedText("<utwor><</utwor>").is_broken_wl()
- True
- >>> GradedText("some text").is_broken_wl()
- False
- """
- if self.is_wl():
- return True
- text = self._text.strip()
- return text.startswith('<utwor>') and text.endswith('</utwor>')
-
- def master(self):
- """
- Gets the master tag.
-
- >>> GradedText("<utwor><powiesc></powiesc></utwor>").master()
- 'powiesc'
- """
- assert self.is_wl()
- return self._master
-
- @obj_memoized
- def has_trim_begin(self):
- return RE_TRIM_BEGIN.search(self._text)
-
- @obj_memoized
- def has_trim_end(self):
- return RE_TRIM_END.search(self._text)
-
-
def _trim(text, trim_begin=True, trim_end=True):
- """
+ """
Cut off everything before RE_TRIM_BEGIN and after RE_TRIM_END, so
that eg. one big XML file can be compiled from many small XML files.
"""
def compile_text(parts):
- """
+ """
Compiles full text from an iterable of parts,
trimming where applicable.
"""
for next_text in parts:
if not next_text:
continue
- # trim the end, because there's more non-empty text
- # don't trim beginning, if `text' is the first non-empty part
- texts.append(_trim(text, trim_begin=trim_begin))
- trim_begin = True
+ if text:
+ # trim the end, because there's more non-empty text
+ # don't trim beginning, if `text' is the first non-empty part
+ texts.append(_trim(text, trim_begin=trim_begin))
+ trim_begin = True
text = next_text
# don't trim the end, because there's no more text coming after `text'
# only trim beginning if it's not still the first non-empty
return "".join(texts)
-def change_master(text, master):
- """
- Changes the master tag in a WL document.
- """
- e = etree.fromstring(text)
- e[-1].tag = master
- return etree.tostring(e, encoding="utf-8")
-
-
-def basic_structure(text, master):
- e = etree.fromstring('''<utwor>
-<master>
-<!--%s--><!--%s-->
-</master>
-</utwor>''' % (TRIM_BEGIN, TRIM_END))
- e[0].tag = master
- e[0][0].tail = "\n"*3 + text + "\n"*3
- return etree.tostring(e, encoding="utf-8")
-
-
def add_trim_begin(text):
trim_tag = etree.Comment(TRIM_BEGIN)
e = etree.fromstring(text)
master.insert(0, trim_tag)
trim_tag.tail = '\n\n\n' + (master.text or '')
master.text = '\n'
- return etree.tostring(e, encoding="utf-8")
+ return unicode(etree.tostring(e, encoding="utf-8"), 'utf-8')
def add_trim_end(text):
prev.tail = (prev.tail or '') + '\n\n\n'
else:
master.text = (master.text or '') + '\n\n\n'
- return etree.tostring(e, encoding="utf-8")
+ return unicode(etree.tostring(e, encoding="utf-8"), 'utf-8')
+
+
+def split_xml(text):
+ """Splits text into chapters.
+
+ All this stuff really must go somewhere else.
+
+ """
+ src = etree.fromstring(text)
+ chunks = []
+
+ splitter = u'naglowek_rozdzial'
+ parts = src.findall('.//naglowek_rozdzial')
+ while parts:
+ # copy the document
+ copied = deepcopy(src)
+
+ element = parts[-1]
+
+ # find the chapter's title
+ name_elem = deepcopy(element)
+ for tag in 'extra', 'motyw', 'pa', 'pe', 'pr', 'pt', 'uwaga':
+ for a in name_elem.findall('.//' + tag):
+ a.text = ''
+ del a[:]
+ name = etree.tostring(name_elem, method='text', encoding='utf-8').strip()
+
+ # in the original, remove everything from the start of the last chapter
+ parent = element.getparent()
+ del parent[parent.index(element):]
+ element, parent = parent, parent.getparent()
+ while parent is not None:
+ del parent[parent.index(element) + 1:]
+ element, parent = parent, parent.getparent()
+
+ # in the copy, remove everything before the last chapter
+ element = copied.findall('.//naglowek_rozdzial')[-1]
+ parent = element.getparent()
+ while parent is not None:
+ parent.text = None
+ while parent[0] is not element:
+ del parent[0]
+ element, parent = parent, parent.getparent()
+ chunks[:0] = [[name, unicode(etree.tostring(copied, encoding='utf-8'), 'utf-8')]]
+
+ parts = src.findall('.//naglowek_rozdzial')
+
+ chunks[:0] = [[u'początek', unicode(etree.tostring(src, encoding='utf-8'), 'utf-8')]]
+
+ for ch in chunks[1:]:
+ ch[1] = add_trim_begin(ch[1])
+ for ch in chunks[:-1]:
+ ch[1] = add_trim_end(ch[1])
+
+ return chunks
+
+
+def wl2_to_wl1(wl2_xml, slug):
+ from lxml import etree
+ import re
+ from StringIO import StringIO
+ from urllib import unquote
+ import os.path
+ from django.conf import settings
+ from fnpdjango.utils.text.slughifi import slughifi
+ from librarian import ParseError, DCNS
+
+ def _register_function(f):
+ """ Register extension function with lxml """
+ ns = etree.FunctionNamespace('http://wolnelektury.pl/functions')
+ ns[f.__name__] = f
+ return f
+
+ @_register_function
+ def slugify(context, text):
+ """Remove unneeded whitespace from beginning and end"""
+ if isinstance(text, list):
+ text = ''.join(text)
+ return slughifi(text)
+
+ @_register_function
+ def rmext(context, text):
+ if isinstance(text, list):
+ text = ''.join(text)
+ text = unquote(text)
+ if '.' in text:
+ name, ext = text.rsplit('.', 1)
+ if ext.lower() in ('doc', 'docx', 'odt', 'pdf', 'jpg', 'jpeg'):
+ text = name
+ return text
+
+ t = etree.parse(os.path.join(settings.PROJECT_ROOT, 'xslt/wl2to1.xslt'))
+ ft = wl2_xml.replace(' ', ' ')
+ f2 = StringIO(ft)
+ i1 = etree.parse(f2)
+
+ for sect in i1.findall('//section'):
+ if sect[0].text and sect[0].text.strip() == u'Przebieg zajęć':
+ # Prostujemy.
+ first = sect.find('section')
+ subs = first.findall('.//section')
+ for sub in subs:
+ sect.append(sub)
+ break
+ else:
+ dc_type = i1.findall('//dc:type', namespaces={'dc': DCNS.uri})
+ if dc_type and dc_type[0] in ('course', 'synthetic'):
+ raise ParseError('Brak przebiegu')
+
+ i1.getroot().attrib['redslug'] = slug
+ i1.getroot().attrib['wlslug'] = slug # THIS!
+ w1t = i1.xslt(t)
+ for h in w1t.findall('//aktywnosc/opis'):
+ if len(h) == 0:
+ raise ParseError('Pusty element aktywnosc/opis')
+ # FIXME assumption that every lesson has at most 9 parts
+ if not h[0].text or not re.match(r'\d\.\s', h[0].text):
+ raise ParseError('Niepoprawny nagłówek (aktywnosc/opis): %s' % repr(h[0].text))
+ h[0].text = h[0].text[3:]
+ return etree.tostring(w1t, encoding='utf-8')
+
+
+EXCEPTIONS = [
+ ('div', 'img'),
+ ('div', 'video'),
+ ('div', 'table.cell'),
+ ('span', 'link'),
+]
+
+
+def remove_element(element):
+ parent = element.getparent()
+ tail = element.tail
+ if tail:
+ prev = element.getprevious()
+ if prev is not None:
+ prev.tail = (prev.tail or '') + tail
+ else:
+ parent.text = (parent.text or '') + tail
+ parent.remove(element)
+
+
+def remove_empty_elements(xml):
+ try:
+ tree = etree.fromstring(force_str(xml.replace(' ', u'\xa0')))
+ except SyntaxError:
+ return None
+ changed = False
+ another_loop = True
+ while another_loop:
+ another_loop = False
+ for element in tree.findall('.//*'):
+ if (not element.text or not element.text.strip()) and len(element) == 0:
+ if (element.tag, element.attrib.get('class')) not in EXCEPTIONS:
+ remove_element(element)
+ changed = True
+ another_loop = True
+ return etree.tostring(tree, encoding=unicode) if changed else None
fnp / redakcja.git / blobdiff