X-Git-Url: https://git.mdrn.pl/librarian.git/blobdiff_plain/fefdce4e24f9e397df5538fe6e7f54b5ece4d841..f164694b5e7ad5ed5f6d95743f9259bd3a9292bd:/src/librarian/dcparser.py?ds=sidebyside diff --git a/src/librarian/dcparser.py b/src/librarian/dcparser.py index 92afc01..ce03be2 100644 --- a/src/librarian/dcparser.py +++ b/src/librarian/dcparser.py @@ -1,189 +1,63 @@ -# -*- coding: utf-8 -*- -# # This file is part of Librarian, licensed under GNU Affero GPLv3 or later. -# Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information. +# Copyright © Fundacja Wolne Lektury. See NOTICE for more information. # -from __future__ import unicode_literals - from xml.parsers.expat import ExpatError from datetime import date -from functools import total_ordering +import io import time import re -import six from librarian.util import roman_to_int from librarian import (ValidationError, NoDublinCore, ParseError, DCNS, RDFNS, - XMLNS, WLURI, WLNS, PLMETNS) + XMLNS, WLNS, PLMETNS) -import lxml.etree as etree # ElementTree API using libxml2 +import lxml.etree as etree from lxml.etree import XMLSyntaxError +from librarian.meta.types.bool import BoolValue +from librarian.meta.types.person import Person +from librarian.meta.types.wluri import WLURI +from librarian.meta.types import text -class TextPlus(six.text_type): - pass - - -class DatePlus(date): - pass - - -# ============== -# = Converters = -# ============== -@six.python_2_unicode_compatible -@total_ordering -class Person(object): - """Single person with last name and a list of first names.""" - def __init__(self, last_name, *first_names): - self.last_name = last_name - self.first_names = first_names - - @classmethod - def from_text(cls, text): - parts = [token.strip() for token in text.split(',')] - if len(parts) == 1: - surname = parts[0] - names = [] - elif len(parts) != 2: - raise ValueError("Invalid person name. There should be at most one comma: \"%s\"." % text.encode('utf-8')) - else: - surname = parts[0] - if len(parts[1]) == 0: - # there is no non-whitespace data after the comma - raise ValueError("Found a comma, but no names given: \"%s\" -> %r." % (text, parts)) - names = parts[1].split() - return cls(surname, *names) - - def readable(self): - return u" ".join(self.first_names + (self.last_name,)) - - def __eq__(self, right): - return self.last_name == right.last_name and self.first_names == right.first_names - - def __lt__(self, other): - return (self.last_name, self.first_names) < (other.last_name, other.first_names) - - def __hash__(self): - return hash((self.last_name, self.first_names)) - - def __str__(self): - if len(self.first_names) > 0: - return '%s, %s' % (self.last_name, ' '.join(self.first_names)) - else: - return self.last_name - - def __repr__(self): - return 'Person(last_name=%r, first_names=*%r)' % (self.last_name, self.first_names) - - -def as_date(text): - """Dates for digitization of pictures. It seems we need the following: -ranges: '1350-1450', -centuries: "XVIII w.' -half centuries/decades: '2 poł. XVIII w.', 'XVII w., l. 20' -later-then: 'po 1450' -circa 'ok. 1813-1814', 'ok.1876-ok.1886 -turn: 1893/1894 -for now we will translate this to some single date losing information of course. - """ - try: - # check out the "N. poł X w." syntax - if isinstance(text, six.binary_type): - text = text.decode("utf-8") - - century_format = u"(?:([12]) *poł[.]? +)?([MCDXVI]+) *w[.,]*(?: *l[.]? *([0-9]+))?" - vague_format = u"(?:po *|ok. *)?([0-9]{4})(-[0-9]{2}-[0-9]{2})?" - - m = re.match(century_format, text) - m2 = re.match(vague_format, text) - if m: - half = m.group(1) - decade = m.group(3) - century = roman_to_int(m.group(2)) - if half is not None: - if decade is not None: - raise ValueError("Bad date format. Cannot specify both half and decade of century") - half = int(half) - t = ((century*100 + (half-1)*50), 1, 1) - else: - decade = int(decade or 0) - t = ((century*100 + decade), 1, 1) - elif m2: - year = m2.group(1) - mon_day = m2.group(2) - if mon_day: - t = time.strptime(year + mon_day, "%Y-%m-%d") - else: - t = time.strptime(year, '%Y') - else: - raise ValueError - - return DatePlus(t[0], t[1], t[2]) - except ValueError as e: - raise ValueError("Unrecognized date format. Try YYYY-MM-DD or YYYY.") - -def as_person(text): - return Person.from_text(text) - - -def as_unicode(text): - if isinstance(text, six.text_type): - return text - else: - return TextPlus(text.decode('utf-8')) - - -def as_wluri_strict(text): - return WLURI.strict(text) - - -class Field(object): - def __init__(self, uri, attr_name, validator=as_unicode, strict=None, multiple=False, salias=None, **kwargs): +class Field: + def __init__(self, uri, attr_name, value_type=text.TextValue, + multiple=False, salias=None, **kwargs): self.uri = uri self.name = attr_name - self.validator = validator - self.strict = strict + self.value_type = value_type self.multiple = multiple self.salias = salias - self.required = kwargs.get('required', True) and 'default' not in kwargs + self.required = (kwargs.get('required', True) + and 'default' not in kwargs) self.default = kwargs.get('default', [] if multiple else [None]) def validate_value(self, val, strict=False): - if strict and self.strict is not None: - validator = self.strict - else: - validator = self.validator + #if strict: + # value.validate() + try: if self.multiple: - if validator is None: - return val - new_values = [] - for v in val: - nv = v - if v is not None: - nv = validator(v) - if hasattr(v, 'lang'): - setattr(nv, 'lang', v.lang) - new_values.append(nv) - return new_values + return val elif len(val) > 1: - raise ValidationError("Multiple values not allowed for field '%s'" % self.uri) + raise ValidationError( + "Multiple values not allowed for field '%s'" % self.uri + ) elif len(val) == 0: - raise ValidationError("Field %s has no value to assign. Check your defaults." % self.uri) + raise ValidationError( + "Field %s has no value to assign. Check your defaults." + % self.uri + ) else: - if validator is None or val[0] is None: - return val[0] - nv = validator(val[0]) - if hasattr(val[0], 'lang'): - setattr(nv, 'lang', val[0].lang) - return nv + return val[0] except ValueError as e: - raise ValidationError("Field '%s' - invald value: %s" % (self.uri, e.message)) + raise ValidationError( + "Field '%s' - invald value: %s" + % (self.uri, str(e)) + ) - def validate(self, fdict, fallbacks=None, strict=False): + def validate(self, fdict, fallbacks=None, strict=False, validate_required=True): if fallbacks is None: fallbacks = {} if self.uri not in fdict: @@ -198,8 +72,10 @@ class Field(object): f = [fallbacks[self.salias]] else: f = self.default - else: + elif validate_required: raise ValidationError("Required field %s not found" % self.uri) + else: + return None else: f = fdict[self.uri] @@ -211,56 +87,96 @@ class Field(object): return False -class DCInfo(type): - def __new__(mcs, classname, bases, class_dict): - fields = list(class_dict['FIELDS']) - - for base in bases[::-1]: - if hasattr(base, 'FIELDS'): - for field in base.FIELDS[::-1]: - try: - fields.index(field) - except ValueError: - fields.insert(0, field) - - class_dict['FIELDS'] = tuple(fields) - return super(DCInfo, mcs).__new__(mcs, classname, bases, class_dict) - - -class WorkInfo(six.with_metaclass(DCInfo, object)): +class BookInfo: FIELDS = ( - Field(DCNS('creator'), 'authors', as_person, salias='author', multiple=True), + Field(DCNS('creator'), 'authors', Person, salias='author', + multiple=True), Field(DCNS('title'), 'title'), Field(DCNS('type'), 'type', required=False, multiple=True), Field(DCNS('contributor.editor'), 'editors', - as_person, salias='editor', multiple=True, required=False), + Person, salias='editor', multiple=True, required=False), Field(DCNS('contributor.technical_editor'), 'technical_editors', - as_person, salias='technical_editor', multiple=True, required=False), - Field(DCNS('contributor.funding'), 'funders', salias='funder', multiple=True, required=False), + Person, salias='technical_editor', multiple=True, + required=False), + Field(DCNS('contributor.funding'), 'funders', salias='funder', + multiple=True, required=False), Field(DCNS('contributor.thanks'), 'thanks', required=False), Field(DCNS('date'), 'created_at'), - Field(DCNS('date.pd'), 'released_to_public_domain_at', as_date, required=False), + Field(DCNS('date.pd'), 'released_to_public_domain_at', + required=False), Field(DCNS('publisher'), 'publisher', multiple=True), Field(DCNS('language'), 'language'), Field(DCNS('description'), 'description', required=False), Field(DCNS('source'), 'source_name', required=False), - Field(DCNS('source.URL'), 'source_urls', salias='source_url', multiple=True, required=False), - Field(DCNS('identifier.url'), 'url', WLURI, strict=as_wluri_strict), + Field(DCNS('source.URL'), 'source_urls', salias='source_url', + multiple=True, required=False), + Field(DCNS('identifier.url'), 'url', WLURI), Field(DCNS('rights.license'), 'license', required=False), Field(DCNS('rights'), 'license_description'), - Field(PLMETNS('digitisationSponsor'), 'sponsors', multiple=True, required=False), + Field(PLMETNS('digitisationSponsor'), 'sponsors', multiple=True, + required=False), Field(WLNS('digitisationSponsorNote'), 'sponsor_note', required=False), + Field(WLNS('contentWarning'), 'content_warnings', multiple=True, + required=False), Field(WLNS('developmentStage'), 'stage', required=False), + + Field(DCNS('audience'), 'audiences', text.Audience, salias='audience', multiple=True, + required=False), + + Field(DCNS('subject.period'), 'epochs', text.Epoch, salias='epoch', multiple=True, + required=False), + Field(DCNS('subject.type'), 'kinds', text.Kind, salias='kind', multiple=True, + required=False), + Field(DCNS('subject.genre'), 'genres', text.Genre, salias='genre', multiple=True, + required=False), + Field('category.legimi', 'legimi', text.LegimiCategory, required=False), + Field('category.thema.main', 'thema_main', text.MainThemaCategory, required=False), + Field('category.thema', 'thema', text.ThemaCategory, required=False, multiple=True), + Field(DCNS('subject.location'), 'location', required=False), + + Field(DCNS('contributor.translator'), 'translators', + Person, salias='translator', multiple=True, required=False), + Field(DCNS('relation.hasPart'), 'parts', WLURI, + multiple=True, required=False), + Field(DCNS('relation.isVariantOf'), 'variant_of', WLURI, + required=False), + + Field(DCNS('relation.coverImage.url'), 'cover_url', required=False), + Field(DCNS('relation.coverImage.attribution'), 'cover_by', + required=False), + Field(DCNS('relation.coverImage.source'), 'cover_source', + required=False), + # WLCover-specific. + Field(WLNS('coverBarColor'), 'cover_bar_color', required=False), + Field(WLNS('coverBoxPosition'), 'cover_box_position', required=False), + Field(WLNS('coverClass'), 'cover_class', default=['default']), + Field(WLNS('coverLogoUrl'), 'cover_logo_urls', multiple=True, + required=False), + Field(WLNS('endnotes'), 'endnotes', BoolValue, + required=False), + + Field('pdf-id', 'isbn_pdf', required=False), + Field('epub-id', 'isbn_epub', required=False), + Field('mobi-id', 'isbn_mobi', required=False), + Field('txt-id', 'isbn_txt', required=False), + Field('html-id', 'isbn_html', required=False), + ) + @classmethod + def get_field_by_uri(cls, uri): + for f in cls.FIELDS: + if f.uri == uri: + return f + @classmethod def from_bytes(cls, xml, *args, **kwargs): - return cls.from_file(six.BytesIO(xml), *args, **kwargs) + return cls.from_file(io.BytesIO(xml), *args, **kwargs) @classmethod def from_file(cls, xmlfile, *args, **kwargs): @@ -292,12 +208,16 @@ class WorkInfo(six.with_metaclass(DCInfo, object)): @classmethod def from_element(cls, rdf_tag, *args, **kwargs): - # the tree is already parsed, so we don't need to worry about Expat errors + # The tree is already parsed, + # so we don't need to worry about Expat errors. field_dict = {} desc = rdf_tag.find(".//" + RDFNS('Description')) if desc is None: - raise NoDublinCore("There must be a '%s' element inside the RDF." % RDFNS('Description')) + raise NoDublinCore( + "There must be a '%s' element inside the RDF." + % RDFNS('Description') + ) lang = None p = desc @@ -306,34 +226,42 @@ class WorkInfo(six.with_metaclass(DCInfo, object)): p = p.getparent() for e in desc.getchildren(): - fv = field_dict.get(e.tag, []) + tag = e.tag + if tag == 'meta': + meta_id = e.attrib.get('id') + if meta_id and meta_id.endswith('-id'): + tag = meta_id + + field = cls.get_field_by_uri(tag) + if field is None: + # Ignore unknown fields. + continue + + fv = field_dict.get(tag, []) if e.text is not None: - text = e.text - if not isinstance(text, six.text_type): - text = text.decode('utf-8') - val = TextPlus(text) + val = field.value_type.from_text(e.text) val.lang = e.attrib.get(XMLNS('lang'), lang) - if e.tag == 'meta': - meta_id = e.attrib.get('id') - if meta_id and meta_id.endswith('-id'): - field_dict[meta_id] = [val.replace('ISBN-', 'ISBN ')] else: val = e.text fv.append(val) - field_dict[e.tag] = fv + field_dict[tag] = fv return cls(desc.attrib, field_dict, *args, **kwargs) - def __init__(self, rdf_attrs, dc_fields, fallbacks=None, strict=False): - """rdf_attrs should be a dictionary-like object with any attributes of the RDF:Description. - dc_fields - dictionary mapping DC fields (with namespace) to list of text values for the - given field. """ + def __init__(self, rdf_attrs, dc_fields, fallbacks=None, strict=False, validate_required=True): + """ + rdf_attrs should be a dictionary-like object with any attributes + of the RDF:Description. + dc_fields - dictionary mapping DC fields (with namespace) to + list of text values for the given field. + """ self.about = rdf_attrs.get(RDFNS('about')) self.fmap = {} for field in self.FIELDS: - value = field.validate(dc_fields, fallbacks=fallbacks, strict=strict) + value = field.validate(dc_fields, fallbacks=fallbacks, + strict=strict, validate_required=validate_required) setattr(self, 'prop_' + field.name, value) self.fmap[field.name] = field if field.salias: @@ -367,8 +295,10 @@ class WorkInfo(six.with_metaclass(DCInfo, object)): return object.__setattr__(self, name, newvalue) def update(self, field_dict): - """Update using field_dict. Verify correctness, but don't check if all - required fields are present.""" + """ + Update using field_dict. Verify correctness, but don't check + if all required fields are present. + """ for field in self.FIELDS: if field.name in field_dict: setattr(self, field.name, field_dict[field.name]) @@ -397,11 +327,11 @@ class WorkInfo(six.with_metaclass(DCInfo, object)): for x in v: e = etree.Element(field.uri) if x is not None: - e.text = six.text_type(x) + e.text = str(x) description.append(e) else: e = etree.Element(field.uri) - e.text = six.text_type(v) + e.text = str(v) description.append(e) return root @@ -416,9 +346,9 @@ class WorkInfo(six.with_metaclass(DCInfo, object)): if field.multiple: if len(v) == 0: continue - v = [six.text_type(x) for x in v if x is not None] + v = [str(x) for x in v if x is not None] else: - v = six.text_type(v) + v = str(v) dc[field.name] = {'uri': field.uri, 'value': v} rdf['fields'] = dc @@ -433,50 +363,18 @@ class WorkInfo(six.with_metaclass(DCInfo, object)): if field.multiple: if len(v) == 0: continue - v = [six.text_type(x) for x in v if x is not None] + v = [str(x) for x in v if x is not None] else: - v = six.text_type(v) + v = str(v) result[field.name] = v if field.salias: v = getattr(self, field.salias) if v is not None: - result[field.salias] = six.text_type(v) + result[field.salias] = str(v) return result -class BookInfo(WorkInfo): - FIELDS = ( - Field(DCNS('audience'), 'audiences', salias='audience', multiple=True, required=False), - - Field(DCNS('subject.period'), 'epochs', salias='epoch', multiple=True, required=False), - Field(DCNS('subject.type'), 'kinds', salias='kind', multiple=True, required=False), - Field(DCNS('subject.genre'), 'genres', salias='genre', multiple=True, required=False), - - Field(DCNS('subject.location'), 'location', required=False), - - Field(DCNS('contributor.translator'), 'translators', - as_person, salias='translator', multiple=True, required=False), - Field(DCNS('relation.hasPart'), 'parts', WLURI, strict=as_wluri_strict, multiple=True, required=False), - Field(DCNS('relation.isVariantOf'), 'variant_of', WLURI, strict=as_wluri_strict, required=False), - - Field(DCNS('relation.coverImage.url'), 'cover_url', required=False), - Field(DCNS('relation.coverImage.attribution'), 'cover_by', required=False), - Field(DCNS('relation.coverImage.source'), 'cover_source', required=False), - # WLCover-specific. - Field(WLNS('coverBarColor'), 'cover_bar_color', required=False), - Field(WLNS('coverBoxPosition'), 'cover_box_position', required=False), - Field(WLNS('coverClass'), 'cover_class', default=['default']), - Field(WLNS('coverLogoUrl'), 'cover_logo_urls', multiple=True, required=False), - - Field('pdf-id', 'isbn_pdf', required=False), - Field('epub-id', 'isbn_epub', required=False), - Field('mobi-id', 'isbn_mobi', required=False), - Field('txt-id', 'isbn_txt', required=False), - Field('html-id', 'isbn_html', required=False), - ) - - def parse(file_name, cls=BookInfo): return cls.from_file(file_name)