1 # -*- coding: utf-8 -*-
3 # This file is part of Librarian, licensed under GNU Affero GPLv3 or later.
4 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
6 from __future__ import with_statement
12 from StringIO import StringIO
13 from copy import deepcopy
14 from lxml import etree
16 from tempfile import mkdtemp, NamedTemporaryFile
17 from shutil import rmtree
18 from mimetypes import guess_type
20 from librarian import RDFNS, WLNS, NCXNS, OPFNS, XHTMLNS, OutputFile
21 from librarian.cover import WLCover, FutureOfCopyrightCover
23 from librarian import functions, get_resource
25 functions.reg_person_name()
29 """ returns node's text and children as a string
31 >>> print inner_xml(etree.fromstring('<a>x<b>y</b>z</a>'))
35 nt = node.text if node.text is not None else ''
36 return ''.join([nt] + [etree.tostring(child) for child in node])
38 def set_inner_xml(node, text):
39 """ sets node's text and children from a string
41 >>> e = etree.fromstring('<a>b<b>x</b>x</a>')
42 >>> set_inner_xml(e, 'x<b>y</b>z')
43 >>> print etree.tostring(e)
47 p = etree.fromstring('<x>%s</x>' % text)
53 """ Find out a node's name
55 >>> print node_name(etree.fromstring('<a>X<b>Y</b>Z</a>'))
59 tempnode = deepcopy(node)
61 for p in ('pe', 'pa', 'pt', 'pr', 'motyw'):
62 for e in tempnode.findall('.//%s' % p):
66 etree.strip_tags(tempnode, '*')
71 if isinstance(xml, etree._Element):
72 xml = etree.ElementTree(xml)
73 with open(sheet) as xsltf:
74 return xml.xslt(etree.parse(xsltf))
77 def replace_characters(node):
78 def replace_chars(text):
81 return text.replace(u"\ufeff", u"")\
82 .replace("---", u"\u2014")\
83 .replace("--", u"\u2013")\
84 .replace(",,", u"\u201E")\
85 .replace('"', u"\u201D")\
86 .replace("'", u"\u2019")
87 if node.tag in ('uwaga', 'extra'):
91 node.text = replace_chars(node.text)
92 node.tail = replace_chars(node.tail)
94 replace_characters(child)
97 def find_annotations(annotations, source, part_no):
99 if child.tag in ('pe', 'pa', 'pt', 'pr'):
100 annotation = deepcopy(child)
101 number = str(len(annotations)+1)
102 annotation.set('number', number)
103 annotation.set('part', str(part_no))
105 annotations.append(annotation)
110 if child.tag not in ('extra', 'uwaga'):
111 find_annotations(annotations, child, part_no)
114 class Stanza(object):
116 Converts / verse endings into verse elements in a stanza.
118 Slashes may only occur directly in the stanza. Any slashes in subelements
119 will be ignored, and the subelements will be put inside verse elements.
121 >>> s = etree.fromstring("<strofa>a <b>c</b> <b>c</b>/\\nb<x>x/\\ny</x>c/ \\nd</strofa>")
122 >>> Stanza(s).versify()
123 >>> print etree.tostring(s)
124 <strofa><wers_normalny>a <b>c</b> <b>c</b></wers_normalny><wers_normalny>b<x>x/
125 y</x>c</wers_normalny><wers_normalny>d</wers_normalny></strofa>
128 def __init__(self, stanza_elem):
129 self.stanza = stanza_elem
131 self.open_verse = None
134 self.push_text(self.stanza.text)
135 for elem in self.stanza:
137 self.push_text(elem.tail)
138 tail = self.stanza.tail
140 self.stanza.tail = tail
141 self.stanza.extend(self.verses)
143 def open_normal_verse(self):
144 self.open_verse = self.stanza.makeelement("wers_normalny")
145 self.verses.append(self.open_verse)
147 def get_open_verse(self):
148 if self.open_verse is None:
149 self.open_normal_verse()
150 return self.open_verse
152 def push_text(self, text):
155 for i, verse_text in enumerate(re.split(r"/\s*\n", text)):
157 self.open_normal_verse()
158 verse = self.get_open_verse()
160 verse[-1].tail = (verse[-1].tail or "") + verse_text
162 verse.text = (verse.text or "") + verse_text
164 def push_elem(self, elem):
165 if elem.tag.startswith("wers"):
166 verse = deepcopy(elem)
168 self.verses.append(verse)
169 self.open_verse = verse
171 appended = deepcopy(elem)
173 self.get_open_verse().append(appended)
176 def replace_by_verse(tree):
177 """ Find stanzas and create new verses in place of a '/' character """
179 stanzas = tree.findall('.//' + WLNS('strofa'))
180 for stanza in stanzas:
181 Stanza(stanza).versify()
184 def add_to_manifest(manifest, partno):
185 """ Adds a node to the manifest section in content.opf file """
187 partstr = 'part%d' % partno
188 e = manifest.makeelement(OPFNS('item'), attrib={
190 'href': partstr + '.html',
191 'media-type': 'application/xhtml+xml',
196 def add_to_spine(spine, partno):
197 """ Adds a node to the spine section in content.opf file """
199 e = spine.makeelement(OPFNS('itemref'), attrib={'idref': 'part%d' % partno});
204 def __init__(self, name=None, part_href=None):
207 self.part_href = part_href
208 self.sub_number = None
210 def add(self, name, part_href, level=0, is_part=True, index=None):
211 assert level == 0 or index is None
212 if level > 0 and self.children:
213 return self.children[-1].add(name, part_href, level-1, is_part)
216 t.part_href = part_href
217 if index is not None:
218 self.children.insert(index, t)
220 self.children.append(t)
222 t.sub_number = len(self.children) + 1
225 def append(self, toc):
226 self.children.append(toc)
228 def extend(self, toc):
229 self.children.extend(toc.children)
233 return max((c.depth() for c in self.children)) + 1
239 if self.sub_number is not None:
240 src += '#sub%d' % self.sub_number
243 def write_to_xml(self, nav_map, counter=1):
244 for child in self.children:
245 nav_point = nav_map.makeelement(NCXNS('navPoint'))
246 nav_point.set('id', 'NavPoint-%d' % counter)
247 nav_point.set('playOrder', str(counter))
249 nav_label = nav_map.makeelement(NCXNS('navLabel'))
250 text = nav_map.makeelement(NCXNS('text'))
251 text.text = child.name
252 nav_label.append(text)
253 nav_point.append(nav_label)
255 content = nav_map.makeelement(NCXNS('content'))
256 content.set('src', child.href())
257 nav_point.append(content)
258 nav_map.append(nav_point)
259 counter = child.write_to_xml(nav_point, counter + 1)
262 def html_part(self, depth=0):
264 for child in self.children:
266 "<div style='margin-left:%dem;'><a href='%s'>%s</a></div>" %
267 (depth, child.href(), child.name))
268 texts.append(child.html_part(depth+1))
269 return "\n".join(texts)
272 with open(get_resource('epub/toc.html')) as f:
273 t = unicode(f.read(), 'utf-8')
274 return t % self.html_part()
277 def used_chars(element):
278 """ Lists characters used in an ETree Element """
279 chars = set((element.text or '') + (element.tail or ''))
280 for child in element:
281 chars = chars.union(used_chars(child))
286 """ divide main content of the XML file into chunks """
288 # prepare a container for each chunk
289 part_xml = etree.Element('utwor')
290 etree.SubElement(part_xml, 'master')
291 main_xml_part = part_xml[0] # master
293 last_node_part = False
294 for one_part in main_text:
296 if name == 'naglowek_czesc':
298 last_node_part = True
299 main_xml_part[:] = [deepcopy(one_part)]
300 elif not last_node_part and name in ("naglowek_rozdzial", "naglowek_akt", "srodtytul"):
302 main_xml_part[:] = [deepcopy(one_part)]
304 main_xml_part.append(deepcopy(one_part))
305 last_node_part = False
309 def transform_chunk(chunk_xml, chunk_no, annotations, empty=False, _empty_html_static=[]):
310 """ transforms one chunk, returns a HTML string, a TOC object and a set of used characters """
313 for element in chunk_xml[0]:
314 if element.tag in ("naglowek_czesc", "naglowek_rozdzial", "naglowek_akt", "srodtytul"):
315 toc.add(node_name(element), "part%d.html" % chunk_no)
316 elif element.tag in ('naglowek_podrozdzial', 'naglowek_scena'):
317 subnumber = toc.add(node_name(element), "part%d.html" % chunk_no, level=1, is_part=False)
318 element.set('sub', str(subnumber))
320 if not _empty_html_static:
321 _empty_html_static.append(open(get_resource('epub/emptyChunk.html')).read())
323 output_html = _empty_html_static[0]
325 find_annotations(annotations, chunk_xml, chunk_no)
326 replace_by_verse(chunk_xml)
327 html_tree = xslt(chunk_xml, get_resource('epub/xsltScheme.xsl'))
328 chars = used_chars(html_tree.getroot())
329 output_html = etree.tostring(html_tree, method="html", pretty_print=True)
330 return output_html, toc, chars
333 def transform(wldoc, verbose=False,
334 style=None, html_toc=False,
335 sample=None, cover=None, flags=None, resources=None,
336 intro_file=None, cover_file=None):
337 """ produces a EPUB file
339 sample=n: generate sample e-book (with at least n paragraphs)
340 cover: a cover.Cover factory or True for default
341 flags: less-advertising, without-fonts, working-copy
344 def transform_file(wldoc, chunk_counter=1, first=True, sample=None):
345 """ processes one input file and proceeds to its children """
347 replace_characters(wldoc.edoc.getroot())
349 # every input file will have a TOC entry,
350 # pointing to starting chunk
351 toc = TOC(wldoc.book_info.title, "part%d.html" % chunk_counter)
354 # write book title page
355 html_tree = xslt(wldoc.edoc, get_resource('epub/xsltTitle.xsl'))
356 chars = used_chars(html_tree.getroot())
357 zip.writestr('OPS/title.html',
358 etree.tostring(html_tree, method="html", pretty_print=True))
359 # add a title page TOC entry
360 toc.add(u"Tytuł", "title.html")
361 elif wldoc.book_info.parts:
362 # write title page for every parent
363 if sample is not None and sample <= 0:
365 html_string = open(get_resource('epub/emptyChunk.html')).read()
367 html_tree = xslt(wldoc.edoc, get_resource('epub/xsltChunkTitle.xsl'))
368 chars = used_chars(html_tree.getroot())
369 html_string = etree.tostring(html_tree, method="html", pretty_print=True)
370 zip.writestr('OPS/part%d.html' % chunk_counter, html_string)
371 add_to_manifest(manifest, chunk_counter)
372 add_to_spine(spine, chunk_counter)
375 if len(wldoc.edoc.getroot()) > 1:
376 # rdf before style master
377 main_text = wldoc.edoc.getroot()[1]
379 # rdf in style master
380 main_text = wldoc.edoc.getroot()[0]
381 if main_text.tag == RDFNS('RDF'):
384 if main_text is not None:
385 for chunk_xml in chop(main_text):
387 if sample is not None:
391 sample -= len(chunk_xml.xpath('//strofa|//akap|//akap_cd|//akap_dialog'))
392 chunk_html, chunk_toc, chunk_chars = transform_chunk(chunk_xml, chunk_counter, annotations, empty)
394 toc.extend(chunk_toc)
395 chars = chars.union(chunk_chars)
396 zip.writestr('OPS/part%d.html' % chunk_counter, chunk_html)
397 add_to_manifest(manifest, chunk_counter)
398 add_to_spine(spine, chunk_counter)
401 for child in wldoc.parts():
402 child_toc, chunk_counter, chunk_chars, sample = transform_file(
403 child, chunk_counter, first=False, sample=sample)
404 toc.append(child_toc)
405 chars = chars.union(chunk_chars)
407 return toc, chunk_counter, chars, sample
410 document = deepcopy(wldoc)
415 document.edoc.getroot().set(flag, 'yes')
418 document.edoc.getroot().set('editors', u', '.join(sorted(
419 editor.readable() for editor in document.editors())))
421 opf = xslt(document.book_info.to_etree(), get_resource('epub/xsltContent.xsl'))
422 manifest = opf.find('.//' + OPFNS('manifest'))
423 guide = opf.find('.//' + OPFNS('guide'))
424 spine = opf.find('.//' + OPFNS('spine'))
426 output_file = NamedTemporaryFile(prefix='librarian', suffix='.epub', delete=False)
427 zip = zipfile.ZipFile(output_file, 'w', zipfile.ZIP_DEFLATED)
429 # write static elements
430 mime = zipfile.ZipInfo()
431 mime.filename = 'mimetype'
432 mime.compress_type = zipfile.ZIP_STORED
434 zip.writestr(mime, 'application/epub+zip')
435 zip.writestr('META-INF/container.xml', '<?xml version="1.0" ?><container version="1.0" ' \
436 'xmlns="urn:oasis:names:tc:opendocument:xmlns:container">' \
437 '<rootfiles><rootfile full-path="OPS/content.opf" ' \
438 'media-type="application/oebps-package+xml" />' \
439 '</rootfiles></container>')
440 zip.write(get_resource('res/wl-logo-small.png'), os.path.join('OPS', 'logo_wolnelektury.png'))
441 zip.write(get_resource('res/logo.png'), os.path.join('OPS', 'logo.png'))
442 zip.write(get_resource('res/jedenprocent.png'), os.path.join('OPS', 'jedenprocent.png'))
444 style = get_resource('epub/style.css')
445 zip.write(style, os.path.join('OPS', 'style.css'))
447 if os.path.isdir(resources):
448 for dp, dirs, files in os.walk(resources):
450 fpath = os.path.join(dp, fname)
451 if os.path.isfile(fpath):
452 zip.write(fpath, os.path.join('OPS', fname))
453 manifest.append(etree.fromstring(
454 '<item id="%s" href="%s" media-type="%s" />' % (os.path.splitext(fname)[0], fname, guess_type(fpath)[0])))
457 print "resources path %s is not directory" % resources
462 cover = FutureOfCopyrightCover
464 cover_file = StringIO()
465 c = cover(document.book_info)
467 c_name = 'cover.%s' % c.ext()
468 zip.writestr(os.path.join('OPS', c_name), cover_file.getvalue())
471 cover_tree = etree.parse(get_resource('epub/cover.html'))
472 cover_tree.find('//' + XHTMLNS('img')).set('src', c_name)
473 zip.writestr('OPS/cover.html', etree.tostring(
474 cover_tree, method="html", pretty_print=True))
477 if document.book_info.cover_by:
478 document.edoc.getroot().set('data-cover-by', document.book_info.cover_by)
479 if document.book_info.cover_source:
480 document.edoc.getroot().set('data-cover-source', document.book_info.cover_source)
482 manifest.append(etree.fromstring(
483 '<item id="cover" href="cover.html" media-type="application/xhtml+xml" />'))
484 manifest.append(etree.fromstring(
485 '<item id="cover-image" href="%s" media-type="%s" />' % (c_name, c.mime_type())))
486 spine.insert(0, etree.fromstring('<itemref idref="cover" linear="no" />'))
487 opf.getroot()[0].append(etree.fromstring('<meta name="cover" content="cover-image"/>'))
488 guide.append(etree.fromstring('<reference href="cover.html" type="cover" title="Okładka"/>'))
492 annotations = etree.Element('annotations')
494 toc_file = etree.fromstring('<?xml version="1.0" encoding="utf-8"?><!DOCTYPE ncx PUBLIC ' \
495 '"-//NISO//DTD ncx 2005-1//EN" "http://www.daisy.org/z3986/2005/ncx-2005-1.dtd">' \
496 '<ncx xmlns="http://www.daisy.org/z3986/2005/ncx/" xml:lang="pl" ' \
497 'version="2005-1"><head></head><docTitle></docTitle><navMap>' \
499 nav_map = toc_file[-1]
501 manifest.append(etree.fromstring(
502 '<item id="first" href="first.html" media-type="application/xhtml+xml" />'))
503 spine.append(etree.fromstring(
504 '<itemref idref="first" />'))
505 html_tree = xslt(document.edoc, get_resource('epub/xsltFirst.xsl'))
506 # chars.update(used_chars(html_tree.getroot()))
507 zip.writestr('OPS/first.html', etree.tostring(
508 html_tree, method="html", pretty_print=True))
511 manifest.append(etree.fromstring(
512 '<item id="intro" href="intro.html" media-type="application/xhtml+xml" />'))
513 spine.append(etree.fromstring(
514 '<itemref idref="intro" />'))
515 zip.writestr('OPS/intro.html', open(intro_file or get_resource('epub/intro.html')).read())
519 manifest.append(etree.fromstring(
520 '<item id="html_toc" href="toc.html" media-type="application/xhtml+xml" />'))
521 spine.append(etree.fromstring(
522 '<itemref idref="html_toc" />'))
523 guide.append(etree.fromstring('<reference href="toc.html" type="toc" title="Table of Contents"/>'))
525 toc, chunk_counter, chars, sample = transform_file(document, sample=sample)
527 toc.add("Informacje redakcyjne", "first.html", index=0)
529 if len(toc.children) < 2:
530 toc.add(u"Początek książki", "part1.html")
532 # Last modifications in container files and EPUB creation
533 if len(annotations) > 0:
534 toc.add("Przypisy", "annotations.html")
535 manifest.append(etree.fromstring(
536 '<item id="annotations" href="annotations.html" media-type="application/xhtml+xml" />'))
537 spine.append(etree.fromstring(
538 '<itemref idref="annotations" />'))
539 replace_by_verse(annotations)
540 html_tree = xslt(annotations, get_resource('epub/xsltAnnotations.xsl'))
541 chars = chars.union(used_chars(html_tree.getroot()))
542 zip.writestr('OPS/annotations.html', etree.tostring(
543 html_tree, method="html", pretty_print=True))
545 # toc.add("Weprzyj Wolne Lektury", "support.html")
546 # manifest.append(etree.fromstring(
547 # '<item id="support" href="support.html" media-type="application/xhtml+xml" />'))
548 # spine.append(etree.fromstring(
549 # '<itemref idref="support" />'))
550 # html_string = open(get_resource('epub/support.html')).read()
551 # chars.update(used_chars(etree.fromstring(html_string)))
552 # zip.writestr('OPS/support.html', html_string)
554 toc.add("Informacje redakcyjne", "last.html")
555 manifest.append(etree.fromstring(
556 '<item id="last" href="last.html" media-type="application/xhtml+xml" />'))
557 spine.append(etree.fromstring(
558 '<itemref idref="last" />'))
559 html_tree = xslt(document.edoc, get_resource('epub/xsltLast.xsl'))
560 chars.update(used_chars(html_tree.getroot()))
561 zip.writestr('OPS/last.html', etree.tostring(
562 html_tree, method="html", pretty_print=True))
564 if not flags or not 'without-fonts' in flags:
566 tmpdir = mkdtemp('-librarian-epub')
572 os.chdir(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'font-optimizer'))
573 for fname in 'DejaVuSerif.ttf', 'DejaVuSerif-Bold.ttf', 'DejaVuSerif-Italic.ttf', 'DejaVuSerif-BoldItalic.ttf':
574 optimizer_call = ['perl', 'subset.pl', '--chars', ''.join(chars).encode('utf-8'),
575 get_resource('fonts/' + fname), os.path.join(tmpdir, fname)]
577 print "Running font-optimizer"
578 subprocess.check_call(optimizer_call)
580 subprocess.check_call(optimizer_call, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
581 zip.write(os.path.join(tmpdir, fname), os.path.join('OPS', fname))
582 manifest.append(etree.fromstring(
583 '<item id="%s" href="%s" media-type="font/ttf" />' % (fname, fname)))
588 zip.writestr('OPS/content.opf', etree.tostring(opf, pretty_print=True))
589 title = document.book_info.title
590 attributes = "dtb:uid", "dtb:depth", "dtb:totalPageCount", "dtb:maxPageNumber"
591 for st in attributes:
592 meta = toc_file.makeelement(NCXNS('meta'))
594 meta.set('content', '0')
595 toc_file[0].append(meta)
596 toc_file[0][0].set('content', ''.join((title, 'WolneLektury.pl')))
597 toc_file[0][1].set('content', str(toc.depth()))
598 set_inner_xml(toc_file[1], ''.join(('<text>', title, '</text>')))
602 toc.add(u"Spis treści", "toc.html", index=1)
603 zip.writestr('OPS/toc.html', toc.html().encode('utf-8'))
604 toc.write_to_xml(nav_map)
605 zip.writestr('OPS/toc.ncx', etree.tostring(toc_file, pretty_print=True))
608 return OutputFile.from_filename(output_file.name)