43eb295b5d6c188dbbb637d41cd3701d14b2dc1b
[librarian.git] / librarian / epub.py
1 # -*- coding: utf-8 -*-
2 #
3 # This file is part of Librarian, licensed under GNU Affero GPLv3 or later.
4 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
5 #
6 from __future__ import with_statement
7
8 import os
9 import os.path
10 import re
11 import subprocess
12 from StringIO import StringIO
13 from copy import deepcopy
14 from lxml import etree
15 import zipfile
16 from tempfile import mkdtemp, NamedTemporaryFile
17 from shutil import rmtree
18 from mimetypes import guess_type
19
20 from librarian import RDFNS, WLNS, NCXNS, OPFNS, XHTMLNS, OutputFile
21 from librarian.cover import WLCover, FutureOfCopyrightCover
22
23 from librarian import functions, get_resource
24
25 functions.reg_person_name()
26
27
28 def inner_xml(node):
29     """ returns node's text and children as a string
30
31     >>> print inner_xml(etree.fromstring('<a>x<b>y</b>z</a>'))
32     x<b>y</b>z
33     """
34
35     nt = node.text if node.text is not None else ''
36     return ''.join([nt] + [etree.tostring(child) for child in node])
37
38 def set_inner_xml(node, text):
39     """ sets node's text and children from a string
40
41     >>> e = etree.fromstring('<a>b<b>x</b>x</a>')
42     >>> set_inner_xml(e, 'x<b>y</b>z')
43     >>> print etree.tostring(e)
44     <a>x<b>y</b>z</a>
45     """
46
47     p = etree.fromstring('<x>%s</x>' % text)
48     node.text = p.text
49     node[:] = p[:]
50
51
52 def node_name(node):
53     """ Find out a node's name
54
55     >>> print node_name(etree.fromstring('<a>X<b>Y</b>Z</a>'))
56     XYZ
57     """
58
59     tempnode = deepcopy(node)
60
61     for p in ('pe', 'pa', 'pt', 'pr', 'motyw'):
62         for e in tempnode.findall('.//%s' % p):
63             t = e.tail
64             e.clear()
65             e.tail = t
66     etree.strip_tags(tempnode, '*')
67     return tempnode.text
68
69
70 def xslt(xml, sheet):
71     if isinstance(xml, etree._Element):
72         xml = etree.ElementTree(xml)
73     with open(sheet) as xsltf:
74         return xml.xslt(etree.parse(xsltf))
75
76
77 def replace_characters(node):
78     def replace_chars(text):
79         if text is None:
80             return None
81         return text.replace(u"\ufeff", u"")\
82                    .replace("---", u"\u2014")\
83                    .replace("--", u"\u2013")\
84                    .replace(",,", u"\u201E")\
85                    .replace('"', u"\u201D")\
86                    .replace("'", u"\u2019")
87     if node.tag in ('uwaga', 'extra'):
88         t = node.tail
89         node.clear()
90         node.tail = t
91     node.text = replace_chars(node.text)
92     node.tail = replace_chars(node.tail)
93     for child in node:
94         replace_characters(child)
95
96
97 def find_annotations(annotations, source, part_no):
98     for child in source:
99         if child.tag in ('pe', 'pa', 'pt', 'pr'):
100             annotation = deepcopy(child)
101             number = str(len(annotations)+1)
102             annotation.set('number', number)
103             annotation.set('part', str(part_no))
104             annotation.tail = ''
105             annotations.append(annotation)
106             tail = child.tail
107             child.clear()
108             child.tail = tail
109             child.text = number
110         if child.tag not in ('extra', 'uwaga'):
111             find_annotations(annotations, child, part_no)
112
113
114 class Stanza(object):
115     """
116     Converts / verse endings into verse elements in a stanza.
117
118     Slashes may only occur directly in the stanza. Any slashes in subelements
119     will be ignored, and the subelements will be put inside verse elements.
120
121     >>> s = etree.fromstring("<strofa>a <b>c</b> <b>c</b>/\\nb<x>x/\\ny</x>c/ \\nd</strofa>")
122     >>> Stanza(s).versify()
123     >>> print etree.tostring(s)
124     <strofa><wers_normalny>a <b>c</b> <b>c</b></wers_normalny><wers_normalny>b<x>x/
125     y</x>c</wers_normalny><wers_normalny>d</wers_normalny></strofa>
126     
127     """
128     def __init__(self, stanza_elem):
129         self.stanza = stanza_elem
130         self.verses = []
131         self.open_verse = None
132
133     def versify(self):
134         self.push_text(self.stanza.text)
135         for elem in self.stanza:
136             self.push_elem(elem)
137             self.push_text(elem.tail)
138         tail = self.stanza.tail
139         self.stanza.clear()
140         self.stanza.tail = tail
141         self.stanza.extend(self.verses)
142
143     def open_normal_verse(self):
144         self.open_verse = self.stanza.makeelement("wers_normalny")
145         self.verses.append(self.open_verse)
146
147     def get_open_verse(self):
148         if self.open_verse is None:
149             self.open_normal_verse()
150         return self.open_verse
151
152     def push_text(self, text):
153         if not text:
154             return
155         for i, verse_text in enumerate(re.split(r"/\s*\n", text)):
156             if i:
157                 self.open_normal_verse()
158             verse = self.get_open_verse()
159             if len(verse):
160                 verse[-1].tail = (verse[-1].tail or "") + verse_text
161             else:
162                 verse.text = (verse.text or "") + verse_text
163
164     def push_elem(self, elem):
165         if elem.tag.startswith("wers"):
166             verse = deepcopy(elem)
167             verse.tail = None
168             self.verses.append(verse)
169             self.open_verse = verse
170         else:
171             appended = deepcopy(elem)
172             appended.tail = None
173             self.get_open_verse().append(appended)
174
175
176 def replace_by_verse(tree):
177     """ Find stanzas and create new verses in place of a '/' character """
178
179     stanzas = tree.findall('.//' + WLNS('strofa'))
180     for stanza in stanzas:
181         Stanza(stanza).versify()
182
183
184 def add_to_manifest(manifest, partno):
185     """ Adds a node to the manifest section in content.opf file """
186
187     partstr = 'part%d' % partno
188     e = manifest.makeelement(OPFNS('item'), attrib={
189                                  'id': partstr,
190                                  'href': partstr + '.html',
191                                  'media-type': 'application/xhtml+xml',
192                              })
193     manifest.append(e)
194
195
196 def add_to_spine(spine, partno):
197     """ Adds a node to the spine section in content.opf file """
198
199     e = spine.makeelement(OPFNS('itemref'), attrib={'idref': 'part%d' % partno});
200     spine.append(e)
201
202
203 class TOC(object):
204     def __init__(self, name=None, part_href=None):
205         self.children = []
206         self.name = name
207         self.part_href = part_href
208         self.sub_number = None
209
210     def add(self, name, part_href, level=0, is_part=True, index=None):
211         assert level == 0 or index is None
212         if level > 0 and self.children:
213             return self.children[-1].add(name, part_href, level-1, is_part)
214         else:
215             t = TOC(name)
216             t.part_href = part_href
217             if index is not None:
218                 self.children.insert(index, t)
219             else:
220                 self.children.append(t)
221             if not is_part:
222                 t.sub_number = len(self.children) + 1
223                 return t.sub_number
224
225     def append(self, toc):
226         self.children.append(toc)
227
228     def extend(self, toc):
229         self.children.extend(toc.children)
230
231     def depth(self):
232         if self.children:
233             return max((c.depth() for c in self.children)) + 1
234         else:
235             return 0
236
237     def href(self):
238         src = self.part_href
239         if self.sub_number is not None:
240             src += '#sub%d' % self.sub_number
241         return src
242
243     def write_to_xml(self, nav_map, counter=1):
244         for child in self.children:
245             nav_point = nav_map.makeelement(NCXNS('navPoint'))
246             nav_point.set('id', 'NavPoint-%d' % counter)
247             nav_point.set('playOrder', str(counter))
248
249             nav_label = nav_map.makeelement(NCXNS('navLabel'))
250             text = nav_map.makeelement(NCXNS('text'))
251             text.text = child.name
252             nav_label.append(text)
253             nav_point.append(nav_label)
254
255             content = nav_map.makeelement(NCXNS('content'))
256             content.set('src', child.href())
257             nav_point.append(content)
258             nav_map.append(nav_point)
259             counter = child.write_to_xml(nav_point, counter + 1)
260         return counter
261
262     def html_part(self, depth=0):
263         texts = []
264         for child in self.children:
265             texts.append(
266                 "<div style='margin-left:%dem;'><a href='%s'>%s</a></div>" %
267                 (depth, child.href(), child.name))
268             texts.append(child.html_part(depth+1))
269         return "\n".join(texts)
270
271     def html(self):
272         with open(get_resource('epub/toc.html')) as f:
273             t = unicode(f.read(), 'utf-8')
274         return t % self.html_part()
275
276
277 def used_chars(element):
278     """ Lists characters used in an ETree Element """
279     chars = set((element.text or '') + (element.tail or ''))
280     for child in element:
281         chars = chars.union(used_chars(child))
282     return chars
283
284
285 def chop(main_text):
286     """ divide main content of the XML file into chunks """
287
288     # prepare a container for each chunk
289     part_xml = etree.Element('utwor')
290     etree.SubElement(part_xml, 'master')
291     main_xml_part = part_xml[0] # master
292
293     last_node_part = False
294     for one_part in main_text:
295         name = one_part.tag
296         if name == 'naglowek_czesc':
297             yield part_xml
298             last_node_part = True
299             main_xml_part[:] = [deepcopy(one_part)]
300         elif not last_node_part and name in ("naglowek_rozdzial", "naglowek_akt", "srodtytul"):
301             yield part_xml
302             main_xml_part[:] = [deepcopy(one_part)]
303         else:
304             main_xml_part.append(deepcopy(one_part))
305             last_node_part = False
306     yield part_xml
307
308
309 def transform_chunk(chunk_xml, chunk_no, annotations, empty=False, _empty_html_static=[]):
310     """ transforms one chunk, returns a HTML string, a TOC object and a set of used characters """
311
312     toc = TOC()
313     for element in chunk_xml[0]:
314         if element.tag in ("naglowek_czesc", "naglowek_rozdzial", "naglowek_akt", "srodtytul"):
315             toc.add(node_name(element), "part%d.html" % chunk_no)
316         elif element.tag in ('naglowek_podrozdzial', 'naglowek_scena'):
317             subnumber = toc.add(node_name(element), "part%d.html" % chunk_no, level=1, is_part=False)
318             element.set('sub', str(subnumber))
319     if empty:
320         if not _empty_html_static:
321             _empty_html_static.append(open(get_resource('epub/emptyChunk.html')).read())
322         chars = set()
323         output_html = _empty_html_static[0]
324     else:
325         find_annotations(annotations, chunk_xml, chunk_no)
326         replace_by_verse(chunk_xml)
327         html_tree = xslt(chunk_xml, get_resource('epub/xsltScheme.xsl'))
328         chars = used_chars(html_tree.getroot())
329         output_html = etree.tostring(html_tree, method="html", pretty_print=True)
330     return output_html, toc, chars
331
332
333 def transform(wldoc, verbose=False,
334               style=None, html_toc=False,
335               sample=None, cover=None, flags=None, resources=None,
336               intro_file=None, cover_file=None):
337     """ produces a EPUB file
338
339     sample=n: generate sample e-book (with at least n paragraphs)
340     cover: a cover.Cover factory or True for default
341     flags: less-advertising, without-fonts, working-copy
342     """
343
344     def transform_file(wldoc, chunk_counter=1, first=True, sample=None):
345         """ processes one input file and proceeds to its children """
346
347         replace_characters(wldoc.edoc.getroot())
348
349         # every input file will have a TOC entry,
350         # pointing to starting chunk
351         toc = TOC(wldoc.book_info.title, "part%d.html" % chunk_counter)
352         chars = set()
353         if first:
354             # write book title page
355             html_tree = xslt(wldoc.edoc, get_resource('epub/xsltTitle.xsl'))
356             chars = used_chars(html_tree.getroot())
357             zip.writestr('OPS/title.html',
358                  etree.tostring(html_tree, method="html", pretty_print=True))
359             # add a title page TOC entry
360             toc.add(u"Tytuł", "title.html")
361         elif wldoc.book_info.parts:
362             # write title page for every parent
363             if sample is not None and sample <= 0:
364                 chars = set()
365                 html_string = open(get_resource('epub/emptyChunk.html')).read()
366             else:
367                 html_tree = xslt(wldoc.edoc, get_resource('epub/xsltChunkTitle.xsl'))
368                 chars = used_chars(html_tree.getroot())
369                 html_string = etree.tostring(html_tree, method="html", pretty_print=True)
370             zip.writestr('OPS/part%d.html' % chunk_counter, html_string)
371             add_to_manifest(manifest, chunk_counter)
372             add_to_spine(spine, chunk_counter)
373             chunk_counter += 1
374
375         if len(wldoc.edoc.getroot()) > 1:
376             # rdf before style master
377             main_text = wldoc.edoc.getroot()[1]
378         else:
379             # rdf in style master
380             main_text = wldoc.edoc.getroot()[0]
381             if main_text.tag == RDFNS('RDF'):
382                 main_text = None
383
384         if main_text is not None:
385             for chunk_xml in chop(main_text):
386                 empty = False
387                 if sample is not None:
388                     if sample <= 0:
389                         empty = True
390                     else:
391                         sample -= len(chunk_xml.xpath('//strofa|//akap|//akap_cd|//akap_dialog'))
392                 chunk_html, chunk_toc, chunk_chars = transform_chunk(chunk_xml, chunk_counter, annotations, empty)
393
394                 toc.extend(chunk_toc)
395                 chars = chars.union(chunk_chars)
396                 zip.writestr('OPS/part%d.html' % chunk_counter, chunk_html)
397                 add_to_manifest(manifest, chunk_counter)
398                 add_to_spine(spine, chunk_counter)
399                 chunk_counter += 1
400
401         for child in wldoc.parts():
402             child_toc, chunk_counter, chunk_chars, sample = transform_file(
403                 child, chunk_counter, first=False, sample=sample)
404             toc.append(child_toc)
405             chars = chars.union(chunk_chars)
406
407         return toc, chunk_counter, chars, sample
408
409
410     document = deepcopy(wldoc)
411     del wldoc
412
413     if flags:
414         for flag in flags:
415             document.edoc.getroot().set(flag, 'yes')
416
417     # add editors info
418     document.edoc.getroot().set('editors', u', '.join(sorted(
419         editor.readable() for editor in document.editors())))
420
421     opf = xslt(document.book_info.to_etree(), get_resource('epub/xsltContent.xsl'))
422     manifest = opf.find('.//' + OPFNS('manifest'))
423     guide = opf.find('.//' + OPFNS('guide'))
424     spine = opf.find('.//' + OPFNS('spine'))
425
426     output_file = NamedTemporaryFile(prefix='librarian', suffix='.epub', delete=False)
427     zip = zipfile.ZipFile(output_file, 'w', zipfile.ZIP_DEFLATED)
428
429     # write static elements
430     mime = zipfile.ZipInfo()
431     mime.filename = 'mimetype'
432     mime.compress_type = zipfile.ZIP_STORED
433     mime.extra = ''
434     zip.writestr(mime, 'application/epub+zip')
435     zip.writestr('META-INF/container.xml', '<?xml version="1.0" ?><container version="1.0" ' \
436                        'xmlns="urn:oasis:names:tc:opendocument:xmlns:container">' \
437                        '<rootfiles><rootfile full-path="OPS/content.opf" ' \
438                        'media-type="application/oebps-package+xml" />' \
439                        '</rootfiles></container>')
440     zip.write(get_resource('res/wl-logo-small.png'), os.path.join('OPS', 'logo_wolnelektury.png'))
441     zip.write(get_resource('res/logo.png'), os.path.join('OPS', 'logo.png'))
442     zip.write(get_resource('res/jedenprocent.png'), os.path.join('OPS', 'jedenprocent.png'))
443     if not style:
444         style = get_resource('epub/style.css')
445     zip.write(style, os.path.join('OPS', 'style.css'))
446     if resources:
447         if os.path.isdir(resources):
448             for dp, dirs, files in os.walk(resources):
449                 for fname in files:
450                     fpath  = os.path.join(dp, fname)
451                     if os.path.isfile(fpath):
452                         zip.write(fpath, os.path.join('OPS', fname))
453                         manifest.append(etree.fromstring(
454                                 '<item id="%s" href="%s" media-type="%s" />' % (os.path.splitext(fname)[0], fname, guess_type(fpath)[0])))
455
456         else:
457             print "resources path %s is not directory" % resources
458                 
459
460     if cover:
461         if cover is True:
462             cover = FutureOfCopyrightCover
463
464         cover_file = StringIO()
465         c = cover(document.book_info)
466         c.save(cover_file)
467         c_name = 'cover.%s' % c.ext()
468         zip.writestr(os.path.join('OPS', c_name), cover_file.getvalue())
469         del cover_file
470
471         cover_tree = etree.parse(get_resource('epub/cover.html'))
472         cover_tree.find('//' + XHTMLNS('img')).set('src', c_name)
473         zip.writestr('OPS/cover.html', etree.tostring(
474                         cover_tree, method="html", pretty_print=True))
475
476         if c.uses_dc_cover:
477             if document.book_info.cover_by:
478                 document.edoc.getroot().set('data-cover-by', document.book_info.cover_by)
479             if document.book_info.cover_source:
480                 document.edoc.getroot().set('data-cover-source', document.book_info.cover_source)
481
482         manifest.append(etree.fromstring(
483             '<item id="cover" href="cover.html" media-type="application/xhtml+xml" />'))
484         manifest.append(etree.fromstring(
485             '<item id="cover-image" href="%s" media-type="%s" />' % (c_name, c.mime_type())))
486         spine.insert(0, etree.fromstring('<itemref idref="cover" linear="no" />'))
487         opf.getroot()[0].append(etree.fromstring('<meta name="cover" content="cover-image"/>'))
488         guide.append(etree.fromstring('<reference href="cover.html" type="cover" title="Okładka"/>'))
489     
490               
491
492     annotations = etree.Element('annotations')
493
494     toc_file = etree.fromstring('<?xml version="1.0" encoding="utf-8"?><!DOCTYPE ncx PUBLIC ' \
495                                '"-//NISO//DTD ncx 2005-1//EN" "http://www.daisy.org/z3986/2005/ncx-2005-1.dtd">' \
496                                '<ncx xmlns="http://www.daisy.org/z3986/2005/ncx/" xml:lang="pl" ' \
497                                'version="2005-1"><head></head><docTitle></docTitle><navMap>' \
498                                '</navMap></ncx>')
499     nav_map = toc_file[-1]
500
501     manifest.append(etree.fromstring(
502         '<item id="first" href="first.html" media-type="application/xhtml+xml" />'))
503     spine.append(etree.fromstring(
504         '<itemref idref="first" />'))
505     html_tree = xslt(document.edoc, get_resource('epub/xsltFirst.xsl'))
506 #    chars.update(used_chars(html_tree.getroot()))
507     zip.writestr('OPS/first.html', etree.tostring(
508                         html_tree, method="html", pretty_print=True))
509
510     if intro_file:
511         manifest.append(etree.fromstring(
512                 '<item id="intro" href="intro.html" media-type="application/xhtml+xml" />'))
513         spine.append(etree.fromstring(
514                 '<itemref idref="intro" />'))
515         zip.writestr('OPS/intro.html', open(intro_file or get_resource('epub/intro.html')).read())
516
517
518     if html_toc:
519         manifest.append(etree.fromstring(
520             '<item id="html_toc" href="toc.html" media-type="application/xhtml+xml" />'))
521         spine.append(etree.fromstring(
522             '<itemref idref="html_toc" />'))
523         guide.append(etree.fromstring('<reference href="toc.html" type="toc" title="Table of Contents"/>'))
524
525     toc, chunk_counter, chars, sample = transform_file(document, sample=sample)
526
527     toc.add("Informacje redakcyjne", "first.html", index=0)
528
529     if len(toc.children) < 2:
530         toc.add(u"Początek książki", "part1.html")
531
532     # Last modifications in container files and EPUB creation
533     if len(annotations) > 0:
534         toc.add("Przypisy", "annotations.html")
535         manifest.append(etree.fromstring(
536             '<item id="annotations" href="annotations.html" media-type="application/xhtml+xml" />'))
537         spine.append(etree.fromstring(
538             '<itemref idref="annotations" />'))
539         replace_by_verse(annotations)
540         html_tree = xslt(annotations, get_resource('epub/xsltAnnotations.xsl'))
541         chars = chars.union(used_chars(html_tree.getroot()))
542         zip.writestr('OPS/annotations.html', etree.tostring(
543                             html_tree, method="html", pretty_print=True))
544
545     # toc.add("Weprzyj Wolne Lektury", "support.html")
546     # manifest.append(etree.fromstring(
547     #     '<item id="support" href="support.html" media-type="application/xhtml+xml" />'))
548     # spine.append(etree.fromstring(
549     #     '<itemref idref="support" />'))
550     # html_string = open(get_resource('epub/support.html')).read()
551     # chars.update(used_chars(etree.fromstring(html_string)))
552     # zip.writestr('OPS/support.html', html_string)
553
554     toc.add("Informacje redakcyjne", "last.html")
555     manifest.append(etree.fromstring(
556         '<item id="last" href="last.html" media-type="application/xhtml+xml" />'))
557     spine.append(etree.fromstring(
558         '<itemref idref="last" />'))
559     html_tree = xslt(document.edoc, get_resource('epub/xsltLast.xsl'))
560     chars.update(used_chars(html_tree.getroot()))
561     zip.writestr('OPS/last.html', etree.tostring(
562                         html_tree, method="html", pretty_print=True))
563
564     if not flags or not 'without-fonts' in flags:
565         # strip fonts
566         tmpdir = mkdtemp('-librarian-epub')
567         try:
568             cwd = os.getcwd()
569         except OSError:
570             cwd = None
571
572         os.chdir(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'font-optimizer'))
573         for fname in 'DejaVuSerif.ttf', 'DejaVuSerif-Bold.ttf', 'DejaVuSerif-Italic.ttf', 'DejaVuSerif-BoldItalic.ttf':
574             optimizer_call = ['perl', 'subset.pl', '--chars', ''.join(chars).encode('utf-8'),
575                               get_resource('fonts/' + fname), os.path.join(tmpdir, fname)]
576             if verbose:
577                 print "Running font-optimizer"
578                 subprocess.check_call(optimizer_call)
579             else:
580                 subprocess.check_call(optimizer_call, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
581             zip.write(os.path.join(tmpdir, fname), os.path.join('OPS', fname))
582             manifest.append(etree.fromstring(
583                 '<item id="%s" href="%s" media-type="font/ttf" />' % (fname, fname)))
584         rmtree(tmpdir)
585         if cwd is not None:
586             os.chdir(cwd)
587
588     zip.writestr('OPS/content.opf', etree.tostring(opf, pretty_print=True))
589     title = document.book_info.title
590     attributes = "dtb:uid", "dtb:depth", "dtb:totalPageCount", "dtb:maxPageNumber"
591     for st in attributes:
592         meta = toc_file.makeelement(NCXNS('meta'))
593         meta.set('name', st)
594         meta.set('content', '0')
595         toc_file[0].append(meta)
596     toc_file[0][0].set('content', ''.join((title, 'WolneLektury.pl')))
597     toc_file[0][1].set('content', str(toc.depth()))
598     set_inner_xml(toc_file[1], ''.join(('<text>', title, '</text>')))
599
600     # write TOC
601     if html_toc:
602         toc.add(u"Spis treści", "toc.html", index=1)
603         zip.writestr('OPS/toc.html', toc.html().encode('utf-8'))
604     toc.write_to_xml(nav_map)
605     zip.writestr('OPS/toc.ncx', etree.tostring(toc_file, pretty_print=True))
606     zip.close()
607
608     return OutputFile.from_filename(output_file.name)