br

[librarian.git] / src / librarian / pdf.py

diff --git a/src/librarian/pdf.py b/src/librarian/pdf.py
index e6d897d..c7de78b 100644 (file)
--- a/src/librarian/pdf.py
+++ b/src/librarian/pdf.py
@@ -1,7 +1,5 @@
-# -*- coding: utf-8 -*-
-#

 # This file is part of Librarian, licensed under GNU Affero GPLv3 or later.
 # This file is part of Librarian, licensed under GNU Affero GPLv3 or later.

-# Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
+# Copyright © Fundacja Wolne Lektury. See NOTICE for more information.

 #
 """PDF creation library.
 
 #
 """PDF creation library.
 


@@ -9,8 +7,7 @@ Creates one big XML from the book and its children, converts it to LaTeX
 with TeXML, then runs it by XeLaTeX.
 
 """
 with TeXML, then runs it by XeLaTeX.
 
 """

-from __future__ import print_function, unicode_literals
-
+import io

 import os
 import os.path
 import shutil
 import os
 import os.path
 import shutil


@@ -19,11 +16,13 @@ import re
 from copy import deepcopy
 from subprocess import call, PIPE
 from itertools import chain
 from copy import deepcopy
 from subprocess import call, PIPE
 from itertools import chain

+import urllib.parse
+import urllib.request

 
 

+from PIL import Image

 from Texml.processor import process
 from lxml import etree
 from lxml.etree import XMLSyntaxError, XSLTApplyError
 from Texml.processor import process
 from lxml import etree
 from lxml.etree import XMLSyntaxError, XSLTApplyError

-import six

 
 from librarian.dcparser import Person
 from librarian.parser import WLDocument
 
 from librarian.dcparser import Person
 from librarian.parser import WLDocument


@@ -54,7 +53,9 @@ STYLESHEETS = {
 
 
 def insert_tags(doc, split_re, tagname, exclude=None):
 
 
 def insert_tags(doc, split_re, tagname, exclude=None):

-    """ inserts <tagname> for every occurence of `split_re' in text nodes in the `doc' tree
+    """
+    Inserts <tagname> for every occurence of `split_re'
+    in text nodes in the `doc' tree.

 
     >>> t = etree.fromstring('<a><b>A-B-C</b>X-Y-Z</a>')
     >>> insert_tags(t, re.compile('-'), 'd')
 
     >>> t = etree.fromstring('<a><b>A-B-C</b>X-Y-Z</a>')
     >>> insert_tags(t, re.compile('-'), 'd')


@@ -84,19 +85,21 @@ def insert_tags(doc, split_re, tagname, exclude=None):
 
 
 def substitute_hyphens(doc):
 
 
 def substitute_hyphens(doc):

-    insert_tags(doc,
-                re.compile("(?<=[^-\s])-(?=[^-\s])"),
-                "dywiz",
-                exclude=[DCNS("identifier.url"), DCNS("rights.license"), "meta"]
-                )
+    insert_tags(
+        doc,
+        re.compile(r"(?<=[^-\s])-(?=[^-\s])"),
+        "dywiz",
+        exclude=[DCNS("identifier.url"), DCNS("rights.license"), "meta"]
+    )

 
 
 def fix_hanging(doc):
 
 
 def fix_hanging(doc):

-    insert_tags(doc,
-                re.compile("(?<=\s\w)\s+"),
-                "nbsp",
-                exclude=[DCNS("identifier.url"), DCNS("rights.license")]
-                )
+    insert_tags(
+        doc,
+        re.compile(r"(?<=\s\w)\s+"),
+        "nbsp",
+        exclude=[DCNS("identifier.url"), DCNS("rights.license")]
+    )

 
 
 def fix_tables(doc):
 
 
 def fix_tables(doc):


@@ -112,25 +115,37 @@ def fix_tables(doc):
 
 
 def mark_subauthors(doc):
 
 
 def mark_subauthors(doc):

-    root_author = ', '.join(elem.text for elem in doc.findall('./' + RDFNS('RDF') + '//' + DCNS('creator_parsed')))
+    root_author = ', '.join(
+        elem.text
+        for elem in doc.findall(
+                './' + RDFNS('RDF') + '//' + DCNS('creator_parsed')
+        )
+    )

     last_author = None
     # jeśli autor jest inny niż autor całości i niż poprzedni autor
     # to wstawiamy jakiś znacznik w rdf?
     for subutwor in doc.xpath('/utwor/utwor'):
     last_author = None
     # jeśli autor jest inny niż autor całości i niż poprzedni autor
     # to wstawiamy jakiś znacznik w rdf?
     for subutwor in doc.xpath('/utwor/utwor'):

-        author = ', '.join(elem.text for elem in subutwor.findall('.//' + DCNS('creator_parsed')))
+        author = ', '.join(
+            elem.text
+            for elem in subutwor.findall('.//' + DCNS('creator_parsed'))
+        )

         if author not in (last_author, root_author):
         if author not in (last_author, root_author):

-            subutwor.find('.//' + RDFNS('RDF')).append(etree.Element('use_subauthor'))
+            subutwor.find('.//' + RDFNS('RDF')).append(
+                etree.Element('use_subauthor')
+            )

         last_author = author
 
 
 def move_motifs_inside(doc):
     """ moves motifs to be into block elements """
     for master in doc.xpath('//powiesc|//opowiadanie|//liryka_l|//liryka_lp|'
         last_author = author
 
 
 def move_motifs_inside(doc):
     """ moves motifs to be into block elements """
     for master in doc.xpath('//powiesc|//opowiadanie|//liryka_l|//liryka_lp|'

-                            '//dramat_wierszowany_l|//dramat_wierszowany_lp|//dramat_wspolczesny'):
+                            '//dramat_wierszowany_l|//dramat_wierszowany_lp|'
+                            '//dramat_wspolczesny'):

         for motif in master.xpath('motyw'):
             for sib in motif.itersiblings():
         for motif in master.xpath('motyw'):
             for sib in motif.itersiblings():

-                if sib.tag not in ('sekcja_swiatlo', 'sekcja_asterysk', 'separator_linia',
-                                   'begin', 'end', 'motyw', 'extra', 'uwaga'):
+                if sib.tag not in ('sekcja_swiatlo', 'sekcja_asterysk',
+                                   'separator_linia', 'begin', 'end',
+                                   'motyw', 'extra', 'uwaga'):

                     # motif shouldn't have a tail - it would be untagged text
                     motif.tail = None
                     motif.getparent().remove(motif)
                     # motif shouldn't have a tail - it would be untagged text
                     motif.tail = None
                     motif.getparent().remove(motif)


@@ -139,18 +154,21 @@ def move_motifs_inside(doc):
 
 
 def hack_motifs(doc):
 
 
 def hack_motifs(doc):

-    """ dirty hack for the marginpar-creates-orphans LaTeX problem
+    """
+    Dirty hack for the marginpar-creates-orphans LaTeX problem

     see http://www.latex-project.org/cgi-bin/ltxbugs2html?pr=latex/2304
 
     see http://www.latex-project.org/cgi-bin/ltxbugs2html?pr=latex/2304
 

-    moves motifs in stanzas from first verse to second
-    and from next to last to last, then inserts negative vspace before them
+    Moves motifs in stanzas from first verse to second and from next
+    to last to last, then inserts negative vspace before them.

     """
     for motif in doc.findall('//strofa//motyw'):
         # find relevant verse-level tag
         verse, stanza = motif, motif.getparent()
         while stanza is not None and stanza.tag != 'strofa':
             verse, stanza = stanza, stanza.getparent()
     """
     for motif in doc.findall('//strofa//motyw'):
         # find relevant verse-level tag
         verse, stanza = motif, motif.getparent()
         while stanza is not None and stanza.tag != 'strofa':
             verse, stanza = stanza, stanza.getparent()

-        breaks_before = sum(1 for i in verse.itersiblings('br', preceding=True))
+        breaks_before = sum(
+            1 for i in verse.itersiblings('br', preceding=True)
+        )

         breaks_after = sum(1 for i in verse.itersiblings('br'))
         if (breaks_before == 0 and breaks_after > 0) or breaks_after == 1:
             move_by = 1
         breaks_after = sum(1 for i in verse.itersiblings('br'))
         if (breaks_before == 0 and breaks_after > 0) or breaks_after == 1:
             move_by = 1


@@ -170,14 +188,40 @@ def hack_motifs(doc):
                 break
 
 
                 break
 
 

+def add_fundraising(doc, fundraising):
+    # Before each naglowek_rozdzial and naglowek_scena and in the end
+    spots = []
+    for naglowek in doc.xpath('//naglowek_czesc|//naglowek_akt'):
+        spot = etree.Element('f_spot')
+        naglowek.addprevious(spot)
+        spots.append(spot)
+    spot = etree.Element('f_spot')
+    doc.getroot()[-1].append(spot)
+    spots.append(spot)
+    e = len(spots)
+    nfunds = len(fundraising)
+    if e > 4 * nfunds:
+        nfunds *= 2
+    for f in range(nfunds):
+        spot_index = int(f / nfunds * e)
+        spots[spot_index].set('active', 'true')
+        elem = etree.fromstring('<f_spot>' + fundraising[f % len(fundraising)] + '</f_spot>')
+        spots[spot_index].text = elem.text
+        for c in elem:
+            spots[spot_index].append(c)
+
+

 def parse_creator(doc):
     """Generates readable versions of creator and translator tags.
 
     Finds all dc:creator and dc.contributor.translator tags
     and adds *_parsed versions with forenames first.
     """
 def parse_creator(doc):
     """Generates readable versions of creator and translator tags.
 
     Finds all dc:creator and dc.contributor.translator tags
     and adds *_parsed versions with forenames first.
     """

-    for person in doc.xpath("|".join('//dc:' + tag for tag in ('creator', 'contributor.translator')),
-                            namespaces={'dc': str(DCNS)})[::-1]:
+    for person in doc.xpath(
+            "|".join('//dc:' + tag for tag in (
+                'creator', 'contributor.translator'
+            )),
+            namespaces={'dc': str(DCNS)})[::-1]:

         if not person.text:
             continue
         p = Person.from_text(person.text)
         if not person.text:
             continue
         p = Person.from_text(person.text)


@@ -193,7 +237,10 @@ def get_stylesheet(name):
 
 
 def package_available(package, args='', verbose=False):
 
 
 def package_available(package, args='', verbose=False):

-    """ check if a verion of a latex package accepting given args is available """
+    """
+    Check if a verion of a latex package accepting given args
+    is available.
+    """

     tempdir = mkdtemp('-wl2pdf-test')
     fpath = os.path.join(tempdir, 'test.tex')
     f = open(fpath, 'w')
     tempdir = mkdtemp('-wl2pdf-test')
     fpath = os.path.join(tempdir, 'test.tex')
     f = open(fpath, 'w')


@@ -207,13 +254,18 @@ def package_available(package, args='', verbose=False):
     if verbose:
         p = call(['xelatex', '-output-directory', tempdir, fpath])
     else:
     if verbose:
         p = call(['xelatex', '-output-directory', tempdir, fpath])
     else:

-        p = call(['xelatex', '-interaction=batchmode', '-output-directory', tempdir, fpath], stdout=PIPE, stderr=PIPE)
+        p = call(
+            ['xelatex', '-interaction=batchmode', '-output-directory',
+             tempdir, fpath],
+            stdout=PIPE, stderr=PIPE
+        )

     shutil.rmtree(tempdir)
     return p == 0
 
 
 def transform(wldoc, verbose=False, save_tex=None, morefloats=None,
     shutil.rmtree(tempdir)
     return p == 0
 
 
 def transform(wldoc, verbose=False, save_tex=None, morefloats=None,

-              cover=None, flags=None, customizations=None, ilustr_path='', latex_dir=False):
+              cover=None, flags=None, customizations=None, base_url='file://./',
+              latex_dir=False, fundraising=None):

     """ produces a PDF file with XeLaTeX
 
     wldoc: a WLDocument
     """ produces a PDF file with XeLaTeX
 
     wldoc: a WLDocument


@@ -222,7 +274,8 @@ def transform(wldoc, verbose=False, save_tex=None, morefloats=None,
     morefloats (old/new/none): force specific morefloats
     cover: a cover.Cover factory or True for default
     flags: less-advertising,
     morefloats (old/new/none): force specific morefloats
     cover: a cover.Cover factory or True for default
     flags: less-advertising,

-    customizations: user requested customizations regarding various formatting parameters (passed to wl LaTeX class)
+    customizations: user requested customizations regarding various
+        formatting parameters (passed to wl LaTeX class)

     """
 
     # Parse XSLT
     """
 
     # Parse XSLT


@@ -252,21 +305,31 @@ def transform(wldoc, verbose=False, save_tex=None, morefloats=None,
         elif package_available('morefloats', 'maxfloats=19'):
             root.set('morefloats', 'new')
 
         elif package_available('morefloats', 'maxfloats=19'):
             root.set('morefloats', 'new')
 

+        if customizations is None:
+            customizations = []
+        else:
+            customizations = list(customizations)
+
+        if book_info.endnotes:
+            customizations.append('endnotes')
+

         # add customizations
         if customizations is not None:
         # add customizations
         if customizations is not None:

-            root.set('customizations', u','.join(customizations))
+            root.set('customizations', ','.join(customizations))

 
         # add editors info
         editors = document.editors()
         if editors:
 
         # add editors info
         editors = document.editors()
         if editors:

-            root.set('editors', u', '.join(sorted(
+            root.set('editors', ', '.join(sorted(

                 editor.readable() for editor in editors)))
         if document.book_info.funders:
                 editor.readable() for editor in editors)))
         if document.book_info.funders:

-            root.set('funders', u', '.join(document.book_info.funders))
+            root.set('funders', ', '.join(document.book_info.funders))

         if document.book_info.thanks:
             root.set('thanks', document.book_info.thanks)
 
         # hack the tree
         if document.book_info.thanks:
             root.set('thanks', document.book_info.thanks)
 
         # hack the tree

+        if fundraising:
+            add_fundraising(document.edoc, fundraising)

         move_motifs_inside(document.edoc)
         hack_motifs(document.edoc)
         parse_creator(document.edoc)
         move_motifs_inside(document.edoc)
         hack_motifs(document.edoc)
         parse_creator(document.edoc)


@@ -274,6 +337,7 @@ def transform(wldoc, verbose=False, save_tex=None, morefloats=None,
         fix_hanging(document.edoc)
         fix_tables(document.edoc)
         mark_subauthors(document.edoc)
         fix_hanging(document.edoc)
         fix_tables(document.edoc)
         mark_subauthors(document.edoc)

+        document.fix_pa_akap()

 
         # wl -> TeXML
         style_filename = get_stylesheet("wl2tex")
 
         # wl -> TeXML
         style_filename = get_stylesheet("wl2tex")


@@ -283,8 +347,30 @@ def transform(wldoc, verbose=False, save_tex=None, morefloats=None,
         # TeXML -> LaTeX
         temp = mkdtemp('-wl2pdf')
 
         # TeXML -> LaTeX
         temp = mkdtemp('-wl2pdf')
 

-        for ilustr in document.edoc.findall("//ilustr"):
-            shutil.copy(os.path.join(ilustr_path, ilustr.get("src")), temp)
+        for i, ilustr in enumerate(document.edoc.findall('//ilustr')):
+            url = urllib.parse.urljoin(
+                base_url,
+                ilustr.get('src')
+            )
+            imgfile = urllib.request.urlopen(url)
+            img = Image.open(imgfile)
+
+            th_format, ext, media_type = {
+                'GIF': ('GIF', 'gif', 'image/gif'),
+                'PNG': ('PNG', 'png', 'image/png'),
+            }.get(img.format, ('JPEG', 'jpg', 'image/jpeg'))
+
+            width = 2400
+            if img.size[0] < width:
+                th = img
+            else:
+                th = img.resize((width, round(width * img.size[1] / img.size[0])))
+
+            file_name = 'image%d.%s' % (i, ext)
+            th.save(os.path.join(temp, file_name))
+            ilustr.set('src', file_name)
+
+            imgfile.close()

 
         for sponsor in book_info.sponsors:
             ins = etree.Element("data-sponsor", name=sponsor)
 
         for sponsor in book_info.sponsors:
             ins = etree.Element("data-sponsor", name=sponsor)


@@ -294,7 +380,7 @@ def transform(wldoc, verbose=False, save_tex=None, morefloats=None,
                 shutil.copy(logo, os.path.join(temp, fname))
                 ins.set('src', fname)
             root.insert(0, ins)
                 shutil.copy(logo, os.path.join(temp, fname))
                 ins.set('src', fname)
             root.insert(0, ins)

-                
+

         if book_info.sponsor_note:
             root.set("sponsor-note", book_info.sponsor_note)
 
         if book_info.sponsor_note:
             root.set("sponsor-note", book_info.sponsor_note)
 


@@ -308,7 +394,7 @@ def transform(wldoc, verbose=False, save_tex=None, morefloats=None,
 
         tex_path = os.path.join(temp, 'doc.tex')
         fout = open(tex_path, 'wb')
 
         tex_path = os.path.join(temp, 'doc.tex')
         fout = open(tex_path, 'wb')

-        process(six.BytesIO(texml), fout, 'utf-8')
+        process(io.BytesIO(texml), fout, 'utf-8')

         fout.close()
         del texml
 
         fout.close()
         del texml
 


@@ -318,6 +404,7 @@ def transform(wldoc, verbose=False, save_tex=None, morefloats=None,
         # LaTeX -> PDF
         shutil.copy(get_resource('pdf/wl.cls'), temp)
         shutil.copy(get_resource('res/wl-logo.png'), temp)
         # LaTeX -> PDF
         shutil.copy(get_resource('pdf/wl.cls'), temp)
         shutil.copy(get_resource('res/wl-logo.png'), temp)

+        shutil.copy(get_resource('fonts/OpenSans-VariableFont_wdth,wght.ttf'), temp)

 
         if latex_dir:
             return temp
 
         if latex_dir:
             return temp


@@ -334,14 +421,18 @@ def transform(wldoc, verbose=False, save_tex=None, morefloats=None,
             if verbose:
                 p = call(['xelatex', tex_path])
             else:
             if verbose:
                 p = call(['xelatex', tex_path])
             else:

-                p = call(['xelatex', '-interaction=batchmode', tex_path], stdout=PIPE, stderr=PIPE)
+                p = call(
+                    ['xelatex', '-interaction=batchmode', tex_path],
+                    stdout=PIPE, stderr=PIPE
+                )

             if p:
                 raise ParseError("Error parsing .tex file")
 
         if cwd is not None:
             os.chdir(cwd)
 
             if p:
                 raise ParseError("Error parsing .tex file")
 
         if cwd is not None:
             os.chdir(cwd)
 

-        output_file = NamedTemporaryFile(prefix='librarian', suffix='.pdf', delete=False)
+        output_file = NamedTemporaryFile(prefix='librarian', suffix='.pdf',
+                                         delete=False)

         pdf_path = os.path.join(temp, 'doc.pdf')
         shutil.move(pdf_path, output_file.name)
         shutil.rmtree(temp)
         pdf_path = os.path.join(temp, 'doc.pdf')
         shutil.move(pdf_path, output_file.name)
         shutil.rmtree(temp)


@@ -353,23 +444,31 @@ def transform(wldoc, verbose=False, save_tex=None, morefloats=None,
 
 def load_including_children(wldoc=None, provider=None, uri=None):
     """ Makes one big xml file with children inserted at end.
 
 def load_including_children(wldoc=None, provider=None, uri=None):
     """ Makes one big xml file with children inserted at end.

-    
+

     Either wldoc or provider and URI must be provided.
     """
 
     if uri and provider:
     Either wldoc or provider and URI must be provided.
     """
 
     if uri and provider:

-        f = provider.by_uri(uri)
+        f = provider.by_slug(uri.slug)

         text = f.read().decode('utf-8')
         f.close()
     elif wldoc is not None:
         text = etree.tostring(wldoc.edoc, encoding='unicode')
         provider = wldoc.provider
     else:
         text = f.read().decode('utf-8')
         f.close()
     elif wldoc is not None:
         text = etree.tostring(wldoc.edoc, encoding='unicode')
         provider = wldoc.provider
     else:

-        raise ValueError('Neither a WLDocument, nor provider and URI were provided.')
+        raise ValueError(
+            'Neither a WLDocument, nor provider and URI were provided.'
+        )

 
 

+    # Cyrrilic

     text = re.sub(r"([\u0400-\u04ff]+)", r"<alien>\1</alien>", text)
     text = re.sub(r"([\u0400-\u04ff]+)", r"<alien>\1</alien>", text)

+    # Geometric shapes.
+    text = re.sub(r"([\u25a0-\u25ff]+)", r"<alien>\1</alien>", text)
+    # Hebrew
+    text = re.sub(r"([\u0590-\u05ff]+)", r"<fallback>\1</fallback>", text)

 
 

-    document = WLDocument.from_bytes(text.encode('utf-8'), parse_dublincore=True, provider=provider)
+    document = WLDocument.from_bytes(text.encode('utf-8'),
+                                     parse_dublincore=True, provider=provider)

     document.swap_endlines()
 
     for child_uri in document.book_info.parts:
     document.swap_endlines()
 
     for child_uri in document.book_info.parts: