+++ /dev/null
-"""
-
-This is a Pure Python module to hyphenate text.
-
-It is inspired by Ruby's Text::Hyphen, but currently reads standard *.dic files,
-that must be installed separately.
-
-In the future it's maybe nice if dictionaries could be distributed together with
-this module, in a slightly prepared form, like in Ruby's Text::Hyphen.
-
-Wilbert Berendsen, March 2008
-info@wilbertberendsen.nl
-
-License: LGPL.
-
-"""
-
-import sys
-import re
-
-__all__ = ("Hyphenator")
-
-# cache of per-file Hyph_dict objects
-hdcache = {}
-
-# precompile some stuff
-parse_hex = re.compile(r'\^{2}([0-9a-f]{2})').sub
-parse = re.compile(r'(\d?)(\D?)').findall
-
-def hexrepl(matchObj):
- return unichr(int(matchObj.group(1), 16))
-
-
-class parse_alt(object):
- """
- Parse nonstandard hyphen pattern alternative.
- The instance returns a special int with data about the current position
- in the pattern when called with an odd value.
- """
- def __init__(self, pat, alt):
- alt = alt.split(',')
- self.change = alt[0]
- if len(alt) > 2:
- self.index = int(alt[1])
- self.cut = int(alt[2]) + 1
- else:
- self.index = 1
- self.cut = len(re.sub(r'[\d\.]', '', pat)) + 1
- if pat.startswith('.'):
- self.index += 1
-
- def __call__(self, val):
- self.index -= 1
- val = int(val)
- if val & 1:
- return dint(val, (self.change, self.index, self.cut))
- else:
- return val
-
-
-class dint(int):
- """
- Just an int some other data can be stuck to in a data attribute.
- Call with ref=other to use the data from the other dint.
- """
- def __new__(cls, value, data=None, ref=None):
- obj = int.__new__(cls, value)
- if ref and type(ref) == dint:
- obj.data = ref.data
- else:
- obj.data = data
- return obj
-
-
-class Hyph_dict(object):
- """
- Reads a hyph_*.dic file and stores the hyphenation patterns.
- Parameters:
- -filename : filename of hyph_*.dic to read
- """
- def __init__(self, filename):
- self.patterns = {}
- f = open(filename)
- charset = f.readline().strip()
- if charset.startswith('charset '):
- charset = charset[8:].strip()
-
- for pat in f:
- pat = pat.decode(charset).strip()
- if not pat or pat[0] == '%': continue
- # replace ^^hh with the real character
- pat = parse_hex(hexrepl, pat)
- # read nonstandard hyphen alternatives
- if '/' in pat:
- pat, alt = pat.split('/', 1)
- factory = parse_alt(pat, alt)
- else:
- factory = int
- tag, value = zip(*[(s, factory(i or "0")) for i, s in parse(pat)])
- # if only zeros, skip this pattern
- if max(value) == 0: continue
- # chop zeros from beginning and end, and store start offset.
- start, end = 0, len(value)
- while not value[start]: start += 1
- while not value[end-1]: end -= 1
- self.patterns[''.join(tag)] = start, value[start:end]
- f.close()
- self.cache = {}
- self.maxlen = max(map(len, self.patterns.keys()))
-
- def positions(self, word):
- """
- Returns a list of positions where the word can be hyphenated.
- E.g. for the dutch word 'lettergrepen' this method returns
- the list [3, 6, 9].
-
- Each position is a 'data int' (dint) with a data attribute.
- If the data attribute is not None, it contains a tuple with
- information about nonstandard hyphenation at that point:
- (change, index, cut)
-
- change: is a string like 'ff=f', that describes how hyphenation
- should take place.
- index: where to substitute the change, counting from the current
- point
- cut: how many characters to remove while substituting the nonstandard
- hyphenation
- """
- word = word.lower()
- points = self.cache.get(word)
- if points is None:
- prepWord = '.%s.' % word
- res = [0] * (len(prepWord) + 1)
- for i in range(len(prepWord) - 1):
- for j in range(i + 1, min(i + self.maxlen, len(prepWord)) + 1):
- p = self.patterns.get(prepWord[i:j])
- if p:
- offset, value = p
- s = slice(i + offset, i + offset + len(value))
- res[s] = map(max, value, res[s])
-
- points = [dint(i - 1, ref=r) for i, r in enumerate(res) if r % 2]
- self.cache[word] = points
- return points
-
-
-class Hyphenator(object):
- """
- Reads a hyph_*.dic file and stores the hyphenation patterns.
- Provides methods to hyphenate strings in various ways.
- Parameters:
- -filename : filename of hyph_*.dic to read
- -left: make the first syllabe not shorter than this
- -right: make the last syllabe not shorter than this
- -cache: if true (default), use a cached copy of the dic file, if possible
-
- left and right may also later be changed:
- h = Hyphenator(file)
- h.left = 1
- """
- def __init__(self, filename, left=2, right=2, cache=True):
- self.left = left
- self.right = right
- if not cache or filename not in hdcache:
- hdcache[filename] = Hyph_dict(filename)
- self.hd = hdcache[filename]
-
- def positions(self, word):
- """
- Returns a list of positions where the word can be hyphenated.
- See also Hyph_dict.positions. The points that are too far to
- the left or right are removed.
- """
- right = len(word) - self.right
- return [i for i in self.hd.positions(word) if self.left <= i <= right]
-
- def iterate(self, word):
- """
- Iterate over all hyphenation possibilities, the longest first.
- """
- if isinstance(word, str):
- word = word.decode('latin1')
- for p in reversed(self.positions(word)):
- if p.data:
- # get the nonstandard hyphenation data
- change, index, cut = p.data
- if word.isupper():
- change = change.upper()
- c1, c2 = change.split('=')
- yield word[:p+index] + c1, c2 + word[p+index+cut:]
- else:
- yield word[:p], word[p:]
-
- def wrap(self, word, width, hyphen='-'):
- """
- Return the longest possible first part and the last part of the
- hyphenated word. The first part has the hyphen already attached.
- Returns None, if there is no hyphenation point before width, or
- if the word could not be hyphenated.
- """
- width -= len(hyphen)
- for w1, w2 in self.iterate(word):
- if len(w1) <= width:
- return w1 + hyphen, w2
-
- def inserted(self, word, hyphen='-'):
- """
- Returns the word as a string with all the possible hyphens inserted.
- E.g. for the dutch word 'lettergrepen' this method returns
- the string 'let-ter-gre-pen'. The hyphen string to use can be
- given as the second parameter, that defaults to '-'.
- """
- if isinstance(word, str):
- word = word.decode('latin1')
- l = list(word)
- for p in reversed(self.positions(word)):
- if p.data:
- # get the nonstandard hyphenation data
- change, index, cut = p.data
- if word.isupper():
- change = change.upper()
- l[p + index : p + index + cut] = change.replace('=', hyphen)
- else:
- l.insert(p, hyphen)
- return ''.join(l)
-
- __call__ = iterate
-
-
-if __name__ == "__main__":
-
- dict_file = sys.argv[1]
- word = sys.argv[2].decode('latin1')
-
- h = Hyphenator(dict_file, left=1, right=1)
-
- for i in h(word):
- print i
-
fnp / librarian.git / blobdiff