3 This is a Pure Python module to hyphenate text.
5 It is inspired by Ruby's Text::Hyphen, but currently reads standard *.dic files,
6 that must be installed separately.
8 In the future it's maybe nice if dictionaries could be distributed together with
9 this module, in a slightly prepared form, like in Ruby's Text::Hyphen.
11 Wilbert Berendsen, March 2008
12 info@wilbertberendsen.nl
17 from __future__ import print_function, unicode_literals
22 __all__ = ("Hyphenator")
24 # cache of per-file Hyph_dict objects
27 # precompile some stuff
28 parse_hex = re.compile(r'\^{2}([0-9a-f]{2})').sub
29 parse = re.compile(r'(\d?)(\D?)').findall
31 def hexrepl(matchObj):
32 return unichr(int(matchObj.group(1), 16))
35 class parse_alt(object):
37 Parse nonstandard hyphen pattern alternative.
38 The instance returns a special int with data about the current position
39 in the pattern when called with an odd value.
41 def __init__(self, pat, alt):
45 self.index = int(alt[1])
46 self.cut = int(alt[2]) + 1
49 self.cut = len(re.sub(r'[\d\.]', '', pat)) + 1
50 if pat.startswith('.'):
53 def __call__(self, val):
57 return dint(val, (self.change, self.index, self.cut))
64 Just an int some other data can be stuck to in a data attribute.
65 Call with ref=other to use the data from the other dint.
67 def __new__(cls, value, data=None, ref=None):
68 obj = int.__new__(cls, value)
69 if ref and type(ref) == dint:
76 class Hyph_dict(object):
78 Reads a hyph_*.dic file and stores the hyphenation patterns.
80 -filename : filename of hyph_*.dic to read
82 def __init__(self, filename):
85 charset = f.readline().strip()
86 if charset.startswith('charset '):
87 charset = charset[8:].strip()
90 pat = pat.decode(charset).strip()
91 if not pat or pat[0] == '%': continue
92 # replace ^^hh with the real character
93 pat = parse_hex(hexrepl, pat)
94 # read nonstandard hyphen alternatives
96 pat, alt = pat.split('/', 1)
97 factory = parse_alt(pat, alt)
100 tag, value = zip(*[(s, factory(i or "0")) for i, s in parse(pat)])
101 # if only zeros, skip this pattern
102 if max(value) == 0: continue
103 # chop zeros from beginning and end, and store start offset.
104 start, end = 0, len(value)
105 while not value[start]: start += 1
106 while not value[end-1]: end -= 1
107 self.patterns[''.join(tag)] = start, value[start:end]
110 self.maxlen = max(map(len, self.patterns.keys()))
112 def positions(self, word):
114 Returns a list of positions where the word can be hyphenated.
115 E.g. for the dutch word 'lettergrepen' this method returns
118 Each position is a 'data int' (dint) with a data attribute.
119 If the data attribute is not None, it contains a tuple with
120 information about nonstandard hyphenation at that point:
123 change: is a string like 'ff=f', that describes how hyphenation
125 index: where to substitute the change, counting from the current
127 cut: how many characters to remove while substituting the nonstandard
131 points = self.cache.get(word)
133 prepWord = '.%s.' % word
134 res = [0] * (len(prepWord) + 1)
135 for i in range(len(prepWord) - 1):
136 for j in range(i + 1, min(i + self.maxlen, len(prepWord)) + 1):
137 p = self.patterns.get(prepWord[i:j])
140 s = slice(i + offset, i + offset + len(value))
141 res[s] = map(max, value, res[s])
143 points = [dint(i - 1, ref=r) for i, r in enumerate(res) if r % 2]
144 self.cache[word] = points
148 class Hyphenator(object):
150 Reads a hyph_*.dic file and stores the hyphenation patterns.
151 Provides methods to hyphenate strings in various ways.
153 -filename : filename of hyph_*.dic to read
154 -left: make the first syllabe not shorter than this
155 -right: make the last syllabe not shorter than this
156 -cache: if true (default), use a cached copy of the dic file, if possible
158 left and right may also later be changed:
162 def __init__(self, filename, left=2, right=2, cache=True):
165 if not cache or filename not in hdcache:
166 hdcache[filename] = Hyph_dict(filename)
167 self.hd = hdcache[filename]
169 def positions(self, word):
171 Returns a list of positions where the word can be hyphenated.
172 See also Hyph_dict.positions. The points that are too far to
173 the left or right are removed.
175 right = len(word) - self.right
176 return [i for i in self.hd.positions(word) if self.left <= i <= right]
178 def iterate(self, word):
180 Iterate over all hyphenation possibilities, the longest first.
182 if isinstance(word, str):
183 word = word.decode('latin1')
184 for p in reversed(self.positions(word)):
186 # get the nonstandard hyphenation data
187 change, index, cut = p.data
189 change = change.upper()
190 c1, c2 = change.split('=')
191 yield word[:p+index] + c1, c2 + word[p+index+cut:]
193 yield word[:p], word[p:]
195 def wrap(self, word, width, hyphen='-'):
197 Return the longest possible first part and the last part of the
198 hyphenated word. The first part has the hyphen already attached.
199 Returns None, if there is no hyphenation point before width, or
200 if the word could not be hyphenated.
203 for w1, w2 in self.iterate(word):
205 return w1 + hyphen, w2
207 def inserted(self, word, hyphen='-'):
209 Returns the word as a string with all the possible hyphens inserted.
210 E.g. for the dutch word 'lettergrepen' this method returns
211 the string 'let-ter-gre-pen'. The hyphen string to use can be
212 given as the second parameter, that defaults to '-'.
214 if isinstance(word, str):
215 word = word.decode('latin1')
217 for p in reversed(self.positions(word)):
219 # get the nonstandard hyphenation data
220 change, index, cut = p.data
222 change = change.upper()
223 l[p + index : p + index + cut] = change.replace('=', hyphen)
231 if __name__ == "__main__":
233 dict_file = sys.argv[1]
234 word = sys.argv[2].decode('latin1')
236 h = Hyphenator(dict_file, left=1, right=1)