--- /dev/null
+package org.apache.lucene.analysis.icu.segmentation;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.text.CharacterIterator;
+
+import com.ibm.icu.lang.UCharacter;
+import com.ibm.icu.text.BreakIterator;
+import com.ibm.icu.text.RuleBasedBreakIterator;
+import com.ibm.icu.text.UnicodeSet;
+
+/**
+ * Syllable iterator for Lao text.
+ * <p>
+ * This breaks Lao text into syllables according to:
+ * <i>Syllabification of Lao Script for Line Breaking</i>
+ * Phonpasit Phissamay, Valaxay Dalolay, Chitaphone Chanhsililath, Oulaiphone Silimasak,
+ * Sarmad Hussain, Nadir Durrani, Science Technology and Environment Agency, CRULP.
+ * <ul>
+ * <li>http://www.panl10n.net/english/final%20reports/pdf%20files/Laos/LAO06.pdf
+ * <li>http://www.panl10n.net/Presentations/Cambodia/Phonpassit/LineBreakingAlgo.pdf
+ * </ul>
+ * <p>
+ * Most work is accomplished with RBBI rules, however some additional special logic is needed
+ * that cannot be coded in a grammar, and this is implemented here.
+ * <p>
+ * For example, what appears to be a final consonant might instead be part of the next syllable.
+ * Rules match in a greedy fashion, leaving an illegal sequence that matches no rules.
+ * <p>
+ * Take for instance the text ກວ່າດອກ
+ * The first rule greedily matches ກວ່າດ, but then ອກ is encountered, which is illegal.
+ * What LaoBreakIterator does, according to the paper:
+ * <ol>
+ * <li>backtrack and remove the ດ from the last syllable, placing it on the current syllable.
+ * <li>verify the modified previous syllable (ກວ່າ ) is still legal.
+ * <li>verify the modified current syllable (ດອກ) is now legal.
+ * <li>If 2 or 3 fails, then restore the ດ to the last syllable and skip the current character.
+ * </ol>
+ * <p>
+ * Finally, LaoBreakIterator also takes care of the second concern mentioned in the paper.
+ * This is the issue of combining marks being in the wrong order (typos).
+ * @lucene.experimental
+ */
+public class LaoBreakIterator extends BreakIterator {
+ RuleBasedBreakIterator rules;
+ CharArrayIterator text;
+
+ CharArrayIterator working = new CharArrayIterator();
+ int workingOffset = 0;
+
+ CharArrayIterator verifyText = new CharArrayIterator();
+ RuleBasedBreakIterator verify;
+
+ private static final UnicodeSet laoSet;
+ static {
+ laoSet = new UnicodeSet("[:Lao:]");
+ laoSet.compact();
+ laoSet.freeze();
+ }
+
+ public LaoBreakIterator(RuleBasedBreakIterator rules) {
+ this.rules = (RuleBasedBreakIterator) rules.clone();
+ this.verify = (RuleBasedBreakIterator) rules.clone();
+ }
+
+ @Override
+ public int current() {
+ int current = rules.current();
+ return current == BreakIterator.DONE ? BreakIterator.DONE : workingOffset + current;
+ }
+
+ @Override
+ public int first() {
+ working.setText(this.text.getText(), this.text.getStart(), this.text.getLength());
+ rules.setText(working);
+ workingOffset = 0;
+ int first = rules.first();
+ return first == BreakIterator.DONE ? BreakIterator.DONE : workingOffset + first;
+ }
+
+ @Override
+ public int following(int offset) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public CharacterIterator getText() {
+ return text;
+ }
+
+ @Override
+ public int last() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int next() {
+ int current = current();
+ int next = rules.next();
+ if (next == BreakIterator.DONE)
+ return next;
+ else
+ next += workingOffset;
+
+ char c = working.current();
+ int following = rules.next(); // lookahead
+ if (following != BreakIterator.DONE) {
+ following += workingOffset;
+ if (rules.getRuleStatus() == 0 && laoSet.contains(c) && verifyPushBack(current, next)) {
+ workingOffset = next - 1;
+ working.setText(text.getText(), text.getStart() + workingOffset, text.getLength() - workingOffset);
+ return next - 1;
+ }
+ rules.previous(); // undo the lookahead
+ }
+
+ return next;
+ }
+
+ @Override
+ public int next(int n) {
+ if (n < 0)
+ throw new UnsupportedOperationException("Backwards traversal is unsupported");
+
+ int result = current();
+ while (n > 0) {
+ result = next();
+ --n;
+ }
+ return result;
+ }
+
+ @Override
+ public int previous() {
+ throw new UnsupportedOperationException("Backwards traversal is unsupported");
+ }
+
+ @Override
+ public void setText(CharacterIterator text) {
+ if (!(text instanceof CharArrayIterator))
+ throw new UnsupportedOperationException("unsupported CharacterIterator");
+ this.text = (CharArrayIterator) text;
+ ccReorder(this.text.getText(), this.text.getStart(), this.text.getLength());
+ working.setText(this.text.getText(), this.text.getStart(), this.text.getLength());
+ rules.setText(working);
+ workingOffset = 0;
+ }
+
+ @Override
+ public void setText(String newText) {
+ CharArrayIterator ci = new CharArrayIterator();
+ ci.setText(newText.toCharArray(), 0, newText.length());
+ setText(ci);
+ }
+
+ private boolean verifyPushBack(int current, int next) {
+ int shortenedSyllable = next - current - 1;
+
+ verifyText.setText(text.getText(), text.getStart() + current, shortenedSyllable);
+ verify.setText(verifyText);
+ if (verify.next() != shortenedSyllable || verify.getRuleStatus() == 0)
+ return false;
+
+
+ verifyText.setText(text.getText(), text.getStart() + next - 1, text.getLength() - next + 1);
+ verify.setText(verifyText);
+
+ return (verify.next() != BreakIterator.DONE && verify.getRuleStatus() != 0);
+ }
+
+ // TODO: only bubblesort around runs of combining marks, instead of the entire text.
+ private void ccReorder(char[] text, int start, int length) {
+ boolean reordered;
+ do {
+ int prevCC = 0;
+ reordered = false;
+ for (int i = start; i < start + length; i++) {
+ final char c = text[i];
+ final int cc = UCharacter.getCombiningClass(c);
+ if (cc > 0 && cc < prevCC) {
+ // swap
+ text[i] = text[i - 1];
+ text[i - 1] = c;
+ reordered = true;
+ } else {
+ prevCC = cc;
+ }
+ }
+
+ } while (reordered == true);
+ }
+
+ /**
+ * Clone method. Creates another LaoBreakIterator with the same behavior
+ * and current state as this one.
+ * @return The clone.
+ */
+ @Override
+ public Object clone() {
+ LaoBreakIterator other = (LaoBreakIterator) super.clone();
+ other.rules = (RuleBasedBreakIterator) rules.clone();
+ other.verify = (RuleBasedBreakIterator) verify.clone();
+ if (text != null)
+ other.text = (CharArrayIterator) text.clone();
+ if (working != null)
+ other.working = (CharArrayIterator) working.clone();
+ if (verifyText != null)
+ other.verifyText = (CharArrayIterator) verifyText.clone();
+ return other;
+ }
+}
fnp / pylucene.git / blobdiff