+++ /dev/null
-package org.apache.lucene.analysis.icu.segmentation;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.text.CharacterIterator;
-
-import com.ibm.icu.lang.UCharacter;
-import com.ibm.icu.text.BreakIterator;
-import com.ibm.icu.text.DictionaryBasedBreakIterator;
-import com.ibm.icu.text.RuleBasedBreakIterator;
-import com.ibm.icu.text.UTF16;
-
-/**
- * Contain all the issues surrounding BreakIterators in ICU in one place.
- * Basically this boils down to the fact that they aren't very friendly to any
- * sort of OO design.
- * <p>
- * http://bugs.icu-project.org/trac/ticket/5901: RBBI.getRuleStatus(), hoist to
- * BreakIterator from RuleBasedBreakIterator
- * <p>
- * DictionaryBasedBreakIterator is a subclass of RuleBasedBreakIterator, but
- * doesn't actually behave as a subclass: it always returns 0 for
- * getRuleStatus():
- * http://bugs.icu-project.org/trac/ticket/4730: Thai RBBI, no boundary type
- * tags
- * @lucene.experimental
- */
-abstract class BreakIteratorWrapper {
- protected final CharArrayIterator textIterator = new CharArrayIterator();
- protected char text[];
- protected int start;
- protected int length;
-
- abstract int next();
- abstract int current();
- abstract int getRuleStatus();
- abstract void setText(CharacterIterator text);
-
- void setText(char text[], int start, int length) {
- this.text = text;
- this.start = start;
- this.length = length;
- textIterator.setText(text, start, length);
- setText(textIterator);
- }
-
- /**
- * If its a DictionaryBasedBreakIterator, it doesn't return rulestatus, so
- * treat it like a generic BreakIterator If its any other
- * RuleBasedBreakIterator, the rule status can be used for token type. If its
- * any other BreakIterator, the rulestatus method is not available, so treat
- * it like a generic BreakIterator.
- */
- static BreakIteratorWrapper wrap(BreakIterator breakIterator) {
- if (breakIterator instanceof RuleBasedBreakIterator
- && !(breakIterator instanceof DictionaryBasedBreakIterator))
- return new RBBIWrapper((RuleBasedBreakIterator) breakIterator);
- else
- return new BIWrapper(breakIterator);
- }
-
- /**
- * RuleBasedBreakIterator wrapper: RuleBasedBreakIterator (as long as its not
- * a DictionaryBasedBreakIterator) behaves correctly.
- */
- static final class RBBIWrapper extends BreakIteratorWrapper {
- private final RuleBasedBreakIterator rbbi;
-
- RBBIWrapper(RuleBasedBreakIterator rbbi) {
- this.rbbi = rbbi;
- }
-
- @Override
- int current() {
- return rbbi.current();
- }
-
- @Override
- int getRuleStatus() {
- return rbbi.getRuleStatus();
- }
-
- @Override
- int next() {
- return rbbi.next();
- }
-
- @Override
- void setText(CharacterIterator text) {
- rbbi.setText(text);
- }
- }
-
- /**
- * Generic BreakIterator wrapper: Either the rulestatus method is not
- * available or always returns 0. Calculate a rulestatus here so it behaves
- * like RuleBasedBreakIterator.
- *
- * Note: This is slower than RuleBasedBreakIterator.
- */
- static final class BIWrapper extends BreakIteratorWrapper {
- private final BreakIterator bi;
- private int status;
-
- BIWrapper(BreakIterator bi) {
- this.bi = bi;
- }
-
- @Override
- int current() {
- return bi.current();
- }
-
- @Override
- int getRuleStatus() {
- return status;
- }
-
- @Override
- int next() {
- int current = bi.current();
- int next = bi.next();
- status = calcStatus(current, next);
- return next;
- }
-
- private int calcStatus(int current, int next) {
- if (current == BreakIterator.DONE || next == BreakIterator.DONE)
- return RuleBasedBreakIterator.WORD_NONE;
-
- int begin = start + current;
- int end = start + next;
-
- int codepoint;
- for (int i = begin; i < end; i += UTF16.getCharCount(codepoint)) {
- codepoint = UTF16.charAt(text, 0, end, begin);
-
- if (UCharacter.isDigit(codepoint))
- return RuleBasedBreakIterator.WORD_NUMBER;
- else if (UCharacter.isLetter(codepoint)) {
- // TODO: try to separately specify ideographic, kana?
- // [currently all bundled as letter for this case]
- return RuleBasedBreakIterator.WORD_LETTER;
- }
- }
-
- return RuleBasedBreakIterator.WORD_NONE;
- }
-
- @Override
- void setText(CharacterIterator text) {
- bi.setText(text);
- status = RuleBasedBreakIterator.WORD_NONE;
- }
- }
-}
fnp / pylucene.git / blobdiff