+++ /dev/null
-package org.apache.lucene.analysis.icu.segmentation;
-
-/**
- * Copyright (C) 1999-2010, International Business Machines
- * Corporation and others. All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, and/or sell copies of the
- * Software, and to permit persons to whom the Software is furnished to do so,
- * provided that the above copyright notice(s) and this permission notice appear
- * in all copies of the Software and that both the above copyright notice(s) and
- * this permission notice appear in supporting documentation.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS.
- * IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE BE
- * LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR
- * ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER
- * IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
- * OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
- *
- * Except as contained in this notice, the name of a copyright holder shall not
- * be used in advertising or otherwise to promote the sale, use or other
- * dealings in this Software without prior written authorization of the
- * copyright holder.
- */
-
-import com.ibm.icu.lang.UCharacter;
-import com.ibm.icu.lang.UScript;
-import com.ibm.icu.text.UTF16;
-
-/**
- * An iterator that locates ISO 15924 script boundaries in text.
- * <p>
- * This is not the same as simply looking at the Unicode block, or even the
- * Script property. Some characters are 'common' across multiple scripts, and
- * some 'inherit' the script value of text surrounding them.
- * <p>
- * This is similar to ICU (internal-only) UScriptRun, with the following
- * differences:
- * <ul>
- * <li>Doesn't attempt to match paired punctuation. For tokenization purposes, this
- * is not necessary. Its also quite expensive.
- * <li>Non-spacing marks inherit the script of their base character, following
- * recommendations from UTR #24.
- * </ul>
- * @lucene.experimental
- */
-final class ScriptIterator {
- private char text[];
- private int start;
- private int limit;
- private int index;
-
- private int scriptStart;
- private int scriptLimit;
- private int scriptCode;
-
- /**
- * Get the start of this script run
- *
- * @return start position of script run
- */
- int getScriptStart() {
- return scriptStart;
- }
-
- /**
- * Get the index of the first character after the end of this script run
- *
- * @return position of the first character after this script run
- */
- int getScriptLimit() {
- return scriptLimit;
- }
-
- /**
- * Get the UScript script code for this script run
- *
- * @return code for the script of the current run
- */
- int getScriptCode() {
- return scriptCode;
- }
-
- /**
- * Iterates to the next script run, returning true if one exists.
- *
- * @return true if there is another script run, false otherwise.
- */
- boolean next() {
- if (scriptLimit >= limit)
- return false;
-
- scriptCode = UScript.COMMON;
- scriptStart = scriptLimit;
-
- while (index < limit) {
- final int ch = UTF16.charAt(text, start, limit, index - start);
- final int sc = getScript(ch);
-
- /*
- * From UTR #24: Implementations that determine the boundaries between
- * characters of given scripts should never break between a non-spacing
- * mark and its base character. Thus for boundary determinations and
- * similar sorts of processing, a non-spacing mark — whatever its script
- * value — should inherit the script value of its base character.
- */
- if (isSameScript(scriptCode, sc)
- || UCharacter.getType(ch) == UCharacter.NON_SPACING_MARK) {
- index += UTF16.getCharCount(ch);
-
- /*
- * Inherited or Common becomes the script code of the surrounding text.
- */
- if (scriptCode <= UScript.INHERITED && sc > UScript.INHERITED) {
- scriptCode = sc;
- }
-
- } else {
- break;
- }
- }
-
- scriptLimit = index;
- return true;
- }
-
- /** Determine if two scripts are compatible. */
- private static boolean isSameScript(int scriptOne, int scriptTwo) {
- return scriptOne <= UScript.INHERITED || scriptTwo <= UScript.INHERITED
- || scriptOne == scriptTwo;
- }
-
- /**
- * Set a new region of text to be examined by this iterator
- *
- * @param text text buffer to examine
- * @param start offset into buffer
- * @param length maximum length to examine
- */
- void setText(char text[], int start, int length) {
- this.text = text;
- this.start = start;
- this.index = start;
- this.limit = start + length;
- this.scriptStart = start;
- this.scriptLimit = start;
- this.scriptCode = UScript.INVALID_CODE;
- }
-
- /** linear fast-path for basic latin case */
- private static final int basicLatin[] = new int[128];
-
- static {
- for (int i = 0; i < basicLatin.length; i++)
- basicLatin[i] = UScript.getScript(i);
- }
-
- /** fast version of UScript.getScript(). Basic Latin is an array lookup */
- private static int getScript(int codepoint) {
- if (0 <= codepoint && codepoint < basicLatin.length)
- return basicLatin[codepoint];
- else
- return UScript.getScript(codepoint);
- }
-}
fnp / pylucene.git / blobdiff