--- /dev/null
+package org.apache.lucene.analysis.th;
+
+/**
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Locale;
+import java.lang.Character.UnicodeBlock;
+import java.text.BreakIterator;
+
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.LowerCaseFilter;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.util.CharArrayIterator;
+import org.apache.lucene.util.AttributeSource;
+import org.apache.lucene.util.Version;
+
+/**
+ * {@link TokenFilter} that use {@link java.text.BreakIterator} to break each
+ * Token that is Thai into separate Token(s) for each Thai word.
+ * <p>Please note: Since matchVersion 3.1 on, this filter no longer lowercases non-thai text.
+ * {@link ThaiAnalyzer} will insert a {@link LowerCaseFilter} before this filter
+ * so the behaviour of the Analyzer does not change. With version 3.1, the filter handles
+ * position increments correctly.
+ * <p>WARNING: this filter may not be supported by all JREs.
+ * It is known to work with Sun/Oracle and Harmony JREs.
+ * If your application needs to be fully portable, consider using ICUTokenizer instead,
+ * which uses an ICU Thai BreakIterator that will always be available.
+ */
+public final class ThaiWordFilter extends TokenFilter {
+ /**
+ * True if the JRE supports a working dictionary-based breakiterator for Thai.
+ * If this is false, this filter will not work at all!
+ */
+ public static final boolean DBBI_AVAILABLE;
+ private static final BreakIterator proto = BreakIterator.getWordInstance(new Locale("th"));
+ static {
+ // check that we have a working dictionary-based break iterator for thai
+ proto.setText("ภาษาไทย");
+ DBBI_AVAILABLE = proto.isBoundary(4);
+ }
+ private final BreakIterator breaker = (BreakIterator) proto.clone();
+ private final CharArrayIterator charIterator = CharArrayIterator.newWordInstance();
+
+ private final boolean handlePosIncr;
+
+ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+ private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
+ private final PositionIncrementAttribute posAtt = addAttribute(PositionIncrementAttribute.class);
+
+ private AttributeSource clonedToken = null;
+ private CharTermAttribute clonedTermAtt = null;
+ private OffsetAttribute clonedOffsetAtt = null;
+ private boolean hasMoreTokensInClone = false;
+
+ /** Creates a new ThaiWordFilter that also lowercases non-thai text.
+ * @deprecated Use the ctor with {@code matchVersion} instead!
+ */
+ @Deprecated
+ public ThaiWordFilter(TokenStream input) {
+ this(Version.LUCENE_30, input);
+ }
+
+ /** Creates a new ThaiWordFilter with the specified match version. */
+ public ThaiWordFilter(Version matchVersion, TokenStream input) {
+ super(matchVersion.onOrAfter(Version.LUCENE_31) ?
+ input : new LowerCaseFilter(matchVersion, input));
+ if (!DBBI_AVAILABLE)
+ throw new UnsupportedOperationException("This JRE does not have support for Thai segmentation");
+ handlePosIncr = matchVersion.onOrAfter(Version.LUCENE_31);
+ }
+
+ @Override
+ public boolean incrementToken() throws IOException {
+ if (hasMoreTokensInClone) {
+ int start = breaker.current();
+ int end = breaker.next();
+ if (end != BreakIterator.DONE) {
+ clonedToken.copyTo(this);
+ termAtt.copyBuffer(clonedTermAtt.buffer(), start, end - start);
+ offsetAtt.setOffset(clonedOffsetAtt.startOffset() + start, clonedOffsetAtt.startOffset() + end);
+ if (handlePosIncr) posAtt.setPositionIncrement(1);
+ return true;
+ }
+ hasMoreTokensInClone = false;
+ }
+
+ if (!input.incrementToken()) {
+ return false;
+ }
+
+ if (termAtt.length() == 0 || UnicodeBlock.of(termAtt.charAt(0)) != UnicodeBlock.THAI) {
+ return true;
+ }
+
+ hasMoreTokensInClone = true;
+
+ // we lazy init the cloned token, as in ctor not all attributes may be added
+ if (clonedToken == null) {
+ clonedToken = cloneAttributes();
+ clonedTermAtt = clonedToken.getAttribute(CharTermAttribute.class);
+ clonedOffsetAtt = clonedToken.getAttribute(OffsetAttribute.class);
+ } else {
+ this.copyTo(clonedToken);
+ }
+
+ // reinit CharacterIterator
+ charIterator.setText(clonedTermAtt.buffer(), 0, clonedTermAtt.length());
+ breaker.setText(charIterator);
+ int end = breaker.next();
+ if (end != BreakIterator.DONE) {
+ termAtt.setLength(end);
+ offsetAtt.setOffset(clonedOffsetAtt.startOffset(), clonedOffsetAtt.startOffset() + end);
+ // position increment keeps as it is for first token
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public void reset() throws IOException {
+ super.reset();
+ hasMoreTokensInClone = false;
+ clonedToken = null;
+ clonedTermAtt = null;
+ clonedOffsetAtt = null;
+ }
+}
fnp / pylucene.git / blobdiff