1 package org.apache.lucene.analysis.th;
4 * Copyright 2006 The Apache Software Foundation
6 * Licensed under the Apache License, Version 2.0 (the "License");
7 * you may not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
10 * http://www.apache.org/licenses/LICENSE-2.0
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
19 import java.io.IOException;
20 import java.util.Locale;
21 import java.lang.Character.UnicodeBlock;
22 import java.text.BreakIterator;
24 import org.apache.lucene.analysis.TokenFilter;
25 import org.apache.lucene.analysis.TokenStream;
26 import org.apache.lucene.analysis.LowerCaseFilter;
27 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
28 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
29 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
30 import org.apache.lucene.analysis.util.CharArrayIterator;
31 import org.apache.lucene.util.AttributeSource;
32 import org.apache.lucene.util.Version;
35 * {@link TokenFilter} that use {@link java.text.BreakIterator} to break each
36 * Token that is Thai into separate Token(s) for each Thai word.
37 * <p>Please note: Since matchVersion 3.1 on, this filter no longer lowercases non-thai text.
38 * {@link ThaiAnalyzer} will insert a {@link LowerCaseFilter} before this filter
39 * so the behaviour of the Analyzer does not change. With version 3.1, the filter handles
40 * position increments correctly.
41 * <p>WARNING: this filter may not be supported by all JREs.
42 * It is known to work with Sun/Oracle and Harmony JREs.
43 * If your application needs to be fully portable, consider using ICUTokenizer instead,
44 * which uses an ICU Thai BreakIterator that will always be available.
46 public final class ThaiWordFilter extends TokenFilter {
48 * True if the JRE supports a working dictionary-based breakiterator for Thai.
49 * If this is false, this filter will not work at all!
51 public static final boolean DBBI_AVAILABLE;
52 private static final BreakIterator proto = BreakIterator.getWordInstance(new Locale("th"));
54 // check that we have a working dictionary-based break iterator for thai
55 proto.setText("ภาษาไทย");
56 DBBI_AVAILABLE = proto.isBoundary(4);
58 private final BreakIterator breaker = (BreakIterator) proto.clone();
59 private final CharArrayIterator charIterator = CharArrayIterator.newWordInstance();
61 private final boolean handlePosIncr;
63 private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
64 private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
65 private final PositionIncrementAttribute posAtt = addAttribute(PositionIncrementAttribute.class);
67 private AttributeSource clonedToken = null;
68 private CharTermAttribute clonedTermAtt = null;
69 private OffsetAttribute clonedOffsetAtt = null;
70 private boolean hasMoreTokensInClone = false;
72 /** Creates a new ThaiWordFilter that also lowercases non-thai text.
73 * @deprecated Use the ctor with {@code matchVersion} instead!
76 public ThaiWordFilter(TokenStream input) {
77 this(Version.LUCENE_30, input);
80 /** Creates a new ThaiWordFilter with the specified match version. */
81 public ThaiWordFilter(Version matchVersion, TokenStream input) {
82 super(matchVersion.onOrAfter(Version.LUCENE_31) ?
83 input : new LowerCaseFilter(matchVersion, input));
85 throw new UnsupportedOperationException("This JRE does not have support for Thai segmentation");
86 handlePosIncr = matchVersion.onOrAfter(Version.LUCENE_31);
90 public boolean incrementToken() throws IOException {
91 if (hasMoreTokensInClone) {
92 int start = breaker.current();
93 int end = breaker.next();
94 if (end != BreakIterator.DONE) {
95 clonedToken.copyTo(this);
96 termAtt.copyBuffer(clonedTermAtt.buffer(), start, end - start);
97 offsetAtt.setOffset(clonedOffsetAtt.startOffset() + start, clonedOffsetAtt.startOffset() + end);
98 if (handlePosIncr) posAtt.setPositionIncrement(1);
101 hasMoreTokensInClone = false;
104 if (!input.incrementToken()) {
108 if (termAtt.length() == 0 || UnicodeBlock.of(termAtt.charAt(0)) != UnicodeBlock.THAI) {
112 hasMoreTokensInClone = true;
114 // we lazy init the cloned token, as in ctor not all attributes may be added
115 if (clonedToken == null) {
116 clonedToken = cloneAttributes();
117 clonedTermAtt = clonedToken.getAttribute(CharTermAttribute.class);
118 clonedOffsetAtt = clonedToken.getAttribute(OffsetAttribute.class);
120 this.copyTo(clonedToken);
123 // reinit CharacterIterator
124 charIterator.setText(clonedTermAtt.buffer(), 0, clonedTermAtt.length());
125 breaker.setText(charIterator);
126 int end = breaker.next();
127 if (end != BreakIterator.DONE) {
128 termAtt.setLength(end);
129 offsetAtt.setOffset(clonedOffsetAtt.startOffset(), clonedOffsetAtt.startOffset() + end);
130 // position increment keeps as it is for first token
137 public void reset() throws IOException {
139 hasMoreTokensInClone = false;
141 clonedTermAtt = null;
142 clonedOffsetAtt = null;