Please note: Since matchVersion 3.1 on, this filter no longer lowercases non-thai text. + * {@link ThaiAnalyzer} will insert a {@link LowerCaseFilter} before this filter + * so the behaviour of the Analyzer does not change. With version 3.1, the filter handles + * position increments correctly. + *
WARNING: this filter may not be supported by all JREs. + * It is known to work with Sun/Oracle and Harmony JREs. + * If your application needs to be fully portable, consider using ICUTokenizer instead, + * which uses an ICU Thai BreakIterator that will always be available. + */ +public final class ThaiWordFilter extends TokenFilter { + /** + * True if the JRE supports a working dictionary-based breakiterator for Thai. + * If this is false, this filter will not work at all! + */ + public static final boolean DBBI_AVAILABLE; + private static final BreakIterator proto = BreakIterator.getWordInstance(new Locale("th")); + static { + // check that we have a working dictionary-based break iterator for thai + proto.setText("ภาษาà¹à¸à¸¢"); + DBBI_AVAILABLE = proto.isBoundary(4); + } + private final BreakIterator breaker = (BreakIterator) proto.clone(); + private final CharArrayIterator charIterator = CharArrayIterator.newWordInstance(); + + private final boolean handlePosIncr; + + private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); + private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class); + private final PositionIncrementAttribute posAtt = addAttribute(PositionIncrementAttribute.class); + + private AttributeSource clonedToken = null; + private CharTermAttribute clonedTermAtt = null; + private OffsetAttribute clonedOffsetAtt = null; + private boolean hasMoreTokensInClone = false; + + /** Creates a new ThaiWordFilter that also lowercases non-thai text. + * @deprecated Use the ctor with {@code matchVersion} instead! + */ + @Deprecated + public ThaiWordFilter(TokenStream input) { + this(Version.LUCENE_30, input); + } + + /** Creates a new ThaiWordFilter with the specified match version. */ + public ThaiWordFilter(Version matchVersion, TokenStream input) { + super(matchVersion.onOrAfter(Version.LUCENE_31) ? + input : new LowerCaseFilter(matchVersion, input)); + if (!DBBI_AVAILABLE) + throw new UnsupportedOperationException("This JRE does not have support for Thai segmentation"); + handlePosIncr = matchVersion.onOrAfter(Version.LUCENE_31); + } + + @Override + public boolean incrementToken() throws IOException { + if (hasMoreTokensInClone) { + int start = breaker.current(); + int end = breaker.next(); + if (end != BreakIterator.DONE) { + clonedToken.copyTo(this); + termAtt.copyBuffer(clonedTermAtt.buffer(), start, end - start); + offsetAtt.setOffset(clonedOffsetAtt.startOffset() + start, clonedOffsetAtt.startOffset() + end); + if (handlePosIncr) posAtt.setPositionIncrement(1); + return true; + } + hasMoreTokensInClone = false; + } + + if (!input.incrementToken()) { + return false; + } + + if (termAtt.length() == 0 || UnicodeBlock.of(termAtt.charAt(0)) != UnicodeBlock.THAI) { + return true; + } + + hasMoreTokensInClone = true; + + // we lazy init the cloned token, as in ctor not all attributes may be added + if (clonedToken == null) { + clonedToken = cloneAttributes(); + clonedTermAtt = clonedToken.getAttribute(CharTermAttribute.class); + clonedOffsetAtt = clonedToken.getAttribute(OffsetAttribute.class); + } else { + this.copyTo(clonedToken); + } + + // reinit CharacterIterator + charIterator.setText(clonedTermAtt.buffer(), 0, clonedTermAtt.length()); + breaker.setText(charIterator); + int end = breaker.next(); + if (end != BreakIterator.DONE) { + termAtt.setLength(end); + offsetAtt.setOffset(clonedOffsetAtt.startOffset(), clonedOffsetAtt.startOffset() + end); + // position increment keeps as it is for first token + return true; + } + return false; + } + + @Override + public void reset() throws IOException { + super.reset(); + hasMoreTokensInClone = false; + clonedToken = null; + clonedTermAtt = null; + clonedOffsetAtt = null; + } +}