Please note: Since matchVersion 3.1 on, this filter no longer lowercases non-thai text. - * {@link ThaiAnalyzer} will insert a {@link LowerCaseFilter} before this filter - * so the behaviour of the Analyzer does not change. With version 3.1, the filter handles - * position increments correctly. - *
WARNING: this filter may not be supported by all JREs. - * It is known to work with Sun/Oracle and Harmony JREs. - * If your application needs to be fully portable, consider using ICUTokenizer instead, - * which uses an ICU Thai BreakIterator that will always be available. - */ -public final class ThaiWordFilter extends TokenFilter { - /** - * True if the JRE supports a working dictionary-based breakiterator for Thai. - * If this is false, this filter will not work at all! - */ - public static final boolean DBBI_AVAILABLE; - private static final BreakIterator proto = BreakIterator.getWordInstance(new Locale("th")); - static { - // check that we have a working dictionary-based break iterator for thai - proto.setText("ภาษาà¹à¸à¸¢"); - DBBI_AVAILABLE = proto.isBoundary(4); - } - private final BreakIterator breaker = (BreakIterator) proto.clone(); - private final Segment charIterator = new Segment(); - - private final boolean handlePosIncr; - - private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); - private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class); - private final PositionIncrementAttribute posAtt = addAttribute(PositionIncrementAttribute.class); - - private AttributeSource clonedToken = null; - private CharTermAttribute clonedTermAtt = null; - private OffsetAttribute clonedOffsetAtt = null; - private boolean hasMoreTokensInClone = false; - - /** Creates a new ThaiWordFilter that also lowercases non-thai text. - * @deprecated Use the ctor with {@code matchVersion} instead! - */ - @Deprecated - public ThaiWordFilter(TokenStream input) { - this(Version.LUCENE_30, input); - } - - /** Creates a new ThaiWordFilter with the specified match version. */ - public ThaiWordFilter(Version matchVersion, TokenStream input) { - super(matchVersion.onOrAfter(Version.LUCENE_31) ? - input : new LowerCaseFilter(matchVersion, input)); - if (!DBBI_AVAILABLE) - throw new UnsupportedOperationException("This JRE does not have support for Thai segmentation"); - handlePosIncr = matchVersion.onOrAfter(Version.LUCENE_31); - } - - @Override - public boolean incrementToken() throws IOException { - if (hasMoreTokensInClone) { - int start = breaker.current(); - int end = breaker.next(); - if (end != BreakIterator.DONE) { - clonedToken.copyTo(this); - termAtt.copyBuffer(clonedTermAtt.buffer(), start, end - start); - offsetAtt.setOffset(clonedOffsetAtt.startOffset() + start, clonedOffsetAtt.startOffset() + end); - if (handlePosIncr) posAtt.setPositionIncrement(1); - return true; - } - hasMoreTokensInClone = false; - } - - if (!input.incrementToken()) { - return false; - } - - if (termAtt.length() == 0 || UnicodeBlock.of(termAtt.charAt(0)) != UnicodeBlock.THAI) { - return true; - } - - hasMoreTokensInClone = true; - - // we lazy init the cloned token, as in ctor not all attributes may be added - if (clonedToken == null) { - clonedToken = cloneAttributes(); - clonedTermAtt = clonedToken.getAttribute(CharTermAttribute.class); - clonedOffsetAtt = clonedToken.getAttribute(OffsetAttribute.class); - } else { - this.copyTo(clonedToken); - } - - // reinit CharacterIterator - charIterator.array = clonedTermAtt.buffer(); - charIterator.offset = 0; - charIterator.count = clonedTermAtt.length(); - breaker.setText(charIterator); - int end = breaker.next(); - if (end != BreakIterator.DONE) { - termAtt.setLength(end); - offsetAtt.setOffset(clonedOffsetAtt.startOffset(), clonedOffsetAtt.startOffset() + end); - // position increment keeps as it is for first token - return true; - } - return false; - } - - @Override - public void reset() throws IOException { - super.reset(); - hasMoreTokensInClone = false; - clonedToken = null; - clonedTermAtt = null; - clonedOffsetAtt = null; - } -}