- * Words are broken across script boundaries, then segmented according to - * the BreakIterator and typing provided by the {@link ICUTokenizerConfig} - *
- * @see ICUTokenizerConfig - * @lucene.experimental - */ -public final class ICUTokenizer extends Tokenizer { - private static final int IOBUFFER = 4096; - private final char buffer[] = new char[IOBUFFER]; - /** true length of text in the buffer */ - private int length = 0; - /** length in buffer that can be evaluated safely, up to a safe end point */ - private int usableLength = 0; - /** accumulated offset of previous buffers for this reader, for offsetAtt */ - private int offset = 0; - - private final CompositeBreakIterator breaker; /* tokenizes a char[] of text */ - private final ICUTokenizerConfig config; - private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class); - private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); - private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class); - private final ScriptAttribute scriptAtt = addAttribute(ScriptAttribute.class); - - /** - * Construct a new ICUTokenizer that breaks text into words from the given - * Reader. - *- * The default script-specific handling is used. - * - * @param input Reader containing text to tokenize. - * @see DefaultICUTokenizerConfig - */ - public ICUTokenizer(Reader input) { - this(input, new DefaultICUTokenizerConfig()); - } - - /** - * Construct a new ICUTokenizer that breaks text into words from the given - * Reader, using a tailored BreakIterator configuration. - * - * @param input Reader containing text to tokenize. - * @param config Tailored BreakIterator configuration - */ - public ICUTokenizer(Reader input, ICUTokenizerConfig config) { - super(input); - this.config = config; - breaker = new CompositeBreakIterator(config); - } - - @Override - public boolean incrementToken() throws IOException { - clearAttributes(); - if (length == 0) - refill(); - while (!incrementTokenBuffer()) { - refill(); - if (length <= 0) // no more bytes to read; - return false; - } - return true; - } - - @Override - public void reset() throws IOException { - super.reset(); - breaker.setText(buffer, 0, 0); - length = usableLength = offset = 0; - } - - @Override - public void reset(Reader input) throws IOException { - super.reset(input); - reset(); - } - - @Override - public void end() throws IOException { - final int finalOffset = (length < 0) ? offset : offset + length; - offsetAtt.setOffset(finalOffset, finalOffset); - } - - /* - * This tokenizes text based upon the longest matching rule, and because of - * this, isn't friendly to a Reader. - * - * Text is read from the input stream in 4kB chunks. Within a 4kB chunk of - * text, the last unambiguous break point is found (in this implementation: - * white space character) Any remaining characters represent possible partial - * words, so are appended to the front of the next chunk. - * - * There is the possibility that there are no unambiguous break points within - * an entire 4kB chunk of text (binary data). So there is a maximum word limit - * of 4kB since it will not try to grow the buffer in this case. - */ - - /** - * Returns the last unambiguous break position in the text. - * - * @return position of character, or -1 if one does not exist - */ - private int findSafeEnd() { - for (int i = length - 1; i >= 0; i--) - if (UCharacter.isWhitespace(buffer[i])) - return i + 1; - return -1; - } - - /** - * Refill the buffer, accumulating the offset and setting usableLength to the - * last unambiguous break position - * - * @throws IOException - */ - private void refill() throws IOException { - offset += usableLength; - int leftover = length - usableLength; - System.arraycopy(buffer, usableLength, buffer, 0, leftover); - int requested = buffer.length - leftover; - int returned = input.read(buffer, leftover, requested); - length = returned < 0 ? leftover : returned + leftover; - if (returned < requested) /* reader has been emptied, process the rest */ - usableLength = length; - else { /* still more data to be read, find a safe-stopping place */ - usableLength = findSafeEnd(); - if (usableLength < 0) - usableLength = length; /* - * more than IOBUFFER of text without space, - * gonna possibly truncate tokens - */ - } - - breaker.setText(buffer, 0, Math.max(0, usableLength)); - } - - /* - * return true if there is a token from the buffer, or null if it is - * exhausted. - */ - private boolean incrementTokenBuffer() { - int start = breaker.current(); - if (start == BreakIterator.DONE) - return false; // BreakIterator exhausted - - // find the next set of boundaries, skipping over non-tokens (rule status 0) - int end = breaker.next(); - while (start != BreakIterator.DONE && breaker.getRuleStatus() == 0) { - start = end; - end = breaker.next(); - } - - if (start == BreakIterator.DONE) - return false; // BreakIterator exhausted - - termAtt.copyBuffer(buffer, start, end - start); - offsetAtt.setOffset(correctOffset(offset + start), correctOffset(offset + end)); - typeAtt.setType(config.getType(breaker.getScriptCode(), breaker.getRuleStatus())); - scriptAtt.setCode(breaker.getScriptCode()); - - return true; - } -}