--- /dev/null
+package org.apache.lucene.analysis.icu.segmentation;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.io.Reader;
+
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.icu.tokenattributes.ScriptAttribute;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
+
+import com.ibm.icu.lang.UCharacter;
+import com.ibm.icu.text.BreakIterator;
+
+/**
+ * Breaks text into words according to UAX #29: Unicode Text Segmentation
+ * (http://www.unicode.org/reports/tr29/)
+ * <p>
+ * Words are broken across script boundaries, then segmented according to
+ * the BreakIterator and typing provided by the {@link ICUTokenizerConfig}
+ * </p>
+ * @see ICUTokenizerConfig
+ * @lucene.experimental
+ */
+public final class ICUTokenizer extends Tokenizer {
+ private static final int IOBUFFER = 4096;
+ private final char buffer[] = new char[IOBUFFER];
+ /** true length of text in the buffer */
+ private int length = 0;
+ /** length in buffer that can be evaluated safely, up to a safe end point */
+ private int usableLength = 0;
+ /** accumulated offset of previous buffers for this reader, for offsetAtt */
+ private int offset = 0;
+
+ private final CompositeBreakIterator breaker; /* tokenizes a char[] of text */
+ private final ICUTokenizerConfig config;
+ private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
+ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+ private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
+ private final ScriptAttribute scriptAtt = addAttribute(ScriptAttribute.class);
+
+ /**
+ * Construct a new ICUTokenizer that breaks text into words from the given
+ * Reader.
+ * <p>
+ * The default script-specific handling is used.
+ *
+ * @param input Reader containing text to tokenize.
+ * @see DefaultICUTokenizerConfig
+ */
+ public ICUTokenizer(Reader input) {
+ this(input, new DefaultICUTokenizerConfig());
+ }
+
+ /**
+ * Construct a new ICUTokenizer that breaks text into words from the given
+ * Reader, using a tailored BreakIterator configuration.
+ *
+ * @param input Reader containing text to tokenize.
+ * @param config Tailored BreakIterator configuration
+ */
+ public ICUTokenizer(Reader input, ICUTokenizerConfig config) {
+ super(input);
+ this.config = config;
+ breaker = new CompositeBreakIterator(config);
+ }
+
+ @Override
+ public boolean incrementToken() throws IOException {
+ clearAttributes();
+ if (length == 0)
+ refill();
+ while (!incrementTokenBuffer()) {
+ refill();
+ if (length <= 0) // no more bytes to read;
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public void reset() throws IOException {
+ super.reset();
+ breaker.setText(buffer, 0, 0);
+ length = usableLength = offset = 0;
+ }
+
+ @Override
+ public void reset(Reader input) throws IOException {
+ super.reset(input);
+ reset();
+ }
+
+ @Override
+ public void end() throws IOException {
+ final int finalOffset = (length < 0) ? offset : offset + length;
+ offsetAtt.setOffset(finalOffset, finalOffset);
+ }
+
+ /*
+ * This tokenizes text based upon the longest matching rule, and because of
+ * this, isn't friendly to a Reader.
+ *
+ * Text is read from the input stream in 4kB chunks. Within a 4kB chunk of
+ * text, the last unambiguous break point is found (in this implementation:
+ * white space character) Any remaining characters represent possible partial
+ * words, so are appended to the front of the next chunk.
+ *
+ * There is the possibility that there are no unambiguous break points within
+ * an entire 4kB chunk of text (binary data). So there is a maximum word limit
+ * of 4kB since it will not try to grow the buffer in this case.
+ */
+
+ /**
+ * Returns the last unambiguous break position in the text.
+ *
+ * @return position of character, or -1 if one does not exist
+ */
+ private int findSafeEnd() {
+ for (int i = length - 1; i >= 0; i--)
+ if (UCharacter.isWhitespace(buffer[i]))
+ return i + 1;
+ return -1;
+ }
+
+ /**
+ * Refill the buffer, accumulating the offset and setting usableLength to the
+ * last unambiguous break position
+ *
+ * @throws IOException
+ */
+ private void refill() throws IOException {
+ offset += usableLength;
+ int leftover = length - usableLength;
+ System.arraycopy(buffer, usableLength, buffer, 0, leftover);
+ int requested = buffer.length - leftover;
+ int returned = input.read(buffer, leftover, requested);
+ length = returned < 0 ? leftover : returned + leftover;
+ if (returned < requested) /* reader has been emptied, process the rest */
+ usableLength = length;
+ else { /* still more data to be read, find a safe-stopping place */
+ usableLength = findSafeEnd();
+ if (usableLength < 0)
+ usableLength = length; /*
+ * more than IOBUFFER of text without space,
+ * gonna possibly truncate tokens
+ */
+ }
+
+ breaker.setText(buffer, 0, Math.max(0, usableLength));
+ }
+
+ /*
+ * return true if there is a token from the buffer, or null if it is
+ * exhausted.
+ */
+ private boolean incrementTokenBuffer() {
+ int start = breaker.current();
+ if (start == BreakIterator.DONE)
+ return false; // BreakIterator exhausted
+
+ // find the next set of boundaries, skipping over non-tokens (rule status 0)
+ int end = breaker.next();
+ while (start != BreakIterator.DONE && breaker.getRuleStatus() == 0) {
+ start = end;
+ end = breaker.next();
+ }
+
+ if (start == BreakIterator.DONE)
+ return false; // BreakIterator exhausted
+
+ termAtt.copyBuffer(buffer, start, end - start);
+ offsetAtt.setOffset(correctOffset(offset + start), correctOffset(offset + end));
+ typeAtt.setType(config.getType(breaker.getScriptCode(), breaker.getRuleStatus()));
+ scriptAtt.setCode(breaker.getScriptCode());
+
+ return true;
+ }
+}
fnp / pylucene.git / blobdiff