+++ /dev/null
-package org.apache.lucene.analysis.cjk;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.IOException;
-import java.io.Reader;
-
-import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
-import org.apache.lucene.util.AttributeSource;
-
-/**
- * CJKTokenizer is designed for Chinese, Japanese, and Korean languages.
- * <p>
- * The tokens returned are every two adjacent characters with overlap match.
- * </p>
- * <p>
- * Example: "java C1C2C3C4" will be segmented to: "java" "C1C2" "C2C3" "C3C4".
- * </p>
- * Additionally, the following is applied to Latin text (such as English):
- * <ul>
- * <li>Text is converted to lowercase.
- * <li>Numeric digits, '+', '#', and '_' are tokenized as letters.
- * <li>Full-width forms are converted to half-width forms.
- * </ul>
- * For more info on Asian language (Chinese, Japanese, and Korean) text segmentation:
- * please search <a
- * href="http://www.google.com/search?q=word+chinese+segment">google</a>
- *
- */
-public final class CJKTokenizer extends Tokenizer {
- //~ Static fields/initializers ---------------------------------------------
- /** Word token type */
- static final int WORD_TYPE = 0;
-
- /** Single byte token type */
- static final int SINGLE_TOKEN_TYPE = 1;
-
- /** Double byte token type */
- static final int DOUBLE_TOKEN_TYPE = 2;
-
- /** Names for token types */
- static final String[] TOKEN_TYPE_NAMES = { "word", "single", "double" };
-
- /** Max word length */
- private static final int MAX_WORD_LEN = 255;
-
- /** buffer size: */
- private static final int IO_BUFFER_SIZE = 256;
-
- //~ Instance fields --------------------------------------------------------
-
- /** word offset, used to imply which character(in ) is parsed */
- private int offset = 0;
-
- /** the index used only for ioBuffer */
- private int bufferIndex = 0;
-
- /** data length */
- private int dataLen = 0;
-
- /**
- * character buffer, store the characters which are used to compose <br>
- * the returned Token
- */
- private final char[] buffer = new char[MAX_WORD_LEN];
-
- /**
- * I/O buffer, used to store the content of the input(one of the <br>
- * members of Tokenizer)
- */
- private final char[] ioBuffer = new char[IO_BUFFER_SIZE];
-
- /** word type: single=>ASCII double=>non-ASCII word=>default */
- private int tokenType = WORD_TYPE;
-
- /**
- * tag: previous character is a cached double-byte character "C1C2C3C4"
- * ----(set the C1 isTokened) C1C2 "C2C3C4" ----(set the C2 isTokened)
- * C1C2 C2C3 "C3C4" ----(set the C3 isTokened) "C1C2 C2C3 C3C4"
- */
- private boolean preIsTokened = false;
-
- private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
- private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
- private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
-
- //~ Constructors -----------------------------------------------------------
-
- /**
- * Construct a token stream processing the given input.
- *
- * @param in I/O reader
- */
- public CJKTokenizer(Reader in) {
- super(in);
- }
-
- public CJKTokenizer(AttributeSource source, Reader in) {
- super(source, in);
- }
-
- public CJKTokenizer(AttributeFactory factory, Reader in) {
- super(factory, in);
- }
-
- //~ Methods ----------------------------------------------------------------
-
- /**
- * Returns true for the next token in the stream, or false at EOS.
- * See http://java.sun.com/j2se/1.3/docs/api/java/lang/Character.UnicodeBlock.html
- * for detail.
- *
- * @return false for end of stream, true otherwise
- *
- * @throws java.io.IOException - throw IOException when read error <br>
- * happened in the InputStream
- *
- */
- @Override
- public boolean incrementToken() throws IOException {
- clearAttributes();
- /** how many character(s) has been stored in buffer */
-
- while(true) { // loop until we find a non-empty token
-
- int length = 0;
-
- /** the position used to create Token */
- int start = offset;
-
- while (true) { // loop until we've found a full token
- /** current character */
- char c;
-
- /** unicode block of current character for detail */
- Character.UnicodeBlock ub;
-
- offset++;
-
- if (bufferIndex >= dataLen) {
- dataLen = input.read(ioBuffer);
- bufferIndex = 0;
- }
-
- if (dataLen == -1) {
- if (length > 0) {
- if (preIsTokened == true) {
- length = 0;
- preIsTokened = false;
- }
- else{
- offset--;
- }
-
- break;
- } else {
- offset--;
- return false;
- }
- } else {
- //get current character
- c = ioBuffer[bufferIndex++];
-
- //get the UnicodeBlock of the current character
- ub = Character.UnicodeBlock.of(c);
- }
-
- //if the current character is ASCII or Extend ASCII
- if ((ub == Character.UnicodeBlock.BASIC_LATIN)
- || (ub == Character.UnicodeBlock.HALFWIDTH_AND_FULLWIDTH_FORMS)
- ) {
- if (ub == Character.UnicodeBlock.HALFWIDTH_AND_FULLWIDTH_FORMS) {
- int i = (int) c;
- if (i >= 65281 && i <= 65374) {
- // convert certain HALFWIDTH_AND_FULLWIDTH_FORMS to BASIC_LATIN
- i = i - 65248;
- c = (char) i;
- }
- }
-
- // if the current character is a letter or "_" "+" "#"
- if (Character.isLetterOrDigit(c)
- || ((c == '_') || (c == '+') || (c == '#'))
- ) {
- if (length == 0) {
- // "javaC1C2C3C4linux" <br>
- // ^--: the current character begin to token the ASCII
- // letter
- start = offset - 1;
- } else if (tokenType == DOUBLE_TOKEN_TYPE) {
- // "javaC1C2C3C4linux" <br>
- // ^--: the previous non-ASCII
- // : the current character
- offset--;
- bufferIndex--;
-
- if (preIsTokened == true) {
- // there is only one non-ASCII has been stored
- length = 0;
- preIsTokened = false;
- break;
- } else {
- break;
- }
- }
-
- // store the LowerCase(c) in the buffer
- buffer[length++] = Character.toLowerCase(c);
- tokenType = SINGLE_TOKEN_TYPE;
-
- // break the procedure if buffer overflowed!
- if (length == MAX_WORD_LEN) {
- break;
- }
- } else if (length > 0) {
- if (preIsTokened == true) {
- length = 0;
- preIsTokened = false;
- } else {
- break;
- }
- }
- } else {
- // non-ASCII letter, e.g."C1C2C3C4"
- if (Character.isLetter(c)) {
- if (length == 0) {
- start = offset - 1;
- buffer[length++] = c;
- tokenType = DOUBLE_TOKEN_TYPE;
- } else {
- if (tokenType == SINGLE_TOKEN_TYPE) {
- offset--;
- bufferIndex--;
-
- //return the previous ASCII characters
- break;
- } else {
- buffer[length++] = c;
- tokenType = DOUBLE_TOKEN_TYPE;
-
- if (length == 2) {
- offset--;
- bufferIndex--;
- preIsTokened = true;
-
- break;
- }
- }
- }
- } else if (length > 0) {
- if (preIsTokened == true) {
- // empty the buffer
- length = 0;
- preIsTokened = false;
- } else {
- break;
- }
- }
- }
- }
-
- if (length > 0) {
- termAtt.copyBuffer(buffer, 0, length);
- offsetAtt.setOffset(correctOffset(start), correctOffset(start+length));
- typeAtt.setType(TOKEN_TYPE_NAMES[tokenType]);
- return true;
- } else if (dataLen == -1) {
- offset--;
- return false;
- }
-
- // Cycle back and try for the next token (don't
- // return an empty string)
- }
- }
-
- @Override
- public final void end() {
- // set final offset
- final int finalOffset = correctOffset(offset);
- this.offsetAtt.setOffset(finalOffset, finalOffset);
- }
-
- @Override
- public void reset() throws IOException {
- super.reset();
- offset = bufferIndex = dataLen = 0;
- preIsTokened = false;
- tokenType = WORD_TYPE;
- }
-
- @Override
- public void reset(Reader reader) throws IOException {
- super.reset(reader);
- reset();
- }
-}
fnp / pylucene.git / blobdiff