+++ /dev/null
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.lucene.analysis.cn.smart.hhmm;
-
-import java.util.List;
-
-import org.apache.lucene.analysis.cn.smart.CharType;
-import org.apache.lucene.analysis.cn.smart.Utility;
-import org.apache.lucene.analysis.cn.smart.WordType;
-import org.apache.lucene.analysis.cn.smart.hhmm.SegToken;//javadoc @link
-
-/**
- * Finds the optimal segmentation of a sentence into Chinese words
- * @lucene.experimental
- */
-public class HHMMSegmenter {
-
- private static WordDictionary wordDict = WordDictionary.getInstance();
-
- /**
- * Create the {@link SegGraph} for a sentence.
- *
- * @param sentence input sentence, without start and end markers
- * @return {@link SegGraph} corresponding to the input sentence.
- */
- @SuppressWarnings("fallthrough")
- private SegGraph createSegGraph(String sentence) {
- int i = 0, j;
- int length = sentence.length();
- int foundIndex;
- int[] charTypeArray = getCharTypes(sentence);
- StringBuilder wordBuf = new StringBuilder();
- SegToken token;
- int frequency = 0; // the number of times word appears.
- boolean hasFullWidth;
- int wordType;
- char[] charArray;
-
- SegGraph segGraph = new SegGraph();
- while (i < length) {
- hasFullWidth = false;
- switch (charTypeArray[i]) {
- case CharType.SPACE_LIKE:
- i++;
- break;
- case CharType.HANZI:
- j = i + 1;
- wordBuf.delete(0, wordBuf.length());
- // It doesn't matter if a single Chinese character (Hanzi) can form a phrase or not,
- // it will store that single Chinese character (Hanzi) in the SegGraph. Otherwise, it will
- // cause word division.
- wordBuf.append(sentence.charAt(i));
- charArray = new char[] { sentence.charAt(i) };
- frequency = wordDict.getFrequency(charArray);
- token = new SegToken(charArray, i, j, WordType.CHINESE_WORD,
- frequency);
- segGraph.addToken(token);
-
- foundIndex = wordDict.getPrefixMatch(charArray);
- while (j <= length && foundIndex != -1) {
- if (wordDict.isEqual(charArray, foundIndex) && charArray.length > 1) {
- // It is the phrase we are looking for; In other words, we have found a phrase SegToken
- // from i to j. It is not a monosyllabic word (single word).
- frequency = wordDict.getFrequency(charArray);
- token = new SegToken(charArray, i, j, WordType.CHINESE_WORD,
- frequency);
- segGraph.addToken(token);
- }
-
- while (j < length && charTypeArray[j] == CharType.SPACE_LIKE)
- j++;
-
- if (j < length && charTypeArray[j] == CharType.HANZI) {
- wordBuf.append(sentence.charAt(j));
- charArray = new char[wordBuf.length()];
- wordBuf.getChars(0, charArray.length, charArray, 0);
- // idArray has been found (foundWordIndex!=-1) as a prefix before.
- // Therefore, idArray after it has been lengthened can only appear after foundWordIndex.
- // So start searching after foundWordIndex.
- foundIndex = wordDict.getPrefixMatch(charArray, foundIndex);
- j++;
- } else {
- break;
- }
- }
- i++;
- break;
- case CharType.FULLWIDTH_LETTER:
- hasFullWidth = true; /* intentional fallthrough */
- case CharType.LETTER:
- j = i + 1;
- while (j < length
- && (charTypeArray[j] == CharType.LETTER || charTypeArray[j] == CharType.FULLWIDTH_LETTER)) {
- if (charTypeArray[j] == CharType.FULLWIDTH_LETTER)
- hasFullWidth = true;
- j++;
- }
- // Found a Token from i to j. Type is LETTER char string.
- charArray = Utility.STRING_CHAR_ARRAY;
- frequency = wordDict.getFrequency(charArray);
- wordType = hasFullWidth ? WordType.FULLWIDTH_STRING : WordType.STRING;
- token = new SegToken(charArray, i, j, wordType, frequency);
- segGraph.addToken(token);
- i = j;
- break;
- case CharType.FULLWIDTH_DIGIT:
- hasFullWidth = true; /* intentional fallthrough */
- case CharType.DIGIT:
- j = i + 1;
- while (j < length
- && (charTypeArray[j] == CharType.DIGIT || charTypeArray[j] == CharType.FULLWIDTH_DIGIT)) {
- if (charTypeArray[j] == CharType.FULLWIDTH_DIGIT)
- hasFullWidth = true;
- j++;
- }
- // Found a Token from i to j. Type is NUMBER char string.
- charArray = Utility.NUMBER_CHAR_ARRAY;
- frequency = wordDict.getFrequency(charArray);
- wordType = hasFullWidth ? WordType.FULLWIDTH_NUMBER : WordType.NUMBER;
- token = new SegToken(charArray, i, j, wordType, frequency);
- segGraph.addToken(token);
- i = j;
- break;
- case CharType.DELIMITER:
- j = i + 1;
- // No need to search the weight for the punctuation. Picking the highest frequency will work.
- frequency = Utility.MAX_FREQUENCE;
- charArray = new char[] { sentence.charAt(i) };
- token = new SegToken(charArray, i, j, WordType.DELIMITER, frequency);
- segGraph.addToken(token);
- i = j;
- break;
- default:
- j = i + 1;
- // Treat the unrecognized char symbol as unknown string.
- // For example, any symbol not in GB2312 is treated as one of these.
- charArray = Utility.STRING_CHAR_ARRAY;
- frequency = wordDict.getFrequency(charArray);
- token = new SegToken(charArray, i, j, WordType.STRING, frequency);
- segGraph.addToken(token);
- i = j;
- break;
- }
- }
-
- // Add two more Tokens: "beginning xx beginning"
- charArray = Utility.START_CHAR_ARRAY;
- frequency = wordDict.getFrequency(charArray);
- token = new SegToken(charArray, -1, 0, WordType.SENTENCE_BEGIN, frequency);
- segGraph.addToken(token);
-
- // "end xx end"
- charArray = Utility.END_CHAR_ARRAY;
- frequency = wordDict.getFrequency(charArray);
- token = new SegToken(charArray, length, length + 1, WordType.SENTENCE_END,
- frequency);
- segGraph.addToken(token);
-
- return segGraph;
- }
-
- /**
- * Get the character types for every character in a sentence.
- *
- * @see Utility#getCharType(char)
- * @param sentence input sentence
- * @return array of character types corresponding to character positions in the sentence
- */
- private static int[] getCharTypes(String sentence) {
- int length = sentence.length();
- int[] charTypeArray = new int[length];
- // the type of each character by position
- for (int i = 0; i < length; i++) {
- charTypeArray[i] = Utility.getCharType(sentence.charAt(i));
- }
-
- return charTypeArray;
- }
-
- /**
- * Return a list of {@link SegToken} representing the best segmentation of a sentence
- * @param sentence input sentence
- * @return best segmentation as a {@link List}
- */
- public List<SegToken> process(String sentence) {
- SegGraph segGraph = createSegGraph(sentence);
- BiSegGraph biSegGraph = new BiSegGraph(segGraph);
- List<SegToken> shortPath = biSegGraph.getShortPath();
- return shortPath;
- }
-}
fnp / pylucene.git / blobdiff