+++ /dev/null
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.lucene.analysis.cn.smart.hhmm;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-import java.io.RandomAccessFile;
-import java.io.UnsupportedEncodingException;
-import java.nio.ByteBuffer;
-import java.nio.ByteOrder;
-
-import org.apache.lucene.analysis.cn.smart.AnalyzerProfile;
-import org.apache.lucene.analysis.cn.smart.Utility;
-
-/**
- * SmartChineseAnalyzer Word Dictionary
- * @lucene.experimental
- */
-class WordDictionary extends AbstractDictionary {
-
- private WordDictionary() {
- }
-
- private static WordDictionary singleInstance;
-
- /**
- * Large prime number for hash function
- */
- public static final int PRIME_INDEX_LENGTH = 12071;
-
- /**
- * wordIndexTable guarantees to hash all Chinese characters in Unicode into
- * PRIME_INDEX_LENGTH array. There will be conflict, but in reality this
- * program only handles the 6768 characters found in GB2312 plus some
- * ASCII characters. Therefore in order to guarantee better precision, it is
- * necessary to retain the original symbol in the charIndexTable.
- */
- private short[] wordIndexTable;
-
- private char[] charIndexTable;
-
- /**
- * To avoid taking too much space, the data structure needed to store the
- * lexicon requires two multidimensional arrays to store word and frequency.
- * Each word is placed in a char[]. Each char represents a Chinese char or
- * other symbol. Each frequency is put into an int. These two arrays
- * correspond to each other one-to-one. Therefore, one can use
- * wordItem_charArrayTable[i][j] to look up word from lexicon, and
- * wordItem_frequencyTable[i][j] to look up the corresponding frequency.
- */
- private char[][][] wordItem_charArrayTable;
-
- private int[][] wordItem_frequencyTable;
-
- // static Logger log = Logger.getLogger(WordDictionary.class);
-
- /**
- * Get the singleton dictionary instance.
- * @return singleton
- */
- public synchronized static WordDictionary getInstance() {
- if (singleInstance == null) {
- singleInstance = new WordDictionary();
- try {
- singleInstance.load();
- } catch (IOException e) {
- String wordDictRoot = AnalyzerProfile.ANALYSIS_DATA_DIR;
- singleInstance.load(wordDictRoot);
- } catch (ClassNotFoundException e) {
- throw new RuntimeException(e);
- }
- }
- return singleInstance;
- }
-
- /**
- * Attempt to load dictionary from provided directory, first trying coredict.mem, failing back on coredict.dct
- *
- * @param dctFileRoot path to dictionary directory
- */
- public void load(String dctFileRoot) {
- String dctFilePath = dctFileRoot + "/coredict.dct";
- File serialObj = new File(dctFileRoot + "/coredict.mem");
-
- if (serialObj.exists() && loadFromObj(serialObj)) {
-
- } else {
- try {
- wordIndexTable = new short[PRIME_INDEX_LENGTH];
- charIndexTable = new char[PRIME_INDEX_LENGTH];
- for (int i = 0; i < PRIME_INDEX_LENGTH; i++) {
- charIndexTable[i] = 0;
- wordIndexTable[i] = -1;
- }
- wordItem_charArrayTable = new char[GB2312_CHAR_NUM][][];
- wordItem_frequencyTable = new int[GB2312_CHAR_NUM][];
- // int total =
- loadMainDataFromFile(dctFilePath);
- expandDelimiterData();
- mergeSameWords();
- sortEachItems();
- // log.info("load dictionary: " + dctFilePath + " total:" + total);
- } catch (IOException e) {
- throw new RuntimeException(e.getMessage());
- }
-
- saveToObj(serialObj);
- }
-
- }
-
- /**
- * Load coredict.mem internally from the jar file.
- *
- * @throws ClassNotFoundException
- * @throws IOException
- */
- public void load() throws IOException, ClassNotFoundException {
- InputStream input = this.getClass().getResourceAsStream("coredict.mem");
- loadFromObjectInputStream(input);
- }
-
- private boolean loadFromObj(File serialObj) {
- try {
- loadFromObjectInputStream(new FileInputStream(serialObj));
- return true;
- } catch (FileNotFoundException e) {
- e.printStackTrace();
- } catch (IOException e) {
- e.printStackTrace();
- } catch (ClassNotFoundException e) {
- e.printStackTrace();
- }
- return false;
- }
-
- private void loadFromObjectInputStream(InputStream serialObjectInputStream)
- throws IOException, ClassNotFoundException {
- ObjectInputStream input = new ObjectInputStream(serialObjectInputStream);
- wordIndexTable = (short[]) input.readObject();
- charIndexTable = (char[]) input.readObject();
- wordItem_charArrayTable = (char[][][]) input.readObject();
- wordItem_frequencyTable = (int[][]) input.readObject();
- // log.info("load core dict from serialization.");
- input.close();
- }
-
- private void saveToObj(File serialObj) {
- try {
- ObjectOutputStream output = new ObjectOutputStream(new FileOutputStream(
- serialObj));
- output.writeObject(wordIndexTable);
- output.writeObject(charIndexTable);
- output.writeObject(wordItem_charArrayTable);
- output.writeObject(wordItem_frequencyTable);
- output.close();
- // log.info("serialize core dict.");
- } catch (Exception e) {
- // log.warn(e.getMessage());
- }
- }
-
- /**
- * Load the datafile into this WordDictionary
- *
- * @param dctFilePath path to word dictionary (coredict.dct)
- * @return number of words read
- * @throws FileNotFoundException
- * @throws IOException
- * @throws UnsupportedEncodingException
- */
- private int loadMainDataFromFile(String dctFilePath)
- throws FileNotFoundException, IOException, UnsupportedEncodingException {
- int i, cnt, length, total = 0;
- // The file only counted 6763 Chinese characters plus 5 reserved slots 3756~3760.
- // The 3756th is used (as a header) to store information.
- int[] buffer = new int[3];
- byte[] intBuffer = new byte[4];
- String tmpword;
- RandomAccessFile dctFile = new RandomAccessFile(dctFilePath, "r");
-
- // GB2312 characters 0 - 6768
- for (i = GB2312_FIRST_CHAR; i < GB2312_FIRST_CHAR + CHAR_NUM_IN_FILE; i++) {
- // if (i == 5231)
- // System.out.println(i);
-
- dctFile.read(intBuffer);
- // the dictionary was developed for C, and byte order must be converted to work with Java
- cnt = ByteBuffer.wrap(intBuffer).order(ByteOrder.LITTLE_ENDIAN).getInt();
- if (cnt <= 0) {
- wordItem_charArrayTable[i] = null;
- wordItem_frequencyTable[i] = null;
- continue;
- }
- wordItem_charArrayTable[i] = new char[cnt][];
- wordItem_frequencyTable[i] = new int[cnt];
- total += cnt;
- int j = 0;
- while (j < cnt) {
- // wordItemTable[i][j] = new WordItem();
- dctFile.read(intBuffer);
- buffer[0] = ByteBuffer.wrap(intBuffer).order(ByteOrder.LITTLE_ENDIAN)
- .getInt();// frequency
- dctFile.read(intBuffer);
- buffer[1] = ByteBuffer.wrap(intBuffer).order(ByteOrder.LITTLE_ENDIAN)
- .getInt();// length
- dctFile.read(intBuffer);
- buffer[2] = ByteBuffer.wrap(intBuffer).order(ByteOrder.LITTLE_ENDIAN)
- .getInt();// handle
-
- // wordItemTable[i][j].frequency = buffer[0];
- wordItem_frequencyTable[i][j] = buffer[0];
-
- length = buffer[1];
- if (length > 0) {
- byte[] lchBuffer = new byte[length];
- dctFile.read(lchBuffer);
- tmpword = new String(lchBuffer, "GB2312");
- // indexTable[i].wordItems[j].word = tmpword;
- // wordItemTable[i][j].charArray = tmpword.toCharArray();
- wordItem_charArrayTable[i][j] = tmpword.toCharArray();
- } else {
- // wordItemTable[i][j].charArray = null;
- wordItem_charArrayTable[i][j] = null;
- }
- // System.out.println(indexTable[i].wordItems[j]);
- j++;
- }
-
- String str = getCCByGB2312Id(i);
- setTableIndex(str.charAt(0), i);
- }
- dctFile.close();
- return total;
- }
-
- /**
- * The original lexicon puts all information with punctuation into a
- * chart (from 1 to 3755). Here it then gets expanded, separately being
- * placed into the chart that has the corresponding symbol.
- */
- private void expandDelimiterData() {
- int i;
- int cnt;
- // Punctuation then treating index 3755 as 1,
- // distribute the original punctuation corresponding dictionary into
- int delimiterIndex = 3755 + GB2312_FIRST_CHAR;
- i = 0;
- while (i < wordItem_charArrayTable[delimiterIndex].length) {
- char c = wordItem_charArrayTable[delimiterIndex][i][0];
- int j = getGB2312Id(c);// the id value of the punctuation
- if (wordItem_charArrayTable[j] == null) {
-
- int k = i;
- // Starting from i, count the number of the following worditem symbol from j
- while (k < wordItem_charArrayTable[delimiterIndex].length
- && wordItem_charArrayTable[delimiterIndex][k][0] == c) {
- k++;
- }
- // c is the punctuation character, j is the id value of c
- // k-1 represents the index of the last punctuation character
- cnt = k - i;
- if (cnt != 0) {
- wordItem_charArrayTable[j] = new char[cnt][];
- wordItem_frequencyTable[j] = new int[cnt];
- }
-
- // Assign value for each wordItem.
- for (k = 0; k < cnt; k++, i++) {
- // wordItemTable[j][k] = new WordItem();
- wordItem_frequencyTable[j][k] = wordItem_frequencyTable[delimiterIndex][i];
- wordItem_charArrayTable[j][k] = new char[wordItem_charArrayTable[delimiterIndex][i].length - 1];
- System.arraycopy(wordItem_charArrayTable[delimiterIndex][i], 1,
- wordItem_charArrayTable[j][k], 0,
- wordItem_charArrayTable[j][k].length);
- }
- setTableIndex(c, j);
- }
- }
- // Delete the original corresponding symbol array.
- wordItem_charArrayTable[delimiterIndex] = null;
- wordItem_frequencyTable[delimiterIndex] = null;
- }
-
- /*
- * since we aren't doing POS-tagging, merge the frequencies for entries of the same word (with different POS)
- */
- private void mergeSameWords() {
- int i;
- for (i = 0; i < GB2312_FIRST_CHAR + CHAR_NUM_IN_FILE; i++) {
- if (wordItem_charArrayTable[i] == null)
- continue;
- int len = 1;
- for (int j = 1; j < wordItem_charArrayTable[i].length; j++) {
- if (Utility.compareArray(wordItem_charArrayTable[i][j], 0,
- wordItem_charArrayTable[i][j - 1], 0) != 0)
- len++;
-
- }
- if (len < wordItem_charArrayTable[i].length) {
- char[][] tempArray = new char[len][];
- int[] tempFreq = new int[len];
- int k = 0;
- tempArray[0] = wordItem_charArrayTable[i][0];
- tempFreq[0] = wordItem_frequencyTable[i][0];
- for (int j = 1; j < wordItem_charArrayTable[i].length; j++) {
- if (Utility.compareArray(wordItem_charArrayTable[i][j], 0,
- tempArray[k], 0) != 0) {
- k++;
- // temp[k] = wordItemTable[i][j];
- tempArray[k] = wordItem_charArrayTable[i][j];
- tempFreq[k] = wordItem_frequencyTable[i][j];
- } else {
- // temp[k].frequency += wordItemTable[i][j].frequency;
- tempFreq[k] += wordItem_frequencyTable[i][j];
- }
- }
- // wordItemTable[i] = temp;
- wordItem_charArrayTable[i] = tempArray;
- wordItem_frequencyTable[i] = tempFreq;
- }
- }
- }
-
- private void sortEachItems() {
- char[] tmpArray;
- int tmpFreq;
- for (int i = 0; i < wordItem_charArrayTable.length; i++) {
- if (wordItem_charArrayTable[i] != null
- && wordItem_charArrayTable[i].length > 1) {
- for (int j = 0; j < wordItem_charArrayTable[i].length - 1; j++) {
- for (int j2 = j + 1; j2 < wordItem_charArrayTable[i].length; j2++) {
- if (Utility.compareArray(wordItem_charArrayTable[i][j], 0,
- wordItem_charArrayTable[i][j2], 0) > 0) {
- tmpArray = wordItem_charArrayTable[i][j];
- tmpFreq = wordItem_frequencyTable[i][j];
- wordItem_charArrayTable[i][j] = wordItem_charArrayTable[i][j2];
- wordItem_frequencyTable[i][j] = wordItem_frequencyTable[i][j2];
- wordItem_charArrayTable[i][j2] = tmpArray;
- wordItem_frequencyTable[i][j2] = tmpFreq;
- }
- }
- }
- }
- }
- }
-
- /*
- * Calculate character c's position in hash table,
- * then initialize the value of that position in the address table.
- */
- private boolean setTableIndex(char c, int j) {
- int index = getAvaliableTableIndex(c);
- if (index != -1) {
- charIndexTable[index] = c;
- wordIndexTable[index] = (short) j;
- return true;
- } else
- return false;
- }
-
- private short getAvaliableTableIndex(char c) {
- int hash1 = (int) (hash1(c) % PRIME_INDEX_LENGTH);
- int hash2 = hash2(c) % PRIME_INDEX_LENGTH;
- if (hash1 < 0)
- hash1 = PRIME_INDEX_LENGTH + hash1;
- if (hash2 < 0)
- hash2 = PRIME_INDEX_LENGTH + hash2;
- int index = hash1;
- int i = 1;
- while (charIndexTable[index] != 0 && charIndexTable[index] != c
- && i < PRIME_INDEX_LENGTH) {
- index = (hash1 + i * hash2) % PRIME_INDEX_LENGTH;
- i++;
- }
- // System.out.println(i - 1);
-
- if (i < PRIME_INDEX_LENGTH
- && (charIndexTable[index] == 0 || charIndexTable[index] == c)) {
- return (short) index;
- } else
- return -1;
- }
-
- private short getWordItemTableIndex(char c) {
- int hash1 = (int) (hash1(c) % PRIME_INDEX_LENGTH);
- int hash2 = hash2(c) % PRIME_INDEX_LENGTH;
- if (hash1 < 0)
- hash1 = PRIME_INDEX_LENGTH + hash1;
- if (hash2 < 0)
- hash2 = PRIME_INDEX_LENGTH + hash2;
- int index = hash1;
- int i = 1;
- while (charIndexTable[index] != 0 && charIndexTable[index] != c
- && i < PRIME_INDEX_LENGTH) {
- index = (hash1 + i * hash2) % PRIME_INDEX_LENGTH;
- i++;
- }
-
- if (i < PRIME_INDEX_LENGTH && charIndexTable[index] == c) {
- return (short) index;
- } else
- return -1;
- }
-
- /**
- * Look up the text string corresponding with the word char array,
- * and return the position of the word list.
- *
- * @param knownHashIndex already figure out position of the first word
- * symbol charArray[0] in hash table. If not calculated yet, can be
- * replaced with function int findInTable(char[] charArray).
- * @param charArray look up the char array corresponding with the word.
- * @return word location in word array. If not found, then return -1.
- */
- private int findInTable(short knownHashIndex, char[] charArray) {
- if (charArray == null || charArray.length == 0)
- return -1;
-
- char[][] items = wordItem_charArrayTable[wordIndexTable[knownHashIndex]];
- int start = 0, end = items.length - 1;
- int mid = (start + end) / 2, cmpResult;
-
- // Binary search for the index of idArray
- while (start <= end) {
- cmpResult = Utility.compareArray(items[mid], 0, charArray, 1);
-
- if (cmpResult == 0)
- return mid;// find it
- else if (cmpResult < 0)
- start = mid + 1;
- else if (cmpResult > 0)
- end = mid - 1;
-
- mid = (start + end) / 2;
- }
- return -1;
- }
-
- /**
- * Find the first word in the dictionary that starts with the supplied prefix
- *
- * @see #getPrefixMatch(char[], int)
- * @param charArray input prefix
- * @return index of word, or -1 if not found
- */
- public int getPrefixMatch(char[] charArray) {
- return getPrefixMatch(charArray, 0);
- }
-
- /**
- * Find the nth word in the dictionary that starts with the supplied prefix
- *
- * @see #getPrefixMatch(char[])
- * @param charArray input prefix
- * @param knownStart relative position in the dictionary to start
- * @return index of word, or -1 if not found
- */
- public int getPrefixMatch(char[] charArray, int knownStart) {
- short index = getWordItemTableIndex(charArray[0]);
- if (index == -1)
- return -1;
- char[][] items = wordItem_charArrayTable[wordIndexTable[index]];
- int start = knownStart, end = items.length - 1;
-
- int mid = (start + end) / 2, cmpResult;
-
- // Binary search for the index of idArray
- while (start <= end) {
- cmpResult = Utility.compareArrayByPrefix(charArray, 1, items[mid], 0);
- if (cmpResult == 0) {
- // Get the first item which match the current word
- while (mid >= 0
- && Utility.compareArrayByPrefix(charArray, 1, items[mid], 0) == 0)
- mid--;
- mid++;
- return mid;// Find the first word that uses charArray as prefix.
- } else if (cmpResult < 0)
- end = mid - 1;
- else
- start = mid + 1;
- mid = (start + end) / 2;
- }
- return -1;
- }
-
- /**
- * Get the frequency of a word from the dictionary
- *
- * @param charArray input word
- * @return word frequency, or zero if the word is not found
- */
- public int getFrequency(char[] charArray) {
- short hashIndex = getWordItemTableIndex(charArray[0]);
- if (hashIndex == -1)
- return 0;
- int itemIndex = findInTable(hashIndex, charArray);
- if (itemIndex != -1)
- return wordItem_frequencyTable[wordIndexTable[hashIndex]][itemIndex];
- return 0;
-
- }
-
- /**
- * Return true if the dictionary entry at itemIndex for table charArray[0] is charArray
- *
- * @param charArray input word
- * @param itemIndex item index for table charArray[0]
- * @return true if the entry exists
- */
- public boolean isEqual(char[] charArray, int itemIndex) {
- short hashIndex = getWordItemTableIndex(charArray[0]);
- return Utility.compareArray(charArray, 1,
- wordItem_charArrayTable[wordIndexTable[hashIndex]][itemIndex], 0) == 0;
- }
-
-}
fnp / pylucene.git / blobdiff