+++ /dev/null
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.lucene.analysis.cn.smart.hhmm;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-import java.io.RandomAccessFile;
-import java.io.UnsupportedEncodingException;
-import java.nio.ByteBuffer;
-import java.nio.ByteOrder;
-
-import org.apache.lucene.analysis.cn.smart.AnalyzerProfile;
-
-/**
- * SmartChineseAnalyzer Bigram dictionary.
- * @lucene.experimental
- */
-class BigramDictionary extends AbstractDictionary {
-
- private BigramDictionary() {
- }
-
- public static final char WORD_SEGMENT_CHAR = '@';
-
- private static BigramDictionary singleInstance;
-
- public static final int PRIME_BIGRAM_LENGTH = 402137;
-
- /*
- * The word associations are stored as FNV1 hashcodes, which have a small probability of collision, but save memory.
- */
- private long[] bigramHashTable;
-
- private int[] frequencyTable;
-
- private int max = 0;
-
- private int repeat = 0;
-
- // static Logger log = Logger.getLogger(BigramDictionary.class);
-
- public synchronized static BigramDictionary getInstance() {
- if (singleInstance == null) {
- singleInstance = new BigramDictionary();
- try {
- singleInstance.load();
- } catch (IOException e) {
- String dictRoot = AnalyzerProfile.ANALYSIS_DATA_DIR;
- singleInstance.load(dictRoot);
- } catch (ClassNotFoundException e) {
- throw new RuntimeException(e);
- }
- }
- return singleInstance;
- }
-
- private boolean loadFromObj(File serialObj) {
- try {
- loadFromInputStream(new FileInputStream(serialObj));
- return true;
- } catch (FileNotFoundException e) {
- e.printStackTrace();
- } catch (IOException e) {
- e.printStackTrace();
- } catch (ClassNotFoundException e) {
- e.printStackTrace();
- }
- return false;
- }
-
- private void loadFromInputStream(InputStream serialObjectInputStream)
- throws IOException, ClassNotFoundException {
- ObjectInputStream input = new ObjectInputStream(serialObjectInputStream);
- bigramHashTable = (long[]) input.readObject();
- frequencyTable = (int[]) input.readObject();
- // log.info("load bigram dict from serialization.");
- input.close();
- }
-
- private void saveToObj(File serialObj) {
- try {
- ObjectOutputStream output = new ObjectOutputStream(new FileOutputStream(
- serialObj));
- output.writeObject(bigramHashTable);
- output.writeObject(frequencyTable);
- output.close();
- // log.info("serialize bigram dict.");
- } catch (Exception e) {
- // log.warn(e.getMessage());
- }
- }
-
- private void load() throws IOException, ClassNotFoundException {
- InputStream input = this.getClass().getResourceAsStream("bigramdict.mem");
- loadFromInputStream(input);
- }
-
- private void load(String dictRoot) {
- String bigramDictPath = dictRoot + "/bigramdict.dct";
-
- File serialObj = new File(dictRoot + "/bigramdict.mem");
-
- if (serialObj.exists() && loadFromObj(serialObj)) {
-
- } else {
- try {
- bigramHashTable = new long[PRIME_BIGRAM_LENGTH];
- frequencyTable = new int[PRIME_BIGRAM_LENGTH];
- for (int i = 0; i < PRIME_BIGRAM_LENGTH; i++) {
- // it is possible for a value to hash to 0, but the probability is extremely low
- bigramHashTable[i] = 0;
- frequencyTable[i] = 0;
- }
- loadFromFile(bigramDictPath);
- } catch (IOException e) {
- throw new RuntimeException(e.getMessage());
- }
- saveToObj(serialObj);
- }
- }
-
- /**
- * Load the datafile into this BigramDictionary
- *
- * @param dctFilePath path to the Bigramdictionary (bigramdict.dct)
- * @throws FileNotFoundException
- * @throws IOException
- * @throws UnsupportedEncodingException
- */
- public void loadFromFile(String dctFilePath) throws FileNotFoundException,
- IOException, UnsupportedEncodingException {
-
- int i, cnt, length, total = 0;
- // The file only counted 6763 Chinese characters plus 5 reserved slots 3756~3760.
- // The 3756th is used (as a header) to store information.
- int[] buffer = new int[3];
- byte[] intBuffer = new byte[4];
- String tmpword;
- RandomAccessFile dctFile = new RandomAccessFile(dctFilePath, "r");
-
- // GB2312 characters 0 - 6768
- for (i = GB2312_FIRST_CHAR; i < GB2312_FIRST_CHAR + CHAR_NUM_IN_FILE; i++) {
- String currentStr = getCCByGB2312Id(i);
- // if (i == 5231)
- // System.out.println(i);
-
- dctFile.read(intBuffer);
- // the dictionary was developed for C, and byte order must be converted to work with Java
- cnt = ByteBuffer.wrap(intBuffer).order(ByteOrder.LITTLE_ENDIAN).getInt();
- if (cnt <= 0) {
- continue;
- }
- total += cnt;
- int j = 0;
- while (j < cnt) {
- dctFile.read(intBuffer);
- buffer[0] = ByteBuffer.wrap(intBuffer).order(ByteOrder.LITTLE_ENDIAN)
- .getInt();// frequency
- dctFile.read(intBuffer);
- buffer[1] = ByteBuffer.wrap(intBuffer).order(ByteOrder.LITTLE_ENDIAN)
- .getInt();// length
- dctFile.read(intBuffer);
- // buffer[2] = ByteBuffer.wrap(intBuffer).order(
- // ByteOrder.LITTLE_ENDIAN).getInt();// handle
-
- length = buffer[1];
- if (length > 0) {
- byte[] lchBuffer = new byte[length];
- dctFile.read(lchBuffer);
- tmpword = new String(lchBuffer, "GB2312");
- if (i != 3755 + GB2312_FIRST_CHAR) {
- tmpword = currentStr + tmpword;
- }
- char carray[] = tmpword.toCharArray();
- long hashId = hash1(carray);
- int index = getAvaliableIndex(hashId, carray);
- if (index != -1) {
- if (bigramHashTable[index] == 0) {
- bigramHashTable[index] = hashId;
- // bigramStringTable[index] = tmpword;
- }
- frequencyTable[index] += buffer[0];
- }
- }
- j++;
- }
- }
- dctFile.close();
- // log.info("load dictionary done! " + dctFilePath + " total:" + total);
- }
-
- private int getAvaliableIndex(long hashId, char carray[]) {
- int hash1 = (int) (hashId % PRIME_BIGRAM_LENGTH);
- int hash2 = hash2(carray) % PRIME_BIGRAM_LENGTH;
- if (hash1 < 0)
- hash1 = PRIME_BIGRAM_LENGTH + hash1;
- if (hash2 < 0)
- hash2 = PRIME_BIGRAM_LENGTH + hash2;
- int index = hash1;
- int i = 1;
- while (bigramHashTable[index] != 0 && bigramHashTable[index] != hashId
- && i < PRIME_BIGRAM_LENGTH) {
- index = (hash1 + i * hash2) % PRIME_BIGRAM_LENGTH;
- i++;
- }
- // System.out.println(i - 1);
-
- if (i < PRIME_BIGRAM_LENGTH
- && (bigramHashTable[index] == 0 || bigramHashTable[index] == hashId)) {
- return index;
- } else
- return -1;
- }
-
- /*
- * lookup the index into the frequency array.
- */
- private int getBigramItemIndex(char carray[]) {
- long hashId = hash1(carray);
- int hash1 = (int) (hashId % PRIME_BIGRAM_LENGTH);
- int hash2 = hash2(carray) % PRIME_BIGRAM_LENGTH;
- if (hash1 < 0)
- hash1 = PRIME_BIGRAM_LENGTH + hash1;
- if (hash2 < 0)
- hash2 = PRIME_BIGRAM_LENGTH + hash2;
- int index = hash1;
- int i = 1;
- repeat++;
- while (bigramHashTable[index] != 0 && bigramHashTable[index] != hashId
- && i < PRIME_BIGRAM_LENGTH) {
- index = (hash1 + i * hash2) % PRIME_BIGRAM_LENGTH;
- i++;
- repeat++;
- if (i > max)
- max = i;
- }
- // System.out.println(i - 1);
-
- if (i < PRIME_BIGRAM_LENGTH && bigramHashTable[index] == hashId) {
- return index;
- } else
- return -1;
- }
-
- public int getFrequency(char[] carray) {
- int index = getBigramItemIndex(carray);
- if (index != -1)
- return frequencyTable[index];
- return 0;
- }
-
-}
fnp / pylucene.git / blobdiff