+ * SmartChineseAnalyzer abstract dictionary implementation. + *
+ *+ * Contains methods for dealing with GB2312 encoding. + *
+ * @lucene.experimental + */ +abstract class AbstractDictionary { + /** + * First Chinese Character in GB2312 (15 * 94) + * Characters in GB2312 are arranged in a grid of 94 * 94, 0-14 are unassigned or punctuation. + */ + public static final int GB2312_FIRST_CHAR = 1410; + + /** + * Last Chinese Character in GB2312 (87 * 94). + * Characters in GB2312 are arranged in a grid of 94 * 94, 88-94 are unassigned. + */ + public static final int GB2312_CHAR_NUM = 87 * 94; + + /** + * Dictionary data contains 6768 Chinese characters with frequency statistics. + */ + public static final int CHAR_NUM_IN_FILE = 6768; + + // ===================================================== + // code +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +A +B +C +D +E +F + // B0A0 å é¿ å æ¨ å å å ç ç è¼ ç® è¾ ç¢ ç± é + // B0B0 é æ°¨ å® ä¿º æ æ 岸 èº æ¡ è® æ ç å¹ æ ç¬ ç¿± + // B0C0 è¢ å² å¥¥ æ æ¾³ è æ æ å å§ ç¬ å « ç¤ å·´ æ è· + // B0D0 é¶ æ è å é¸ ç½¢ ç¸ ç½ æ ç¾ æ ä½° è´¥ æ ç¨ æ + // B0E0 ç æ¬ æ³ è¬ é¢ æ¿ ç æ® æ ä¼´ ç£ å å ç» é¦ å¸® + // B0F0 æ¢ æ¦ è ç» æ£ ç£ è é å è°¤ è è å è¤ å¥ + // ===================================================== + // + // GB2312 character setï¼ + // 01 94 Symbols + // 02 72 Numbers + // 03 94 Latin + // 04 83 Kana + // 05 86 Katakana + // 06 48 Greek + // 07 66 Cyrillic + // 08 63 Phonetic Symbols + // 09 76 Drawing Symbols + // 10-15 Unassigned + // 16-55 3755 Plane 1, in pinyin order + // 56-87 3008 Plane 2, in radical/stroke order + // 88-94 Unassigned + // ====================================================== + + /** + *+ * Transcode from GB2312 ID to Unicode + *
+ *+ * GB2312 is divided into a 94 * 94 grid, containing 7445 characters consisting of 6763 Chinese characters and 682 symbols. + * Some regions are unassigned (reserved). + *
+ * + * @param ccid GB2312 id + * @return unicode String + */ + public String getCCByGB2312Id(int ccid) { + if (ccid < 0 || ccid > WordDictionary.GB2312_CHAR_NUM) + return ""; + int cc1 = ccid / 94 + 161; + int cc2 = ccid % 94 + 161; + byte[] buffer = new byte[2]; + buffer[0] = (byte) cc1; + buffer[1] = (byte) cc2; + try { + String cchar = new String(buffer, "GB2312"); + return cchar; + } catch (UnsupportedEncodingException e) { + return ""; + } + } + + /** + * Transcode from Unicode to GB2312 + * + * @param ch input character in Unicode, or character in Basic Latin range. + * @return position in GB2312 + */ + public short getGB2312Id(char ch) { + try { + byte[] buffer = Character.toString(ch).getBytes("GB2312"); + if (buffer.length != 2) { + // Should be a two-byte character + return -1; + } + int b0 = (buffer[0] & 0x0FF) - 161; // Code starts from A1, therefore subtract 0xA1=161 + int b1 = (buffer[1] & 0x0FF) - 161; // There is no Chinese char for the first and last symbol. + // Therefore, each code page only has 16*6-2=94 characters. + return (short) (b0 * 94 + b1); + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + } + return -1; + } + + /** + * 32-bit FNV Hash Function + * + * @param c input character + * @return hashcode + */ + public long hash1(char c) { + final long p = 1099511628211L; + long hash = 0xcbf29ce484222325L; + hash = (hash ^ (c & 0x00FF)) * p; + hash = (hash ^ (c >> 8)) * p; + hash += hash << 13; + hash ^= hash >> 7; + hash += hash << 3; + hash ^= hash >> 17; + hash += hash << 5; + return hash; + } + + /** + * 32-bit FNV Hash Function + * + * @param carray character array + * @return hashcode + */ + public long hash1(char carray[]) { + final long p = 1099511628211L; + long hash = 0xcbf29ce484222325L; + for (int i = 0; i < carray.length; i++) { + char d = carray[i]; + hash = (hash ^ (d & 0x00FF)) * p; + hash = (hash ^ (d >> 8)) * p; + } + + // hash += hash << 13; + // hash ^= hash >> 7; + // hash += hash << 3; + // hash ^= hash >> 17; + // hash += hash << 5; + return hash; + } + + /** + * djb2 hash algorithmï¼this algorithm (k=33) was first reported by dan + * bernstein many years ago in comp.lang.c. another version of this algorithm + * (now favored by bernstein) uses xor: hash(i) = hash(i - 1) * 33 ^ str[i]; + * the magic of number 33 (why it works better than many other constants, + * prime or not) has never been adequately explained. + * + * @param c character + * @return hashcode + */ + public int hash2(char c) { + int hash = 5381; + + /* hash 33 + c */ + hash = ((hash << 5) + hash) + c & 0x00FF; + hash = ((hash << 5) + hash) + c >> 8; + + return hash; + } + + /** + * djb2 hash algorithmï¼this algorithm (k=33) was first reported by dan + * bernstein many years ago in comp.lang.c. another version of this algorithm + * (now favored by bernstein) uses xor: hash(i) = hash(i - 1) * 33 ^ str[i]; + * the magic of number 33 (why it works better than many other constants, + * prime or not) has never been adequately explained. + * + * @param carray character array + * @return hashcode + */ + public int hash2(char carray[]) { + int hash = 5381; + + /* hash 33 + c */ + for (int i = 0; i < carray.length; i++) { + char d = carray[i]; + hash = ((hash << 5) + hash) + d & 0x00FF; + hash = ((hash << 5) + hash) + d >> 8; + } + + return hash; + } +}