--- /dev/null
+package org.apache.lucene.analysis.in;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.BitSet;
+import java.util.IdentityHashMap;
+import static java.lang.Character.UnicodeBlock.*;
+import static org.apache.lucene.analysis.util.StemmerUtil.*;
+
+/**
+ * Normalizes the Unicode representation of text in Indian languages.
+ * <p>
+ * Follows guidelines from Unicode 5.2, chapter 6, South Asian Scripts I
+ * and graphical decompositions from http://ldc.upenn.edu/myl/IndianScriptsUnicode.html
+ * </p>
+ */
+public class IndicNormalizer {
+
+ private static class ScriptData {
+ final int flag;
+ final int base;
+ BitSet decompMask;
+
+ ScriptData(int flag, int base) {
+ this.flag = flag;
+ this.base = base;
+ }
+ }
+
+ private static final IdentityHashMap<Character.UnicodeBlock,ScriptData> scripts =
+ new IdentityHashMap<Character.UnicodeBlock,ScriptData>(9);
+
+ private static int flag(Character.UnicodeBlock ub) {
+ return scripts.get(ub).flag;
+ }
+
+ static {
+ scripts.put(DEVANAGARI, new ScriptData(1, 0x0900));
+ scripts.put(BENGALI, new ScriptData(2, 0x0980));
+ scripts.put(GURMUKHI, new ScriptData(4, 0x0A00));
+ scripts.put(GUJARATI, new ScriptData(8, 0x0A80));
+ scripts.put(ORIYA, new ScriptData(16, 0x0B00));
+ scripts.put(TAMIL, new ScriptData(32, 0x0B80));
+ scripts.put(TELUGU, new ScriptData(64, 0x0C00));
+ scripts.put(KANNADA, new ScriptData(128, 0x0C80));
+ scripts.put(MALAYALAM, new ScriptData(256, 0x0D00));
+ }
+
+ /**
+ * Decompositions according to Unicode 5.2,
+ * and http://ldc.upenn.edu/myl/IndianScriptsUnicode.html
+ *
+ * Most of these are not handled by unicode normalization anyway.
+ *
+ * The numbers here represent offsets into the respective codepages,
+ * with -1 representing null and 0xFF representing zero-width joiner.
+ *
+ * the columns are: ch1, ch2, ch3, res, flags
+ * ch1, ch2, and ch3 are the decomposition
+ * res is the composition, and flags are the scripts to which it applies.
+ */
+ private static final int decompositions[][] = {
+ /* devanagari, gujarati vowel candra O */
+ { 0x05, 0x3E, 0x45, 0x11, flag(DEVANAGARI) | flag(GUJARATI) },
+ /* devanagari short O */
+ { 0x05, 0x3E, 0x46, 0x12, flag(DEVANAGARI) },
+ /* devanagari, gujarati letter O */
+ { 0x05, 0x3E, 0x47, 0x13, flag(DEVANAGARI) | flag(GUJARATI) },
+ /* devanagari letter AI, gujarati letter AU */
+ { 0x05, 0x3E, 0x48, 0x14, flag(DEVANAGARI) | flag(GUJARATI) },
+ /* devanagari, bengali, gurmukhi, gujarati, oriya AA */
+ { 0x05, 0x3E, -1, 0x06, flag(DEVANAGARI) | flag(BENGALI) | flag(GURMUKHI) | flag(GUJARATI) | flag(ORIYA) },
+ /* devanagari letter candra A */
+ { 0x05, 0x45, -1, 0x72, flag(DEVANAGARI) },
+ /* gujarati vowel candra E */
+ { 0x05, 0x45, -1, 0x0D, flag(GUJARATI) },
+ /* devanagari letter short A */
+ { 0x05, 0x46, -1, 0x04, flag(DEVANAGARI) },
+ /* gujarati letter E */
+ { 0x05, 0x47, -1, 0x0F, flag(GUJARATI) },
+ /* gurmukhi, gujarati letter AI */
+ { 0x05, 0x48, -1, 0x10, flag(GURMUKHI) | flag(GUJARATI) },
+ /* devanagari, gujarati vowel candra O */
+ { 0x05, 0x49, -1, 0x11, flag(DEVANAGARI) | flag(GUJARATI) },
+ /* devanagari short O */
+ { 0x05, 0x4A, -1, 0x12, flag(DEVANAGARI) },
+ /* devanagari, gujarati letter O */
+ { 0x05, 0x4B, -1, 0x13, flag(DEVANAGARI) | flag(GUJARATI) },
+ /* devanagari letter AI, gurmukhi letter AU, gujarati letter AU */
+ { 0x05, 0x4C, -1, 0x14, flag(DEVANAGARI) | flag(GURMUKHI) | flag(GUJARATI) },
+ /* devanagari, gujarati vowel candra O */
+ { 0x06, 0x45, -1, 0x11, flag(DEVANAGARI) | flag(GUJARATI) },
+ /* devanagari short O */
+ { 0x06, 0x46, -1, 0x12, flag(DEVANAGARI) },
+ /* devanagari, gujarati letter O */
+ { 0x06, 0x47, -1, 0x13, flag(DEVANAGARI) | flag(GUJARATI) },
+ /* devanagari letter AI, gujarati letter AU */
+ { 0x06, 0x48, -1, 0x14, flag(DEVANAGARI) | flag(GUJARATI) },
+ /* malayalam letter II */
+ { 0x07, 0x57, -1, 0x08, flag(MALAYALAM) },
+ /* devanagari letter UU */
+ { 0x09, 0x41, -1, 0x0A, flag(DEVANAGARI) },
+ /* tamil, malayalam letter UU (some styles) */
+ { 0x09, 0x57, -1, 0x0A, flag(TAMIL) | flag(MALAYALAM) },
+ /* malayalam letter AI */
+ { 0x0E, 0x46, -1, 0x10, flag(MALAYALAM) },
+ /* devanagari candra E */
+ { 0x0F, 0x45, -1, 0x0D, flag(DEVANAGARI) },
+ /* devanagari short E */
+ { 0x0F, 0x46, -1, 0x0E, flag(DEVANAGARI) },
+ /* devanagari AI */
+ { 0x0F, 0x47, -1, 0x10, flag(DEVANAGARI) },
+ /* oriya AI */
+ { 0x0F, 0x57, -1, 0x10, flag(ORIYA) },
+ /* malayalam letter OO */
+ { 0x12, 0x3E, -1, 0x13, flag(MALAYALAM) },
+ /* telugu, kannada letter AU */
+ { 0x12, 0x4C, -1, 0x14, flag(TELUGU) | flag(KANNADA) },
+ /* telugu letter OO */
+ { 0x12, 0x55, -1, 0x13, flag(TELUGU) },
+ /* tamil, malayalam letter AU */
+ { 0x12, 0x57, -1, 0x14, flag(TAMIL) | flag(MALAYALAM) },
+ /* oriya letter AU */
+ { 0x13, 0x57, -1, 0x14, flag(ORIYA) },
+ /* devanagari qa */
+ { 0x15, 0x3C, -1, 0x58, flag(DEVANAGARI) },
+ /* devanagari, gurmukhi khha */
+ { 0x16, 0x3C, -1, 0x59, flag(DEVANAGARI) | flag(GURMUKHI) },
+ /* devanagari, gurmukhi ghha */
+ { 0x17, 0x3C, -1, 0x5A, flag(DEVANAGARI) | flag(GURMUKHI) },
+ /* devanagari, gurmukhi za */
+ { 0x1C, 0x3C, -1, 0x5B, flag(DEVANAGARI) | flag(GURMUKHI) },
+ /* devanagari dddha, bengali, oriya rra */
+ { 0x21, 0x3C, -1, 0x5C, flag(DEVANAGARI) | flag(BENGALI) | flag(ORIYA) },
+ /* devanagari, bengali, oriya rha */
+ { 0x22, 0x3C, -1, 0x5D, flag(DEVANAGARI) | flag(BENGALI) | flag(ORIYA) },
+ /* malayalam chillu nn */
+ { 0x23, 0x4D, 0xFF, 0x7A, flag(MALAYALAM) },
+ /* bengali khanda ta */
+ { 0x24, 0x4D, 0xFF, 0x4E, flag(BENGALI) },
+ /* devanagari nnna */
+ { 0x28, 0x3C, -1, 0x29, flag(DEVANAGARI) },
+ /* malayalam chillu n */
+ { 0x28, 0x4D, 0xFF, 0x7B, flag(MALAYALAM) },
+ /* devanagari, gurmukhi fa */
+ { 0x2B, 0x3C, -1, 0x5E, flag(DEVANAGARI) | flag(GURMUKHI) },
+ /* devanagari, bengali yya */
+ { 0x2F, 0x3C, -1, 0x5F, flag(DEVANAGARI) | flag(BENGALI) },
+ /* telugu letter vocalic R */
+ { 0x2C, 0x41, 0x41, 0x0B, flag(TELUGU) },
+ /* devanagari rra */
+ { 0x30, 0x3C, -1, 0x31, flag(DEVANAGARI) },
+ /* malayalam chillu rr */
+ { 0x30, 0x4D, 0xFF, 0x7C, flag(MALAYALAM) },
+ /* malayalam chillu l */
+ { 0x32, 0x4D, 0xFF, 0x7D, flag(MALAYALAM) },
+ /* devanagari llla */
+ { 0x33, 0x3C, -1, 0x34, flag(DEVANAGARI) },
+ /* malayalam chillu ll */
+ { 0x33, 0x4D, 0xFF, 0x7E, flag(MALAYALAM) },
+ /* telugu letter MA */
+ { 0x35, 0x41, -1, 0x2E, flag(TELUGU) },
+ /* devanagari, gujarati vowel sign candra O */
+ { 0x3E, 0x45, -1, 0x49, flag(DEVANAGARI) | flag(GUJARATI) },
+ /* devanagari vowel sign short O */
+ { 0x3E, 0x46, -1, 0x4A, flag(DEVANAGARI) },
+ /* devanagari, gujarati vowel sign O */
+ { 0x3E, 0x47, -1, 0x4B, flag(DEVANAGARI) | flag(GUJARATI) },
+ /* devanagari, gujarati vowel sign AU */
+ { 0x3E, 0x48, -1, 0x4C, flag(DEVANAGARI) | flag(GUJARATI) },
+ /* kannada vowel sign II */
+ { 0x3F, 0x55, -1, 0x40, flag(KANNADA) },
+ /* gurmukhi vowel sign UU (when stacking) */
+ { 0x41, 0x41, -1, 0x42, flag(GURMUKHI) },
+ /* tamil, malayalam vowel sign O */
+ { 0x46, 0x3E, -1, 0x4A, flag(TAMIL) | flag(MALAYALAM) },
+ /* kannada vowel sign OO */
+ { 0x46, 0x42, 0x55, 0x4B, flag(KANNADA) },
+ /* kannada vowel sign O */
+ { 0x46, 0x42, -1, 0x4A, flag(KANNADA) },
+ /* malayalam vowel sign AI (if reordered twice) */
+ { 0x46, 0x46, -1, 0x48, flag(MALAYALAM) },
+ /* telugu, kannada vowel sign EE */
+ { 0x46, 0x55, -1, 0x47, flag(TELUGU) | flag(KANNADA) },
+ /* telugu, kannada vowel sign AI */
+ { 0x46, 0x56, -1, 0x48, flag(TELUGU) | flag(KANNADA) },
+ /* tamil, malayalam vowel sign AU */
+ { 0x46, 0x57, -1, 0x4C, flag(TAMIL) | flag(MALAYALAM) },
+ /* bengali, oriya vowel sign O, tamil, malayalam vowel sign OO */
+ { 0x47, 0x3E, -1, 0x4B, flag(BENGALI) | flag(ORIYA) | flag(TAMIL) | flag(MALAYALAM) },
+ /* bengali, oriya vowel sign AU */
+ { 0x47, 0x57, -1, 0x4C, flag(BENGALI) | flag(ORIYA) },
+ /* kannada vowel sign OO */
+ { 0x4A, 0x55, -1, 0x4B, flag(KANNADA) },
+ /* gurmukhi letter I */
+ { 0x72, 0x3F, -1, 0x07, flag(GURMUKHI) },
+ /* gurmukhi letter II */
+ { 0x72, 0x40, -1, 0x08, flag(GURMUKHI) },
+ /* gurmukhi letter EE */
+ { 0x72, 0x47, -1, 0x0F, flag(GURMUKHI) },
+ /* gurmukhi letter U */
+ { 0x73, 0x41, -1, 0x09, flag(GURMUKHI) },
+ /* gurmukhi letter UU */
+ { 0x73, 0x42, -1, 0x0A, flag(GURMUKHI) },
+ /* gurmukhi letter OO */
+ { 0x73, 0x4B, -1, 0x13, flag(GURMUKHI) },
+ };
+
+ static {
+ for (ScriptData sd : scripts.values()) {
+ sd.decompMask = new BitSet(0x7F);
+ for (int i = 0; i < decompositions.length; i++) {
+ final int ch = decompositions[i][0];
+ final int flags = decompositions[i][4];
+ if ((flags & sd.flag) != 0)
+ sd.decompMask.set(ch);
+ }
+ }
+ }
+
+ /**
+ * Normalizes input text, and returns the new length.
+ * The length will always be less than or equal to the existing length.
+ *
+ * @param text input text
+ * @param len valid length
+ * @return normalized length
+ */
+ public int normalize(char text[], int len) {
+ for (int i = 0; i < len; i++) {
+ final Character.UnicodeBlock block = Character.UnicodeBlock.of(text[i]);
+ final ScriptData sd = scripts.get(block);
+ if (sd != null) {
+ final int ch = text[i] - sd.base;
+ if (sd.decompMask.get(ch))
+ len = compose(ch, block, sd, text, i, len);
+ }
+ }
+ return len;
+ }
+
+ /**
+ * Compose into standard form any compositions in the decompositions table.
+ */
+ private int compose(int ch0, Character.UnicodeBlock block0, ScriptData sd,
+ char text[], int pos, int len) {
+ if (pos + 1 >= len) /* need at least 2 chars! */
+ return len;
+
+ final int ch1 = text[pos + 1] - sd.base;
+ final Character.UnicodeBlock block1 = Character.UnicodeBlock.of(text[pos + 1]);
+ if (block1 != block0) /* needs to be the same writing system */
+ return len;
+
+ int ch2 = -1;
+
+ if (pos + 2 < len) {
+ ch2 = text[pos + 2] - sd.base;
+ Character.UnicodeBlock block2 = Character.UnicodeBlock.of(text[pos + 2]);
+ if (text[pos + 2] == '\u200D') // ZWJ
+ ch2 = 0xFF;
+ else if (block2 != block1) // still allow a 2-char match
+ ch2 = -1;
+ }
+
+ for (int i = 0; i < decompositions.length; i++)
+ if (decompositions[i][0] == ch0 && (decompositions[i][4] & sd.flag) != 0) {
+ if (decompositions[i][1] == ch1 && (decompositions[i][2] < 0 || decompositions[i][2] == ch2)) {
+ text[pos] = (char) (sd.base + decompositions[i][3]);
+ len = delete(text, pos + 1, len);
+ if (decompositions[i][2] >= 0)
+ len = delete(text, pos + 1, len);
+ return len;
+ }
+ }
+
+ return len;
+ }
+}
fnp / pylucene.git / blobdiff