pylucene 3.5.0-3

[pylucene.git] / lucene-java-3.5.0 / lucene / contrib / analyzers / common / src / java / org / apache / lucene / analysis / cn / ChineseFilter.java

diff --git a/lucene-java-3.5.0/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cn/ChineseFilter.java b/lucene-java-3.5.0/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cn/ChineseFilter.java
new file mode 100644 (file)
index 0000000..14b951d
--- /dev/null
+++ b/lucene-java-3.5.0/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cn/ChineseFilter.java
@@ -0,0 +1,105 @@
+package org.apache.lucene.analysis.cn;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.apache.lucene.analysis.CharArraySet;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.StopFilter;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.util.Version;
+
+/**
+ * A {@link TokenFilter} with a stop word table.  
+ * <ul>
+ * <li>Numeric tokens are removed.
+ * <li>English tokens must be larger than 1 character.
+ * <li>One Chinese character as one Chinese word.
+ * </ul>
+ * TO DO:
+ * <ol>
+ * <li>Add Chinese stop words, such as \ue400
+ * <li>Dictionary based Chinese word extraction
+ * <li>Intelligent Chinese word extraction
+ * </ol>
+ * 
+ * @version 1.0
+ * @deprecated Use {@link StopFilter} instead, which has the same functionality.
+ * This filter will be removed in Lucene 5.0
+ */
+@Deprecated
+public final class ChineseFilter extends TokenFilter {
+
+
+    // Only English now, Chinese to be added later.
+    public static final String[] STOP_WORDS = {
+    "and", "are", "as", "at", "be", "but", "by",
+    "for", "if", "in", "into", "is", "it",
+    "no", "not", "of", "on", "or", "such",
+    "that", "the", "their", "then", "there", "these",
+    "they", "this", "to", "was", "will", "with"
+    };
+
+
+    private CharArraySet stopTable;
+
+    private CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+    
+    public ChineseFilter(TokenStream in) {
+        super(in);
+
+        stopTable = new CharArraySet(Version.LUCENE_CURRENT, Arrays.asList(STOP_WORDS), false);
+    }
+
+    @Override
+    public boolean incrementToken() throws IOException {
+
+        while (input.incrementToken()) {
+            char text[] = termAtt.buffer();
+            int termLength = termAtt.length();
+
+          // why not key off token type here assuming ChineseTokenizer comes first?
+            if (!stopTable.contains(text, 0, termLength)) {
+                switch (Character.getType(text[0])) {
+
+                case Character.LOWERCASE_LETTER:
+                case Character.UPPERCASE_LETTER:
+
+                    // English word/token should larger than 1 character.
+                    if (termLength>1) {
+                        return true;
+                    }
+                    break;
+                case Character.OTHER_LETTER:
+
+                    // One Chinese character as one Chinese word.
+                    // Chinese word extraction to be added later here.
+
+                    return true;
+                }
+
+            }
+
+        }
+        return false;
+    }
+
+}
\ No newline at end of file