X-Git-Url: https://git.mdrn.pl/pylucene.git/blobdiff_plain/a2e61f0c04805cfcb8706176758d1283c7e3a55c..aaeed5504b982cf3545252ab528713250aa33eed:/lucene-java-3.5.0/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/PatternAnalyzerTest.java

diff --git a/lucene-java-3.5.0/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/PatternAnalyzerTest.java b/lucene-java-3.5.0/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/PatternAnalyzerTest.java
new file mode 100644
index 0000000..a346470
--- /dev/null
+++ b/lucene-java-3.5.0/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/PatternAnalyzerTest.java
@@ -0,0 +1,134 @@
+package org.apache.lucene.analysis.miscellaneous;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.regex.Pattern;
+
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.StopAnalyzer;
+import org.apache.lucene.analysis.TokenStream;
+
+/**
+ * Verifies the behavior of PatternAnalyzer.
+ */
+public class PatternAnalyzerTest extends BaseTokenStreamTestCase {
+
+  /**
+   * Test PatternAnalyzer when it is configured with a non-word pattern.
+   * Behavior can be similar to SimpleAnalyzer (depending upon options)
+   */
+  public void testNonWordPattern() throws IOException {
+    // Split on non-letter pattern, do not lowercase, no stopwords
+    PatternAnalyzer a = new PatternAnalyzer(TEST_VERSION_CURRENT, PatternAnalyzer.NON_WORD_PATTERN,
+        false, null);
+    check(a, "The quick brown Fox,the abcd1234 (56.78) dc.", new String[] {
+        "The", "quick", "brown", "Fox", "the", "abcd", "dc" });
+
+    // split on non-letter pattern, lowercase, english stopwords
+    PatternAnalyzer b = new PatternAnalyzer(TEST_VERSION_CURRENT, PatternAnalyzer.NON_WORD_PATTERN,
+        true, StopAnalyzer.ENGLISH_STOP_WORDS_SET);
+    check(b, "The quick brown Fox,the abcd1234 (56.78) dc.", new String[] {
+        "quick", "brown", "fox", "abcd", "dc" });
+  }
+
+  /**
+   * Test PatternAnalyzer when it is configured with a whitespace pattern.
+   * Behavior can be similar to WhitespaceAnalyzer (depending upon options)
+   */
+  public void testWhitespacePattern() throws IOException {
+    // Split on whitespace patterns, do not lowercase, no stopwords
+    PatternAnalyzer a = new PatternAnalyzer(TEST_VERSION_CURRENT, PatternAnalyzer.WHITESPACE_PATTERN,
+        false, null);
+    check(a, "The quick brown Fox,the abcd1234 (56.78) dc.", new String[] {
+        "The", "quick", "brown", "Fox,the", "abcd1234", "(56.78)", "dc." });
+
+    // Split on whitespace patterns, lowercase, english stopwords
+    PatternAnalyzer b = new PatternAnalyzer(TEST_VERSION_CURRENT, PatternAnalyzer.WHITESPACE_PATTERN,
+        true, StopAnalyzer.ENGLISH_STOP_WORDS_SET);
+    check(b, "The quick brown Fox,the abcd1234 (56.78) dc.", new String[] {
+        "quick", "brown", "fox,the", "abcd1234", "(56.78)", "dc." });
+  }
+
+  /**
+   * Test PatternAnalyzer when it is configured with a custom pattern. In this
+   * case, text is tokenized on the comma ","
+   */
+  public void testCustomPattern() throws IOException {
+    // Split on comma, do not lowercase, no stopwords
+    PatternAnalyzer a = new PatternAnalyzer(TEST_VERSION_CURRENT, Pattern.compile(","), false, null);
+    check(a, "Here,Are,some,Comma,separated,words,", new String[] { "Here",
+        "Are", "some", "Comma", "separated", "words" });
+
+    // split on comma, lowercase, english stopwords
+    PatternAnalyzer b = new PatternAnalyzer(TEST_VERSION_CURRENT, Pattern.compile(","), true,
+        StopAnalyzer.ENGLISH_STOP_WORDS_SET);
+    check(b, "Here,Are,some,Comma,separated,words,", new String[] { "here",
+        "some", "comma", "separated", "words" });
+  }
+
+  /**
+   * Test PatternAnalyzer against a large document.
+   */
+  public void testHugeDocument() throws IOException {
+    StringBuilder document = new StringBuilder();
+    // 5000 a's
+    char largeWord[] = new char[5000];
+    Arrays.fill(largeWord, 'a');
+    document.append(largeWord);
+
+    // a space
+    document.append(' ');
+
+    // 2000 b's
+    char largeWord2[] = new char[2000];
+    Arrays.fill(largeWord2, 'b');
+    document.append(largeWord2);
+
+    // Split on whitespace patterns, do not lowercase, no stopwords
+    PatternAnalyzer a = new PatternAnalyzer(TEST_VERSION_CURRENT, PatternAnalyzer.WHITESPACE_PATTERN,
+        false, null);
+    check(a, document.toString(), new String[] { new String(largeWord),
+        new String(largeWord2) });
+  }
+
+  /**
+   * Verify the analyzer analyzes to the expected contents. For PatternAnalyzer,
+   * several methods are verified:
+   * <ul>
+   * <li>Analysis with a normal Reader
+   * <li>Analysis with a FastStringReader
+   * <li>Analysis with a String
+   * </ul>
+   */
+  private void check(PatternAnalyzer analyzer, String document,
+      String expected[]) throws IOException {
+    // ordinary analysis of a Reader
+    assertAnalyzesTo(analyzer, document, expected);
+
+    // analysis with a "FastStringReader"
+    TokenStream ts = analyzer.tokenStream("dummy",
+        new PatternAnalyzer.FastStringReader(document));
+    assertTokenStreamContents(ts, expected);
+
+    // analysis of a String, uses PatternAnalyzer.tokenStream(String, String)
+    TokenStream ts2 = analyzer.tokenStream("dummy", document);
+    assertTokenStreamContents(ts2, expected);
+  }
+}