X-Git-Url: https://git.mdrn.pl/pylucene.git/blobdiff_plain/a2e61f0c04805cfcb8706176758d1283c7e3a55c..aaeed5504b982cf3545252ab528713250aa33eed:/lucene-java-3.4.0/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/PatternAnalyzerTest.java?ds=sidebyside diff --git a/lucene-java-3.4.0/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/PatternAnalyzerTest.java b/lucene-java-3.4.0/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/PatternAnalyzerTest.java deleted file mode 100644 index a346470..0000000 --- a/lucene-java-3.4.0/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/PatternAnalyzerTest.java +++ /dev/null @@ -1,134 +0,0 @@ -package org.apache.lucene.analysis.miscellaneous; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.Arrays; -import java.util.regex.Pattern; - -import org.apache.lucene.analysis.BaseTokenStreamTestCase; -import org.apache.lucene.analysis.StopAnalyzer; -import org.apache.lucene.analysis.TokenStream; - -/** - * Verifies the behavior of PatternAnalyzer. - */ -public class PatternAnalyzerTest extends BaseTokenStreamTestCase { - - /** - * Test PatternAnalyzer when it is configured with a non-word pattern. - * Behavior can be similar to SimpleAnalyzer (depending upon options) - */ - public void testNonWordPattern() throws IOException { - // Split on non-letter pattern, do not lowercase, no stopwords - PatternAnalyzer a = new PatternAnalyzer(TEST_VERSION_CURRENT, PatternAnalyzer.NON_WORD_PATTERN, - false, null); - check(a, "The quick brown Fox,the abcd1234 (56.78) dc.", new String[] { - "The", "quick", "brown", "Fox", "the", "abcd", "dc" }); - - // split on non-letter pattern, lowercase, english stopwords - PatternAnalyzer b = new PatternAnalyzer(TEST_VERSION_CURRENT, PatternAnalyzer.NON_WORD_PATTERN, - true, StopAnalyzer.ENGLISH_STOP_WORDS_SET); - check(b, "The quick brown Fox,the abcd1234 (56.78) dc.", new String[] { - "quick", "brown", "fox", "abcd", "dc" }); - } - - /** - * Test PatternAnalyzer when it is configured with a whitespace pattern. - * Behavior can be similar to WhitespaceAnalyzer (depending upon options) - */ - public void testWhitespacePattern() throws IOException { - // Split on whitespace patterns, do not lowercase, no stopwords - PatternAnalyzer a = new PatternAnalyzer(TEST_VERSION_CURRENT, PatternAnalyzer.WHITESPACE_PATTERN, - false, null); - check(a, "The quick brown Fox,the abcd1234 (56.78) dc.", new String[] { - "The", "quick", "brown", "Fox,the", "abcd1234", "(56.78)", "dc." }); - - // Split on whitespace patterns, lowercase, english stopwords - PatternAnalyzer b = new PatternAnalyzer(TEST_VERSION_CURRENT, PatternAnalyzer.WHITESPACE_PATTERN, - true, StopAnalyzer.ENGLISH_STOP_WORDS_SET); - check(b, "The quick brown Fox,the abcd1234 (56.78) dc.", new String[] { - "quick", "brown", "fox,the", "abcd1234", "(56.78)", "dc." }); - } - - /** - * Test PatternAnalyzer when it is configured with a custom pattern. In this - * case, text is tokenized on the comma "," - */ - public void testCustomPattern() throws IOException { - // Split on comma, do not lowercase, no stopwords - PatternAnalyzer a = new PatternAnalyzer(TEST_VERSION_CURRENT, Pattern.compile(","), false, null); - check(a, "Here,Are,some,Comma,separated,words,", new String[] { "Here", - "Are", "some", "Comma", "separated", "words" }); - - // split on comma, lowercase, english stopwords - PatternAnalyzer b = new PatternAnalyzer(TEST_VERSION_CURRENT, Pattern.compile(","), true, - StopAnalyzer.ENGLISH_STOP_WORDS_SET); - check(b, "Here,Are,some,Comma,separated,words,", new String[] { "here", - "some", "comma", "separated", "words" }); - } - - /** - * Test PatternAnalyzer against a large document. - */ - public void testHugeDocument() throws IOException { - StringBuilder document = new StringBuilder(); - // 5000 a's - char largeWord[] = new char[5000]; - Arrays.fill(largeWord, 'a'); - document.append(largeWord); - - // a space - document.append(' '); - - // 2000 b's - char largeWord2[] = new char[2000]; - Arrays.fill(largeWord2, 'b'); - document.append(largeWord2); - - // Split on whitespace patterns, do not lowercase, no stopwords - PatternAnalyzer a = new PatternAnalyzer(TEST_VERSION_CURRENT, PatternAnalyzer.WHITESPACE_PATTERN, - false, null); - check(a, document.toString(), new String[] { new String(largeWord), - new String(largeWord2) }); - } - - /** - * Verify the analyzer analyzes to the expected contents. For PatternAnalyzer, - * several methods are verified: - * - */ - private void check(PatternAnalyzer analyzer, String document, - String expected[]) throws IOException { - // ordinary analysis of a Reader - assertAnalyzesTo(analyzer, document, expected); - - // analysis with a "FastStringReader" - TokenStream ts = analyzer.tokenStream("dummy", - new PatternAnalyzer.FastStringReader(document)); - assertTokenStreamContents(ts, expected); - - // analysis of a String, uses PatternAnalyzer.tokenStream(String, String) - TokenStream ts2 = analyzer.tokenStream("dummy", document); - assertTokenStreamContents(ts2, expected); - } -}