1 package org.apache.lucene.analysis.miscellaneous;
4 * Licensed to the Apache Software Foundation (ASF) under one or more
5 * contributor license agreements. See the NOTICE file distributed with
6 * this work for additional information regarding copyright ownership.
7 * The ASF licenses this file to You under the Apache License, Version 2.0
8 * (the "License"); you may not use this file except in compliance with
9 * the License. You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
20 import java.io.IOException;
21 import java.util.Arrays;
22 import java.util.regex.Pattern;
24 import org.apache.lucene.analysis.BaseTokenStreamTestCase;
25 import org.apache.lucene.analysis.StopAnalyzer;
26 import org.apache.lucene.analysis.TokenStream;
29 * Verifies the behavior of PatternAnalyzer.
31 public class PatternAnalyzerTest extends BaseTokenStreamTestCase {
34 * Test PatternAnalyzer when it is configured with a non-word pattern.
35 * Behavior can be similar to SimpleAnalyzer (depending upon options)
37 public void testNonWordPattern() throws IOException {
38 // Split on non-letter pattern, do not lowercase, no stopwords
39 PatternAnalyzer a = new PatternAnalyzer(TEST_VERSION_CURRENT, PatternAnalyzer.NON_WORD_PATTERN,
41 check(a, "The quick brown Fox,the abcd1234 (56.78) dc.", new String[] {
42 "The", "quick", "brown", "Fox", "the", "abcd", "dc" });
44 // split on non-letter pattern, lowercase, english stopwords
45 PatternAnalyzer b = new PatternAnalyzer(TEST_VERSION_CURRENT, PatternAnalyzer.NON_WORD_PATTERN,
46 true, StopAnalyzer.ENGLISH_STOP_WORDS_SET);
47 check(b, "The quick brown Fox,the abcd1234 (56.78) dc.", new String[] {
48 "quick", "brown", "fox", "abcd", "dc" });
52 * Test PatternAnalyzer when it is configured with a whitespace pattern.
53 * Behavior can be similar to WhitespaceAnalyzer (depending upon options)
55 public void testWhitespacePattern() throws IOException {
56 // Split on whitespace patterns, do not lowercase, no stopwords
57 PatternAnalyzer a = new PatternAnalyzer(TEST_VERSION_CURRENT, PatternAnalyzer.WHITESPACE_PATTERN,
59 check(a, "The quick brown Fox,the abcd1234 (56.78) dc.", new String[] {
60 "The", "quick", "brown", "Fox,the", "abcd1234", "(56.78)", "dc." });
62 // Split on whitespace patterns, lowercase, english stopwords
63 PatternAnalyzer b = new PatternAnalyzer(TEST_VERSION_CURRENT, PatternAnalyzer.WHITESPACE_PATTERN,
64 true, StopAnalyzer.ENGLISH_STOP_WORDS_SET);
65 check(b, "The quick brown Fox,the abcd1234 (56.78) dc.", new String[] {
66 "quick", "brown", "fox,the", "abcd1234", "(56.78)", "dc." });
70 * Test PatternAnalyzer when it is configured with a custom pattern. In this
71 * case, text is tokenized on the comma ","
73 public void testCustomPattern() throws IOException {
74 // Split on comma, do not lowercase, no stopwords
75 PatternAnalyzer a = new PatternAnalyzer(TEST_VERSION_CURRENT, Pattern.compile(","), false, null);
76 check(a, "Here,Are,some,Comma,separated,words,", new String[] { "Here",
77 "Are", "some", "Comma", "separated", "words" });
79 // split on comma, lowercase, english stopwords
80 PatternAnalyzer b = new PatternAnalyzer(TEST_VERSION_CURRENT, Pattern.compile(","), true,
81 StopAnalyzer.ENGLISH_STOP_WORDS_SET);
82 check(b, "Here,Are,some,Comma,separated,words,", new String[] { "here",
83 "some", "comma", "separated", "words" });
87 * Test PatternAnalyzer against a large document.
89 public void testHugeDocument() throws IOException {
90 StringBuilder document = new StringBuilder();
92 char largeWord[] = new char[5000];
93 Arrays.fill(largeWord, 'a');
94 document.append(largeWord);
100 char largeWord2[] = new char[2000];
101 Arrays.fill(largeWord2, 'b');
102 document.append(largeWord2);
104 // Split on whitespace patterns, do not lowercase, no stopwords
105 PatternAnalyzer a = new PatternAnalyzer(TEST_VERSION_CURRENT, PatternAnalyzer.WHITESPACE_PATTERN,
107 check(a, document.toString(), new String[] { new String(largeWord),
108 new String(largeWord2) });
112 * Verify the analyzer analyzes to the expected contents. For PatternAnalyzer,
113 * several methods are verified:
115 * <li>Analysis with a normal Reader
116 * <li>Analysis with a FastStringReader
117 * <li>Analysis with a String
120 private void check(PatternAnalyzer analyzer, String document,
121 String expected[]) throws IOException {
122 // ordinary analysis of a Reader
123 assertAnalyzesTo(analyzer, document, expected);
125 // analysis with a "FastStringReader"
126 TokenStream ts = analyzer.tokenStream("dummy",
127 new PatternAnalyzer.FastStringReader(document));
128 assertTokenStreamContents(ts, expected);
130 // analysis of a String, uses PatternAnalyzer.tokenStream(String, String)
131 TokenStream ts2 = analyzer.tokenStream("dummy", document);
132 assertTokenStreamContents(ts2, expected);