1 package org.apache.lucene.queryParser.standard;
4 * Licensed to the Apache Software Foundation (ASF) under one or more
5 * contributor license agreements. See the NOTICE file distributed with
6 * this work for additional information regarding copyright ownership.
7 * The ASF licenses this file to You under the Apache License, Version 2.0
8 * (the "License"); you may not use this file except in compliance with
9 * the License. You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
20 import java.io.IOException;
21 import java.io.Reader;
23 import org.apache.lucene.analysis.Analyzer;
24 import org.apache.lucene.analysis.LowerCaseFilter;
25 import org.apache.lucene.analysis.TokenFilter;
26 import org.apache.lucene.analysis.TokenStream;
27 import org.apache.lucene.analysis.standard.StandardTokenizer;
28 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
29 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
30 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
31 import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
32 import org.apache.lucene.queryParser.core.QueryNodeException;
33 import org.apache.lucene.queryParser.standard.config.StandardQueryConfigHandler;
34 import org.apache.lucene.util.LuceneTestCase;
37 * This test case is a copy of the core Lucene query parser test, it was adapted
38 * to use new QueryParserHelper instead of the old query parser.
40 * Test QueryParser's ability to deal with Analyzers that return more than one
41 * token per position or that return tokens with a position increment > 1.
43 public class TestMultiAnalyzerQPHelper extends LuceneTestCase {
45 private static int multiToken = 0;
47 @SuppressWarnings("deprecation")
48 public void testMultiAnalyzer() throws QueryNodeException {
50 StandardQueryParser qp = new StandardQueryParser();
51 qp.setAnalyzer(new MultiAnalyzer());
53 // trivial, no multiple tokens:
54 assertEquals("foo", qp.parse("foo", "").toString());
55 assertEquals("foo", qp.parse("\"foo\"", "").toString());
56 assertEquals("foo foobar", qp.parse("foo foobar", "").toString());
57 assertEquals("\"foo foobar\"", qp.parse("\"foo foobar\"", "").toString());
58 assertEquals("\"foo foobar blah\"", qp.parse("\"foo foobar blah\"", "")
61 // two tokens at the same position:
62 assertEquals("(multi multi2) foo", qp.parse("multi foo", "").toString());
63 assertEquals("foo (multi multi2)", qp.parse("foo multi", "").toString());
64 assertEquals("(multi multi2) (multi multi2)", qp.parse("multi multi", "")
66 assertEquals("+(foo (multi multi2)) +(bar (multi multi2))", qp.parse(
67 "+(foo multi) +(bar multi)", "").toString());
68 assertEquals("+(foo (multi multi2)) field:\"bar (multi multi2)\"", qp
69 .parse("+(foo multi) field:\"bar multi\"", "").toString());
72 assertEquals("\"(multi multi2) foo\"", qp.parse("\"multi foo\"", "")
74 assertEquals("\"foo (multi multi2)\"", qp.parse("\"foo multi\"", "")
76 assertEquals("\"foo (multi multi2) foobar (multi multi2)\"", qp.parse(
77 "\"foo multi foobar multi\"", "").toString());
80 assertEquals("(field:multi field:multi2) field:foo", qp.parse(
81 "field:multi field:foo", "").toString());
82 assertEquals("field:\"(multi multi2) foo\"", qp.parse(
83 "field:\"multi foo\"", "").toString());
85 // three tokens at one position:
86 assertEquals("triplemulti multi3 multi2", qp.parse("triplemulti", "")
88 assertEquals("foo (triplemulti multi3 multi2) foobar", qp.parse(
89 "foo triplemulti foobar", "").toString());
91 // phrase with non-default slop:
92 assertEquals("\"(multi multi2) foo\"~10", qp.parse("\"multi foo\"~10", "")
95 // phrase with non-default boost:
96 assertEquals("\"(multi multi2) foo\"^2.0", qp.parse("\"multi foo\"^2", "")
99 // phrase after changing default slop
100 qp.setDefaultPhraseSlop(99);
101 assertEquals("\"(multi multi2) foo\"~99 bar", qp.parse("\"multi foo\" bar",
103 assertEquals("\"(multi multi2) foo\"~99 \"foo bar\"~2", qp.parse(
104 "\"multi foo\" \"foo bar\"~2", "").toString());
105 qp.setDefaultPhraseSlop(0);
107 // non-default operator:
108 qp.setDefaultOperator(StandardQueryConfigHandler.Operator.AND);
109 assertEquals("+(multi multi2) +foo", qp.parse("multi foo", "").toString());
113 // public void testMultiAnalyzerWithSubclassOfQueryParser() throws
115 // this test doesn't make sense when using the new QueryParser API
116 // DumbQueryParser qp = new DumbQueryParser("", new MultiAnalyzer());
117 // qp.setPhraseSlop(99); // modified default slop
119 // // direct call to (super's) getFieldQuery to demonstrate differnce
120 // // between phrase and multiphrase with modified default slop
121 // assertEquals("\"foo bar\"~99",
122 // qp.getSuperFieldQuery("","foo bar").toString());
123 // assertEquals("\"(multi multi2) bar\"~99",
124 // qp.getSuperFieldQuery("","multi bar").toString());
127 // // ask sublcass to parse phrase with modified default slop
128 // assertEquals("\"(multi multi2) foo\"~99 bar",
129 // qp.parse("\"multi foo\" bar").toString());
133 public void testPosIncrementAnalyzer() throws QueryNodeException {
134 StandardQueryParser qp = new StandardQueryParser();
135 qp.setAnalyzer(new PosIncrementAnalyzer());
137 assertEquals("quick brown", qp.parse("the quick brown", "").toString());
138 assertEquals("\"quick brown\"", qp.parse("\"the quick brown\"", "")
140 assertEquals("quick brown fox", qp.parse("the quick brown fox", "")
142 assertEquals("\"quick brown fox\"", qp.parse("\"the quick brown fox\"", "")
147 * Expands "multi" to "multi" and "multi2", both at the same position, and
148 * expands "triplemulti" to "triplemulti", "multi3", and "multi2".
150 private class MultiAnalyzer extends Analyzer {
152 public MultiAnalyzer() {
156 public TokenStream tokenStream(String fieldName, Reader reader) {
157 TokenStream result = new StandardTokenizer(TEST_VERSION_CURRENT, reader);
158 result = new TestFilter(result);
159 result = new LowerCaseFilter(TEST_VERSION_CURRENT, result);
164 private final class TestFilter extends TokenFilter {
166 private String prevType;
167 private int prevStartOffset;
168 private int prevEndOffset;
170 private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
171 private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
172 private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
173 private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
175 public TestFilter(TokenStream in) {
180 public final boolean incrementToken() throws java.io.IOException {
181 if (multiToken > 0) {
182 termAtt.setEmpty().append("multi" + (multiToken + 1));
183 offsetAtt.setOffset(prevStartOffset, prevEndOffset);
184 typeAtt.setType(prevType);
185 posIncrAtt.setPositionIncrement(0);
189 boolean next = input.incrementToken();
193 prevType = typeAtt.type();
194 prevStartOffset = offsetAtt.startOffset();
195 prevEndOffset = offsetAtt.endOffset();
196 String text = termAtt.toString();
197 if (text.equals("triplemulti")) {
200 } else if (text.equals("multi")) {
210 public void reset() throws IOException {
212 this.prevType = null;
213 this.prevStartOffset = 0;
214 this.prevEndOffset = 0;
219 * Analyzes "the quick brown" as: quick(incr=2) brown(incr=1). Does not work
220 * correctly for input other than "the quick brown ...".
222 private class PosIncrementAnalyzer extends Analyzer {
224 public PosIncrementAnalyzer() {
228 public TokenStream tokenStream(String fieldName, Reader reader) {
229 TokenStream result = new StandardTokenizer(TEST_VERSION_CURRENT, reader);
230 result = new TestPosIncrementFilter(result);
231 result = new LowerCaseFilter(TEST_VERSION_CURRENT, result);
236 private class TestPosIncrementFilter extends TokenFilter {
238 private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
239 private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
241 public TestPosIncrementFilter(TokenStream in) {
246 public final boolean incrementToken() throws java.io.IOException {
247 while (input.incrementToken()) {
248 if (termAtt.toString().equals("the")) {
249 // stopword, do nothing
250 } else if (termAtt.toString().equals("quick")) {
251 posIncrAtt.setPositionIncrement(2);
254 posIncrAtt.setPositionIncrement(1);