1 package org.apache.lucene.queryParser.standard;
4 * Licensed to the Apache Software Foundation (ASF) under one or more
5 * contributor license agreements. See the NOTICE file distributed with
6 * this work for additional information regarding copyright ownership.
7 * The ASF licenses this file to You under the Apache License, Version 2.0
8 * (the "License"); you may not use this file except in compliance with
9 * the License. You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
20 import java.io.Reader;
22 import org.apache.lucene.analysis.Analyzer;
23 import org.apache.lucene.analysis.LowerCaseFilter;
24 import org.apache.lucene.analysis.TokenFilter;
25 import org.apache.lucene.analysis.TokenStream;
26 import org.apache.lucene.analysis.standard.StandardTokenizer;
27 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
28 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
29 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
30 import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
31 import org.apache.lucene.queryParser.ParseException;
32 import org.apache.lucene.util.LuceneTestCase;
35 * This test case is a copy of the core Lucene query parser test, it was adapted
36 * to use new QueryParserWrapper instead of the old query parser.
38 * Test QueryParser's ability to deal with Analyzers that return more than one
39 * token per position or that return tokens with a position increment > 1.
42 public class TestMultiAnalyzerWrapper extends LuceneTestCase {
44 private static int multiToken = 0;
46 @SuppressWarnings("deprecation")
47 public void testMultiAnalyzer() throws ParseException {
49 QueryParserWrapper qp = new QueryParserWrapper("", new MultiAnalyzer());
51 // trivial, no multiple tokens:
52 assertEquals("foo", qp.parse("foo").toString());
53 assertEquals("foo", qp.parse("\"foo\"").toString());
54 assertEquals("foo foobar", qp.parse("foo foobar").toString());
55 assertEquals("\"foo foobar\"", qp.parse("\"foo foobar\"").toString());
56 assertEquals("\"foo foobar blah\"", qp.parse("\"foo foobar blah\"")
59 // two tokens at the same position:
60 assertEquals("(multi multi2) foo", qp.parse("multi foo").toString());
61 assertEquals("foo (multi multi2)", qp.parse("foo multi").toString());
62 assertEquals("(multi multi2) (multi multi2)", qp.parse("multi multi")
64 assertEquals("+(foo (multi multi2)) +(bar (multi multi2))", qp.parse(
65 "+(foo multi) +(bar multi)").toString());
66 assertEquals("+(foo (multi multi2)) field:\"bar (multi multi2)\"", qp
67 .parse("+(foo multi) field:\"bar multi\"").toString());
70 assertEquals("\"(multi multi2) foo\"", qp.parse("\"multi foo\"").toString());
71 assertEquals("\"foo (multi multi2)\"", qp.parse("\"foo multi\"").toString());
72 assertEquals("\"foo (multi multi2) foobar (multi multi2)\"", qp.parse(
73 "\"foo multi foobar multi\"").toString());
76 assertEquals("(field:multi field:multi2) field:foo", qp.parse(
77 "field:multi field:foo").toString());
78 assertEquals("field:\"(multi multi2) foo\"", qp
79 .parse("field:\"multi foo\"").toString());
81 // three tokens at one position:
82 assertEquals("triplemulti multi3 multi2", qp.parse("triplemulti")
84 assertEquals("foo (triplemulti multi3 multi2) foobar", qp.parse(
85 "foo triplemulti foobar").toString());
87 // phrase with non-default slop:
88 assertEquals("\"(multi multi2) foo\"~10", qp.parse("\"multi foo\"~10")
91 // phrase with non-default boost:
92 assertEquals("\"(multi multi2) foo\"^2.0", qp.parse("\"multi foo\"^2")
95 // phrase after changing default slop
97 assertEquals("\"(multi multi2) foo\"~99 bar", qp.parse("\"multi foo\" bar")
99 assertEquals("\"(multi multi2) foo\"~99 \"foo bar\"~2", qp.parse(
100 "\"multi foo\" \"foo bar\"~2").toString());
103 // non-default operator:
104 qp.setDefaultOperator(QueryParserWrapper.AND_OPERATOR);
105 assertEquals("+(multi multi2) +foo", qp.parse("multi foo").toString());
109 // public void testMultiAnalyzerWithSubclassOfQueryParser() throws
111 // this test doesn't make sense when using the new QueryParser API
112 // DumbQueryParser qp = new DumbQueryParser("", new MultiAnalyzer());
113 // qp.setPhraseSlop(99); // modified default slop
115 // // direct call to (super's) getFieldQuery to demonstrate differnce
116 // // between phrase and multiphrase with modified default slop
117 // assertEquals("\"foo bar\"~99",
118 // qp.getSuperFieldQuery("","foo bar").toString());
119 // assertEquals("\"(multi multi2) bar\"~99",
120 // qp.getSuperFieldQuery("","multi bar").toString());
123 // // ask sublcass to parse phrase with modified default slop
124 // assertEquals("\"(multi multi2) foo\"~99 bar",
125 // qp.parse("\"multi foo\" bar").toString());
129 @SuppressWarnings("deprecation")
130 public void testPosIncrementAnalyzer() throws ParseException {
131 QueryParserWrapper qp = new QueryParserWrapper("",
132 new PosIncrementAnalyzer());
133 assertEquals("quick brown", qp.parse("the quick brown").toString());
134 assertEquals("\"quick brown\"", qp.parse("\"the quick brown\"").toString());
135 assertEquals("quick brown fox", qp.parse("the quick brown fox").toString());
136 assertEquals("\"quick brown fox\"", qp.parse("\"the quick brown fox\"")
141 * Expands "multi" to "multi" and "multi2", both at the same position, and
142 * expands "triplemulti" to "triplemulti", "multi3", and "multi2".
144 private class MultiAnalyzer extends Analyzer {
146 public MultiAnalyzer() {
150 public TokenStream tokenStream(String fieldName, Reader reader) {
151 TokenStream result = new StandardTokenizer(TEST_VERSION_CURRENT, reader);
152 result = new TestFilter(result);
153 result = new LowerCaseFilter(TEST_VERSION_CURRENT, result);
158 private final class TestFilter extends TokenFilter {
160 private String prevType;
161 private int prevStartOffset;
162 private int prevEndOffset;
164 private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
165 private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
166 private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
167 private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
169 public TestFilter(TokenStream in) {
174 public final boolean incrementToken() throws java.io.IOException {
175 if (multiToken > 0) {
176 termAtt.setEmpty().append("multi" + (multiToken + 1));
177 offsetAtt.setOffset(prevStartOffset, prevEndOffset);
178 typeAtt.setType(prevType);
179 posIncrAtt.setPositionIncrement(0);
183 boolean next = input.incrementToken();
187 prevType = typeAtt.type();
188 prevStartOffset = offsetAtt.startOffset();
189 prevEndOffset = offsetAtt.endOffset();
190 String text = termAtt.toString();
191 if (text.equals("triplemulti")) {
194 } else if (text.equals("multi")) {
206 * Analyzes "the quick brown" as: quick(incr=2) brown(incr=1). Does not work
207 * correctly for input other than "the quick brown ...".
209 private class PosIncrementAnalyzer extends Analyzer {
211 public PosIncrementAnalyzer() {
215 public TokenStream tokenStream(String fieldName, Reader reader) {
216 TokenStream result = new StandardTokenizer(TEST_VERSION_CURRENT, reader);
217 result = new TestPosIncrementFilter(result);
218 result = new LowerCaseFilter(TEST_VERSION_CURRENT, result);
223 private class TestPosIncrementFilter extends TokenFilter {
225 private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
226 private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
228 public TestPosIncrementFilter(TokenStream in) {
233 public final boolean incrementToken() throws java.io.IOException {
234 while (input.incrementToken()) {
235 if (termAtt.toString().equals("the")) {
236 // stopword, do nothing
237 } else if (termAtt.toString().equals("quick")) {
238 posIncrAtt.setPositionIncrement(2);
241 posIncrAtt.setPositionIncrement(1);