1 package org.apache.lucene.queryParser;
4 * Licensed to the Apache Software Foundation (ASF) under one or more
5 * contributor license agreements. See the NOTICE file distributed with
6 * this work for additional information regarding copyright ownership.
7 * The ASF licenses this file to You under the Apache License, Version 2.0
8 * (the "License"); you may not use this file except in compliance with
9 * the License. You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
20 import java.io.IOException;
21 import java.io.Reader;
22 import java.util.HashMap;
25 import org.apache.lucene.analysis.Analyzer;
26 import org.apache.lucene.analysis.MockAnalyzer;
27 import org.apache.lucene.analysis.TokenStream;
28 import org.apache.lucene.analysis.standard.StandardAnalyzer;
29 import org.apache.lucene.document.Document;
30 import org.apache.lucene.document.Field;
31 import org.apache.lucene.index.IndexWriter;
32 import org.apache.lucene.search.BooleanClause;
33 import org.apache.lucene.search.IndexSearcher;
34 import org.apache.lucene.search.Query;
35 import org.apache.lucene.search.ScoreDoc;
36 import org.apache.lucene.search.BooleanClause.Occur;
37 import org.apache.lucene.store.Directory;
38 import org.apache.lucene.util.LuceneTestCase;
43 public class TestMultiFieldQueryParser extends LuceneTestCase {
45 /** test stop words parsing for both the non static form, and for the
46 * corresponding static form (qtxt, fields[]). */
47 public void testStopwordsParsing() throws Exception {
48 assertStopQueryEquals("one", "b:one t:one");
49 assertStopQueryEquals("one stop", "b:one t:one");
50 assertStopQueryEquals("one (stop)", "b:one t:one");
51 assertStopQueryEquals("one ((stop))", "b:one t:one");
52 assertStopQueryEquals("stop", "");
53 assertStopQueryEquals("(stop)", "");
54 assertStopQueryEquals("((stop))", "");
57 // verify parsing of query using a stopping analyzer
58 private void assertStopQueryEquals (String qtxt, String expectedRes) throws Exception {
59 String[] fields = {"b", "t"};
60 Occur occur[] = {Occur.SHOULD, Occur.SHOULD};
61 TestQueryParser.QPTestAnalyzer a = new TestQueryParser.QPTestAnalyzer();
62 MultiFieldQueryParser mfqp = new MultiFieldQueryParser(TEST_VERSION_CURRENT, fields, a);
64 Query q = mfqp.parse(qtxt);
65 assertEquals(expectedRes, q.toString());
67 q = MultiFieldQueryParser.parse(TEST_VERSION_CURRENT, qtxt, fields, occur, a);
68 assertEquals(expectedRes, q.toString());
71 public void testSimple() throws Exception {
72 String[] fields = {"b", "t"};
73 MultiFieldQueryParser mfqp = new MultiFieldQueryParser(TEST_VERSION_CURRENT, fields, new MockAnalyzer(random));
75 Query q = mfqp.parse("one");
76 assertEquals("b:one t:one", q.toString());
78 q = mfqp.parse("one two");
79 assertEquals("(b:one t:one) (b:two t:two)", q.toString());
81 q = mfqp.parse("+one +two");
82 assertEquals("+(b:one t:one) +(b:two t:two)", q.toString());
84 q = mfqp.parse("+one -two -three");
85 assertEquals("+(b:one t:one) -(b:two t:two) -(b:three t:three)", q.toString());
87 q = mfqp.parse("one^2 two");
88 assertEquals("((b:one t:one)^2.0) (b:two t:two)", q.toString());
90 q = mfqp.parse("one~ two");
91 assertEquals("(b:one~0.5 t:one~0.5) (b:two t:two)", q.toString());
93 q = mfqp.parse("one~0.8 two^2");
94 assertEquals("(b:one~0.8 t:one~0.8) ((b:two t:two)^2.0)", q.toString());
96 q = mfqp.parse("one* two*");
97 assertEquals("(b:one* t:one*) (b:two* t:two*)", q.toString());
99 q = mfqp.parse("[a TO c] two");
100 assertEquals("(b:[a TO c] t:[a TO c]) (b:two t:two)", q.toString());
102 q = mfqp.parse("w?ldcard");
103 assertEquals("b:w?ldcard t:w?ldcard", q.toString());
105 q = mfqp.parse("\"foo bar\"");
106 assertEquals("b:\"foo bar\" t:\"foo bar\"", q.toString());
108 q = mfqp.parse("\"aa bb cc\" \"dd ee\"");
109 assertEquals("(b:\"aa bb cc\" t:\"aa bb cc\") (b:\"dd ee\" t:\"dd ee\")", q.toString());
111 q = mfqp.parse("\"foo bar\"~4");
112 assertEquals("b:\"foo bar\"~4 t:\"foo bar\"~4", q.toString());
114 // LUCENE-1213: MultiFieldQueryParser was ignoring slop when phrase had a field.
115 q = mfqp.parse("b:\"foo bar\"~4");
116 assertEquals("b:\"foo bar\"~4", q.toString());
118 // make sure that terms which have a field are not touched:
119 q = mfqp.parse("one f:two");
120 assertEquals("(b:one t:one) f:two", q.toString());
123 mfqp.setDefaultOperator(QueryParser.AND_OPERATOR);
124 q = mfqp.parse("one two");
125 assertEquals("+(b:one t:one) +(b:two t:two)", q.toString());
126 q = mfqp.parse("\"aa bb cc\" \"dd ee\"");
127 assertEquals("+(b:\"aa bb cc\" t:\"aa bb cc\") +(b:\"dd ee\" t:\"dd ee\")", q.toString());
131 public void testBoostsSimple() throws Exception {
132 Map<String,Float> boosts = new HashMap<String,Float>();
133 boosts.put("b", Float.valueOf(5));
134 boosts.put("t", Float.valueOf(10));
135 String[] fields = {"b", "t"};
136 MultiFieldQueryParser mfqp = new MultiFieldQueryParser(TEST_VERSION_CURRENT, fields, new MockAnalyzer(random), boosts);
140 Query q = mfqp.parse("one");
141 assertEquals("b:one^5.0 t:one^10.0", q.toString());
144 q = mfqp.parse("one AND two");
145 assertEquals("+(b:one^5.0 t:one^10.0) +(b:two^5.0 t:two^10.0)", q.toString());
148 q = mfqp.parse("one OR two");
149 assertEquals("(b:one^5.0 t:one^10.0) (b:two^5.0 t:two^10.0)", q.toString());
151 //Check for AND and a field
152 q = mfqp.parse("one AND two AND foo:test");
153 assertEquals("+(b:one^5.0 t:one^10.0) +(b:two^5.0 t:two^10.0) +foo:test", q.toString());
155 q = mfqp.parse("one^3 AND two^4");
156 assertEquals("+((b:one^5.0 t:one^10.0)^3.0) +((b:two^5.0 t:two^10.0)^4.0)", q.toString());
159 public void testStaticMethod1() throws ParseException {
160 String[] fields = {"b", "t"};
161 String[] queries = {"one", "two"};
162 Query q = MultiFieldQueryParser.parse(TEST_VERSION_CURRENT, queries, fields, new MockAnalyzer(random));
163 assertEquals("b:one t:two", q.toString());
165 String[] queries2 = {"+one", "+two"};
166 q = MultiFieldQueryParser.parse(TEST_VERSION_CURRENT, queries2, fields, new MockAnalyzer(random));
167 assertEquals("(+b:one) (+t:two)", q.toString());
169 String[] queries3 = {"one", "+two"};
170 q = MultiFieldQueryParser.parse(TEST_VERSION_CURRENT, queries3, fields, new MockAnalyzer(random));
171 assertEquals("b:one (+t:two)", q.toString());
173 String[] queries4 = {"one +more", "+two"};
174 q = MultiFieldQueryParser.parse(TEST_VERSION_CURRENT, queries4, fields, new MockAnalyzer(random));
175 assertEquals("(b:one +b:more) (+t:two)", q.toString());
177 String[] queries5 = {"blah"};
179 q = MultiFieldQueryParser.parse(TEST_VERSION_CURRENT, queries5, fields, new MockAnalyzer(random));
181 } catch(IllegalArgumentException e) {
182 // expected exception, array length differs
185 // check also with stop words for this static form (qtxts[], fields[]).
186 TestQueryParser.QPTestAnalyzer stopA = new TestQueryParser.QPTestAnalyzer();
188 String[] queries6 = {"((+stop))", "+((stop))"};
189 q = MultiFieldQueryParser.parse(TEST_VERSION_CURRENT, queries6, fields, stopA);
190 assertEquals("", q.toString());
192 String[] queries7 = {"one ((+stop)) +more", "+((stop)) +two"};
193 q = MultiFieldQueryParser.parse(TEST_VERSION_CURRENT, queries7, fields, stopA);
194 assertEquals("(b:one +b:more) (+t:two)", q.toString());
198 public void testStaticMethod2() throws ParseException {
199 String[] fields = {"b", "t"};
200 BooleanClause.Occur[] flags = {BooleanClause.Occur.MUST, BooleanClause.Occur.MUST_NOT};
201 Query q = MultiFieldQueryParser.parse(TEST_VERSION_CURRENT, "one", fields, flags, new MockAnalyzer(random));
202 assertEquals("+b:one -t:one", q.toString());
204 q = MultiFieldQueryParser.parse(TEST_VERSION_CURRENT, "one two", fields, flags, new MockAnalyzer(random));
205 assertEquals("+(b:one b:two) -(t:one t:two)", q.toString());
208 BooleanClause.Occur[] flags2 = {BooleanClause.Occur.MUST};
209 q = MultiFieldQueryParser.parse(TEST_VERSION_CURRENT, "blah", fields, flags2, new MockAnalyzer(random));
211 } catch(IllegalArgumentException e) {
212 // expected exception, array length differs
216 public void testStaticMethod2Old() throws ParseException {
217 String[] fields = {"b", "t"};
218 //int[] flags = {MultiFieldQueryParser.REQUIRED_FIELD, MultiFieldQueryParser.PROHIBITED_FIELD};
219 BooleanClause.Occur[] flags = {BooleanClause.Occur.MUST, BooleanClause.Occur.MUST_NOT};
221 Query q = MultiFieldQueryParser.parse(TEST_VERSION_CURRENT, "one", fields, flags, new MockAnalyzer(random));//, fields, flags, new MockAnalyzer(random));
222 assertEquals("+b:one -t:one", q.toString());
224 q = MultiFieldQueryParser.parse(TEST_VERSION_CURRENT, "one two", fields, flags, new MockAnalyzer(random));
225 assertEquals("+(b:one b:two) -(t:one t:two)", q.toString());
228 BooleanClause.Occur[] flags2 = {BooleanClause.Occur.MUST};
229 q = MultiFieldQueryParser.parse(TEST_VERSION_CURRENT, "blah", fields, flags2, new MockAnalyzer(random));
231 } catch(IllegalArgumentException e) {
232 // expected exception, array length differs
236 public void testStaticMethod3() throws ParseException {
237 String[] queries = {"one", "two", "three"};
238 String[] fields = {"f1", "f2", "f3"};
239 BooleanClause.Occur[] flags = {BooleanClause.Occur.MUST,
240 BooleanClause.Occur.MUST_NOT, BooleanClause.Occur.SHOULD};
241 Query q = MultiFieldQueryParser.parse(TEST_VERSION_CURRENT, queries, fields, flags, new MockAnalyzer(random));
242 assertEquals("+f1:one -f2:two f3:three", q.toString());
245 BooleanClause.Occur[] flags2 = {BooleanClause.Occur.MUST};
246 q = MultiFieldQueryParser.parse(TEST_VERSION_CURRENT, queries, fields, flags2, new MockAnalyzer(random));
248 } catch(IllegalArgumentException e) {
249 // expected exception, array length differs
253 public void testStaticMethod3Old() throws ParseException {
254 String[] queries = {"one", "two"};
255 String[] fields = {"b", "t"};
256 BooleanClause.Occur[] flags = {BooleanClause.Occur.MUST, BooleanClause.Occur.MUST_NOT};
257 Query q = MultiFieldQueryParser.parse(TEST_VERSION_CURRENT, queries, fields, flags, new MockAnalyzer(random));
258 assertEquals("+b:one -t:two", q.toString());
261 BooleanClause.Occur[] flags2 = {BooleanClause.Occur.MUST};
262 q = MultiFieldQueryParser.parse(TEST_VERSION_CURRENT, queries, fields, flags2, new MockAnalyzer(random));
264 } catch(IllegalArgumentException e) {
265 // expected exception, array length differs
269 public void testAnalyzerReturningNull() throws ParseException {
270 String[] fields = new String[] { "f1", "f2", "f3" };
271 MultiFieldQueryParser parser = new MultiFieldQueryParser(TEST_VERSION_CURRENT, fields, new AnalyzerReturningNull());
272 Query q = parser.parse("bla AND blo");
273 assertEquals("+(f2:bla f3:bla) +(f2:blo f3:blo)", q.toString());
274 // the following queries are not affected as their terms are not analyzed anyway:
275 q = parser.parse("bla*");
276 assertEquals("f1:bla* f2:bla* f3:bla*", q.toString());
277 q = parser.parse("bla~");
278 assertEquals("f1:bla~0.5 f2:bla~0.5 f3:bla~0.5", q.toString());
279 q = parser.parse("[a TO c]");
280 assertEquals("f1:[a TO c] f2:[a TO c] f3:[a TO c]", q.toString());
283 public void testStopWordSearching() throws Exception {
284 Analyzer analyzer = new MockAnalyzer(random);
285 Directory ramDir = newDirectory();
286 IndexWriter iw = new IndexWriter(ramDir, newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer));
287 Document doc = new Document();
288 doc.add(newField("body", "blah the footest blah", Field.Store.NO, Field.Index.ANALYZED));
292 MultiFieldQueryParser mfqp =
293 new MultiFieldQueryParser(TEST_VERSION_CURRENT, new String[] {"body"}, analyzer);
294 mfqp.setDefaultOperator(QueryParser.Operator.AND);
295 Query q = mfqp.parse("the footest");
296 IndexSearcher is = new IndexSearcher(ramDir, true);
297 ScoreDoc[] hits = is.search(q, null, 1000).scoreDocs;
298 assertEquals(1, hits.length);
304 * Return empty tokens for field "f1".
306 private static class AnalyzerReturningNull extends Analyzer {
307 MockAnalyzer stdAnalyzer = new MockAnalyzer(random);
309 public AnalyzerReturningNull() {
313 public TokenStream tokenStream(String fieldName, Reader reader) {
314 if ("f1".equals(fieldName)) {
315 return new EmptyTokenStream();
317 return stdAnalyzer.tokenStream(fieldName, reader);
321 private static class EmptyTokenStream extends TokenStream {
323 public boolean incrementToken() throws IOException {