1 package org.apache.lucene.queryParser.standard;
4 * Licensed to the Apache Software Foundation (ASF) under one or more
5 * contributor license agreements. See the NOTICE file distributed with
6 * this work for additional information regarding copyright ownership.
7 * The ASF licenses this file to You under the Apache License, Version 2.0
8 * (the "License"); you may not use this file except in compliance with
9 * the License. You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
20 import java.io.Reader;
21 import java.util.HashMap;
24 import org.apache.lucene.analysis.Analyzer;
25 import org.apache.lucene.analysis.MockAnalyzer;
26 import org.apache.lucene.analysis.TokenStream;
27 import org.apache.lucene.document.Document;
28 import org.apache.lucene.document.Field;
29 import org.apache.lucene.index.IndexWriter;
30 import org.apache.lucene.queryParser.core.QueryNodeException;
31 import org.apache.lucene.queryParser.standard.config.StandardQueryConfigHandler;
32 import org.apache.lucene.search.BooleanClause;
33 import org.apache.lucene.search.IndexSearcher;
34 import org.apache.lucene.search.Query;
35 import org.apache.lucene.search.ScoreDoc;
36 import org.apache.lucene.search.BooleanClause.Occur;
37 import org.apache.lucene.store.Directory;
38 import org.apache.lucene.util.LuceneTestCase;
41 * This test case is a copy of the core Lucene query parser test, it was adapted
42 * to use new QueryParserHelper instead of the old query parser.
46 public class TestMultiFieldQPHelper extends LuceneTestCase {
49 * test stop words parsing for both the non static form, and for the
50 * corresponding static form (qtxt, fields[]).
52 public void testStopwordsParsing() throws Exception {
53 assertStopQueryEquals("one", "b:one t:one");
54 assertStopQueryEquals("one stop", "b:one t:one");
55 assertStopQueryEquals("one (stop)", "b:one t:one");
56 assertStopQueryEquals("one ((stop))", "b:one t:one");
57 assertStopQueryEquals("stop", "");
58 assertStopQueryEquals("(stop)", "");
59 assertStopQueryEquals("((stop))", "");
62 // verify parsing of query using a stopping analyzer
63 private void assertStopQueryEquals(String qtxt, String expectedRes)
65 String[] fields = { "b", "t" };
66 Occur occur[] = { Occur.SHOULD, Occur.SHOULD };
67 TestQPHelper.QPTestAnalyzer a = new TestQPHelper.QPTestAnalyzer();
68 StandardQueryParser mfqp = new StandardQueryParser();
69 mfqp.setMultiFields(fields);
72 Query q = mfqp.parse(qtxt, null);
73 assertEquals(expectedRes, q.toString());
75 q = QueryParserUtil.parse(qtxt, fields, occur, a);
76 assertEquals(expectedRes, q.toString());
79 public void testSimple() throws Exception {
80 String[] fields = { "b", "t" };
81 StandardQueryParser mfqp = new StandardQueryParser();
82 mfqp.setMultiFields(fields);
83 mfqp.setAnalyzer(new MockAnalyzer(random));
85 Query q = mfqp.parse("one", null);
86 assertEquals("b:one t:one", q.toString());
88 q = mfqp.parse("one two", null);
89 assertEquals("(b:one t:one) (b:two t:two)", q.toString());
91 q = mfqp.parse("+one +two", null);
92 assertEquals("+(b:one t:one) +(b:two t:two)", q.toString());
94 q = mfqp.parse("+one -two -three", null);
95 assertEquals("+(b:one t:one) -(b:two t:two) -(b:three t:three)", q
98 q = mfqp.parse("one^2 two", null);
99 assertEquals("((b:one t:one)^2.0) (b:two t:two)", q.toString());
101 q = mfqp.parse("one~ two", null);
102 assertEquals("(b:one~0.5 t:one~0.5) (b:two t:two)", q.toString());
104 q = mfqp.parse("one~0.8 two^2", null);
105 assertEquals("(b:one~0.8 t:one~0.8) ((b:two t:two)^2.0)", q.toString());
107 q = mfqp.parse("one* two*", null);
108 assertEquals("(b:one* t:one*) (b:two* t:two*)", q.toString());
110 q = mfqp.parse("[a TO c] two", null);
111 assertEquals("(b:[a TO c] t:[a TO c]) (b:two t:two)", q.toString());
113 q = mfqp.parse("w?ldcard", null);
114 assertEquals("b:w?ldcard t:w?ldcard", q.toString());
116 q = mfqp.parse("\"foo bar\"", null);
117 assertEquals("b:\"foo bar\" t:\"foo bar\"", q.toString());
119 q = mfqp.parse("\"aa bb cc\" \"dd ee\"", null);
120 assertEquals("(b:\"aa bb cc\" t:\"aa bb cc\") (b:\"dd ee\" t:\"dd ee\")", q
123 q = mfqp.parse("\"foo bar\"~4", null);
124 assertEquals("b:\"foo bar\"~4 t:\"foo bar\"~4", q.toString());
126 // LUCENE-1213: QueryParser was ignoring slop when phrase
128 q = mfqp.parse("b:\"foo bar\"~4", null);
129 assertEquals("b:\"foo bar\"~4", q.toString());
131 // make sure that terms which have a field are not touched:
132 q = mfqp.parse("one f:two", null);
133 assertEquals("(b:one t:one) f:two", q.toString());
136 mfqp.setDefaultOperator(StandardQueryConfigHandler.Operator.AND);
137 q = mfqp.parse("one two", null);
138 assertEquals("+(b:one t:one) +(b:two t:two)", q.toString());
139 q = mfqp.parse("\"aa bb cc\" \"dd ee\"", null);
140 assertEquals("+(b:\"aa bb cc\" t:\"aa bb cc\") +(b:\"dd ee\" t:\"dd ee\")",
145 public void testBoostsSimple() throws Exception {
146 Map<String,Float> boosts = new HashMap<String,Float>();
147 boosts.put("b", Float.valueOf(5));
148 boosts.put("t", Float.valueOf(10));
149 String[] fields = { "b", "t" };
150 StandardQueryParser mfqp = new StandardQueryParser();
151 mfqp.setMultiFields(fields);
152 mfqp.setFieldsBoost(boosts);
153 mfqp.setAnalyzer(new MockAnalyzer(random));
156 Query q = mfqp.parse("one", null);
157 assertEquals("b:one^5.0 t:one^10.0", q.toString());
160 q = mfqp.parse("one AND two", null);
161 assertEquals("+(b:one^5.0 t:one^10.0) +(b:two^5.0 t:two^10.0)", q
165 q = mfqp.parse("one OR two", null);
166 assertEquals("(b:one^5.0 t:one^10.0) (b:two^5.0 t:two^10.0)", q.toString());
168 // Check for AND and a field
169 q = mfqp.parse("one AND two AND foo:test", null);
170 assertEquals("+(b:one^5.0 t:one^10.0) +(b:two^5.0 t:two^10.0) +foo:test", q
173 q = mfqp.parse("one^3 AND two^4", null);
174 assertEquals("+((b:one^5.0 t:one^10.0)^3.0) +((b:two^5.0 t:two^10.0)^4.0)",
178 public void testStaticMethod1() throws QueryNodeException {
179 String[] fields = { "b", "t" };
180 String[] queries = { "one", "two" };
181 Query q = QueryParserUtil.parse(queries, fields, new MockAnalyzer(random));
182 assertEquals("b:one t:two", q.toString());
184 String[] queries2 = { "+one", "+two" };
185 q = QueryParserUtil.parse(queries2, fields, new MockAnalyzer(random));
186 assertEquals("(+b:one) (+t:two)", q.toString());
188 String[] queries3 = { "one", "+two" };
189 q = QueryParserUtil.parse(queries3, fields, new MockAnalyzer(random));
190 assertEquals("b:one (+t:two)", q.toString());
192 String[] queries4 = { "one +more", "+two" };
193 q = QueryParserUtil.parse(queries4, fields, new MockAnalyzer(random));
194 assertEquals("(b:one +b:more) (+t:two)", q.toString());
196 String[] queries5 = { "blah" };
198 q = QueryParserUtil.parse(queries5, fields, new MockAnalyzer(random));
200 } catch (IllegalArgumentException e) {
201 // expected exception, array length differs
204 // check also with stop words for this static form (qtxts[], fields[]).
205 TestQPHelper.QPTestAnalyzer stopA = new TestQPHelper.QPTestAnalyzer();
207 String[] queries6 = { "((+stop))", "+((stop))" };
208 q = QueryParserUtil.parse(queries6, fields, stopA);
209 assertEquals("", q.toString());
211 String[] queries7 = { "one ((+stop)) +more", "+((stop)) +two" };
212 q = QueryParserUtil.parse(queries7, fields, stopA);
213 assertEquals("(b:one +b:more) (+t:two)", q.toString());
217 public void testStaticMethod2() throws QueryNodeException {
218 String[] fields = { "b", "t" };
219 BooleanClause.Occur[] flags = { BooleanClause.Occur.MUST,
220 BooleanClause.Occur.MUST_NOT };
221 Query q = QueryParserUtil.parse("one", fields, flags,
222 new MockAnalyzer(random));
223 assertEquals("+b:one -t:one", q.toString());
225 q = QueryParserUtil.parse("one two", fields, flags, new MockAnalyzer(random));
226 assertEquals("+(b:one b:two) -(t:one t:two)", q.toString());
229 BooleanClause.Occur[] flags2 = { BooleanClause.Occur.MUST };
230 q = QueryParserUtil.parse("blah", fields, flags2, new MockAnalyzer(random));
232 } catch (IllegalArgumentException e) {
233 // expected exception, array length differs
237 public void testStaticMethod2Old() throws QueryNodeException {
238 String[] fields = { "b", "t" };
239 BooleanClause.Occur[] flags = { BooleanClause.Occur.MUST,
240 BooleanClause.Occur.MUST_NOT };
241 StandardQueryParser parser = new StandardQueryParser();
242 parser.setMultiFields(fields);
243 parser.setAnalyzer(new MockAnalyzer(random));
245 Query q = QueryParserUtil.parse("one", fields, flags,
246 new MockAnalyzer(random));// , fields, flags, new
248 assertEquals("+b:one -t:one", q.toString());
250 q = QueryParserUtil.parse("one two", fields, flags, new MockAnalyzer(random));
251 assertEquals("+(b:one b:two) -(t:one t:two)", q.toString());
254 BooleanClause.Occur[] flags2 = { BooleanClause.Occur.MUST };
255 q = QueryParserUtil.parse("blah", fields, flags2, new MockAnalyzer(random));
257 } catch (IllegalArgumentException e) {
258 // expected exception, array length differs
262 public void testStaticMethod3() throws QueryNodeException {
263 String[] queries = { "one", "two", "three" };
264 String[] fields = { "f1", "f2", "f3" };
265 BooleanClause.Occur[] flags = { BooleanClause.Occur.MUST,
266 BooleanClause.Occur.MUST_NOT, BooleanClause.Occur.SHOULD };
267 Query q = QueryParserUtil.parse(queries, fields, flags,
268 new MockAnalyzer(random));
269 assertEquals("+f1:one -f2:two f3:three", q.toString());
272 BooleanClause.Occur[] flags2 = { BooleanClause.Occur.MUST };
274 .parse(queries, fields, flags2, new MockAnalyzer(random));
276 } catch (IllegalArgumentException e) {
277 // expected exception, array length differs
281 public void testStaticMethod3Old() throws QueryNodeException {
282 String[] queries = { "one", "two" };
283 String[] fields = { "b", "t" };
284 BooleanClause.Occur[] flags = { BooleanClause.Occur.MUST,
285 BooleanClause.Occur.MUST_NOT };
286 Query q = QueryParserUtil.parse(queries, fields, flags,
287 new MockAnalyzer(random));
288 assertEquals("+b:one -t:two", q.toString());
291 BooleanClause.Occur[] flags2 = { BooleanClause.Occur.MUST };
293 .parse(queries, fields, flags2, new MockAnalyzer(random));
295 } catch (IllegalArgumentException e) {
296 // expected exception, array length differs
300 public void testAnalyzerReturningNull() throws QueryNodeException {
301 String[] fields = new String[] { "f1", "f2", "f3" };
302 StandardQueryParser parser = new StandardQueryParser();
303 parser.setMultiFields(fields);
304 parser.setAnalyzer(new AnalyzerReturningNull());
306 Query q = parser.parse("bla AND blo", null);
307 assertEquals("+(f2:bla f3:bla) +(f2:blo f3:blo)", q.toString());
308 // the following queries are not affected as their terms are not
310 q = parser.parse("bla*", null);
311 assertEquals("f1:bla* f2:bla* f3:bla*", q.toString());
312 q = parser.parse("bla~", null);
313 assertEquals("f1:bla~0.5 f2:bla~0.5 f3:bla~0.5", q.toString());
314 q = parser.parse("[a TO c]", null);
315 assertEquals("f1:[a TO c] f2:[a TO c] f3:[a TO c]", q.toString());
318 public void testStopWordSearching() throws Exception {
319 Analyzer analyzer = new MockAnalyzer(random);
320 Directory ramDir = newDirectory();
321 IndexWriter iw = new IndexWriter(ramDir, newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer));
322 Document doc = new Document();
323 doc.add(newField("body", "blah the footest blah", Field.Store.NO,
324 Field.Index.ANALYZED));
328 StandardQueryParser mfqp = new StandardQueryParser();
330 mfqp.setMultiFields(new String[] { "body" });
331 mfqp.setAnalyzer(analyzer);
332 mfqp.setDefaultOperator(StandardQueryConfigHandler.Operator.AND);
333 Query q = mfqp.parse("the footest", null);
334 IndexSearcher is = new IndexSearcher(ramDir, true);
335 ScoreDoc[] hits = is.search(q, null, 1000).scoreDocs;
336 assertEquals(1, hits.length);
342 * Return empty tokens for field "f1".
344 private static final class AnalyzerReturningNull extends Analyzer {
345 MockAnalyzer stdAnalyzer = new MockAnalyzer(random);
347 public AnalyzerReturningNull() {
351 public TokenStream tokenStream(String fieldName, Reader reader) {
352 if ("f1".equals(fieldName)) {
353 return new EmptyTokenStream();
355 return stdAnalyzer.tokenStream(fieldName, reader);
359 private static class EmptyTokenStream extends TokenStream {
361 public boolean incrementToken() {