1 package org.apache.lucene.search;
4 * Licensed to the Apache Software Foundation (ASF) under one or more
5 * contributor license agreements. See the NOTICE file distributed with
6 * this work for additional information regarding copyright ownership.
7 * The ASF licenses this file to You under the Apache License, Version 2.0
8 * (the "License"); you may not use this file except in compliance with
9 * the License. You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
20 import java.io.ByteArrayInputStream;
21 import java.io.ByteArrayOutputStream;
22 import java.io.IOException;
23 import java.io.ObjectInputStream;
24 import java.io.ObjectOutputStream;
25 import java.util.Random;
27 import junit.framework.Assert;
29 import org.apache.lucene.analysis.MockAnalyzer;
30 import org.apache.lucene.document.Document;
31 import org.apache.lucene.index.IndexReader;
32 import org.apache.lucene.index.IndexWriter;
33 import org.apache.lucene.index.IndexWriterConfig;
34 import org.apache.lucene.index.MultiReader;
35 import org.apache.lucene.store.Directory;
36 import org.apache.lucene.store.MockDirectoryWrapper;
37 import org.apache.lucene.store.RAMDirectory;
38 import org.apache.lucene.util._TestUtil;
40 import static org.apache.lucene.util.LuceneTestCase.TEST_VERSION_CURRENT;
45 public class QueryUtils {
47 /** Check the types of things query objects should be able to do. */
48 public static void check(Query q) {
52 /** check very basic hashCode and equals */
53 public static void checkHashEquals(Query q) {
54 Query q2 = (Query)q.clone();
57 Query q3 = (Query)q.clone();
58 q3.setBoost(7.21792348f);
61 // test that a class check is done so that no exception is thrown
62 // in the implementation of equals()
63 Query whacky = new Query() {
65 public String toString(String field) {
66 return "My Whacky Query";
69 whacky.setBoost(q.getBoost());
70 checkUnequal(q, whacky);
73 Assert.assertFalse(q.equals(null));
76 public static void checkEqual(Query q1, Query q2) {
77 Assert.assertEquals(q1, q2);
78 Assert.assertEquals(q1.hashCode(), q2.hashCode());
81 public static void checkUnequal(Query q1, Query q2) {
82 Assert.assertTrue(!q1.equals(q2));
83 Assert.assertTrue(!q2.equals(q1));
85 // possible this test can fail on a hash collision... if that
86 // happens, please change test to use a different example.
87 Assert.assertTrue(q1.hashCode() != q2.hashCode());
90 /** deep check that explanations of a query 'score' correctly */
91 public static void checkExplanations (final Query q, final Searcher s) throws IOException {
92 CheckHits.checkExplanations(q, null, s, true);
96 * Various query sanity checks on a searcher, some checks are only done for
97 * instanceof IndexSearcher.
100 * @see #checkFirstSkipTo
102 * @see #checkExplanations
103 * @see #checkSerialization
106 public static void check(Random random, Query q1, Searcher s) {
107 check(random, q1, s, true);
109 private static void check(Random random, Query q1, Searcher s, boolean wrap) {
113 if (s instanceof IndexSearcher) {
114 IndexSearcher is = (IndexSearcher)s;
115 checkFirstSkipTo(q1,is);
118 check(random, q1, wrapUnderlyingReader(random, is, -1), false);
119 check(random, q1, wrapUnderlyingReader(random, is, 0), false);
120 check(random, q1, wrapUnderlyingReader(random, is, +1), false);
124 check(random,q1, wrapSearcher(random, s, -1), false);
125 check(random,q1, wrapSearcher(random, s, 0), false);
126 check(random,q1, wrapSearcher(random, s, +1), false);
128 checkExplanations(q1,s);
129 checkSerialization(q1,s);
131 Query q2 = (Query)q1.clone();
132 checkEqual(s.rewrite(q1),
135 } catch (IOException e) {
136 throw new RuntimeException(e);
141 * Given an IndexSearcher, returns a new IndexSearcher whose IndexReader
142 * is a MultiReader containing the Reader of the original IndexSearcher,
143 * as well as several "empty" IndexReaders -- some of which will have
144 * deleted documents in them. This new IndexSearcher should
145 * behave exactly the same as the original IndexSearcher.
146 * @param s the searcher to wrap
147 * @param edge if negative, s will be the first sub; if 0, s will be in the middle, if positive s will be the last sub
149 public static IndexSearcher wrapUnderlyingReader(Random random, final IndexSearcher s, final int edge)
152 IndexReader r = s.getIndexReader();
154 // we can't put deleted docs before the nested reader, because
155 // it will throw off the docIds
156 IndexReader[] readers = new IndexReader[] {
157 edge < 0 ? r : IndexReader.open(makeEmptyIndex(random, 0), true),
158 IndexReader.open(makeEmptyIndex(random, 0), true),
159 new MultiReader(new IndexReader[] {
160 IndexReader.open(makeEmptyIndex(random, edge < 0 ? 4 : 0), true),
161 IndexReader.open(makeEmptyIndex(random, 0), true),
162 0 == edge ? r : IndexReader.open(makeEmptyIndex(random, 0), true)
164 IndexReader.open(makeEmptyIndex(random, 0 < edge ? 0 : 7), true),
165 IndexReader.open(makeEmptyIndex(random, 0), true),
166 new MultiReader(new IndexReader[] {
167 IndexReader.open(makeEmptyIndex(random, 0 < edge ? 0 : 5), true),
168 IndexReader.open(makeEmptyIndex(random, 0), true),
169 0 < edge ? r : IndexReader.open(makeEmptyIndex(random, 0), true)
172 IndexSearcher out = new IndexSearcher(new MultiReader(readers));
173 out.setSimilarity(s.getSimilarity());
177 * Given a Searcher, returns a new MultiSearcher wrapping the
178 * the original Searcher,
179 * as well as several "empty" IndexSearchers -- some of which will have
180 * deleted documents in them. This new MultiSearcher
181 * should behave exactly the same as the original Searcher.
182 * @param s the Searcher to wrap
183 * @param edge if negative, s will be the first sub; if 0, s will be in hte middle, if positive s will be the last sub
185 public static MultiSearcher wrapSearcher(Random random, final Searcher s, final int edge)
188 // we can't put deleted docs before the nested reader, because
189 // it will through off the docIds
190 Searcher[] searchers = new Searcher[] {
191 edge < 0 ? s : new IndexSearcher(makeEmptyIndex(random, 0), true),
192 new MultiSearcher(new Searcher[] {
193 new IndexSearcher(makeEmptyIndex(random, edge < 0 ? 65 : 0), true),
194 new IndexSearcher(makeEmptyIndex(random, 0), true),
195 0 == edge ? s : new IndexSearcher(makeEmptyIndex(random, 0), true)
197 new IndexSearcher(makeEmptyIndex(random, 0 < edge ? 0 : 3), true),
198 new IndexSearcher(makeEmptyIndex(random, 0), true),
199 new MultiSearcher(new Searcher[] {
200 new IndexSearcher(makeEmptyIndex(random, 0 < edge ? 0 : 5), true),
201 new IndexSearcher(makeEmptyIndex(random, 0), true),
202 0 < edge ? s : new IndexSearcher(makeEmptyIndex(random, 0), true)
205 MultiSearcher out = new MultiSearcher(searchers);
206 out.setSimilarity(s.getSimilarity());
210 private static Directory makeEmptyIndex(Random random, final int numDeletedDocs)
212 Directory d = new MockDirectoryWrapper(random, new RAMDirectory());
213 IndexWriter w = new IndexWriter(d, new IndexWriterConfig(
214 TEST_VERSION_CURRENT, new MockAnalyzer(random)));
216 for (int i = 0; i < numDeletedDocs; i++) {
217 w.addDocument(new Document());
220 w.deleteDocuments( new MatchAllDocsQuery() );
221 _TestUtil.keepFullyDeletedSegments(w);
224 if (0 < numDeletedDocs)
225 Assert.assertTrue("writer has no deletions", w.hasDeletions());
227 Assert.assertEquals("writer is missing some deleted docs",
228 numDeletedDocs, w.maxDoc());
229 Assert.assertEquals("writer has non-deleted docs",
232 IndexReader r = IndexReader.open(d, true);
233 Assert.assertEquals("reader has wrong number of deleted docs",
234 numDeletedDocs, r.numDeletedDocs());
240 /** check that the query weight is serializable.
241 * @throws IOException if serialization check fail.
243 private static void checkSerialization(Query q, Searcher s) throws IOException {
244 Weight w = s.createNormalizedWeight(q);
246 ByteArrayOutputStream bos = new ByteArrayOutputStream();
247 ObjectOutputStream oos = new ObjectOutputStream(bos);
250 ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(bos.toByteArray()));
254 //skip equals() test for now - most weights don't override equals() and we won't add this just for the tests.
255 //TestCase.assertEquals("writeObject(w) != w. ("+w+")",w2,w);
257 } catch (Exception e) {
258 IOException e2 = new IOException("Serialization failed for "+w);
265 /** alternate scorer skipTo(),skipTo(),next(),next(),skipTo(),skipTo(), etc
266 * and ensure a hitcollector receives same docs and scores
268 public static void checkSkipTo(final Query q, final IndexSearcher s) throws IOException {
269 //System.out.println("Checking "+q);
271 if (s.createNormalizedWeight(q).scoresDocsOutOfOrder()) return; // in this case order of skipTo() might differ from that of next().
273 final int skip_op = 0;
274 final int next_op = 1;
275 final int orders [][] = {
280 {skip_op, skip_op, next_op, next_op},
281 {next_op, next_op, skip_op, skip_op},
282 {skip_op, skip_op, skip_op, next_op, next_op},
284 for (int k = 0; k < orders.length; k++) {
286 final int order[] = orders[k];
287 // System.out.print("Order:");for (int i = 0; i < order.length; i++)
288 // System.out.print(order[i]==skip_op ? " skip()":" next()");
289 // System.out.println();
290 final int opidx[] = { 0 };
291 final int lastDoc[] = {-1};
293 // FUTURE: ensure scorer.doc()==-1
295 final float maxDiff = 1e-5f;
296 final IndexReader lastReader[] = {null};
298 s.search(q, new Collector() {
300 private IndexReader reader;
301 private Scorer scorer;
304 public void setScorer(Scorer scorer) throws IOException {
309 public void collect(int doc) throws IOException {
310 float score = sc.score();
313 if (scorer == null) {
314 Weight w = s.createNormalizedWeight(q);
315 scorer = w.scorer(reader, true, false);
318 int op = order[(opidx[0]++) % order.length];
319 // System.out.println(op==skip_op ?
320 // "skip("+(sdoc[0]+1)+")":"next()");
321 boolean more = op == skip_op ? scorer.advance(scorer.docID() + 1) != DocIdSetIterator.NO_MORE_DOCS
322 : scorer.nextDoc() != DocIdSetIterator.NO_MORE_DOCS;
323 int scorerDoc = scorer.docID();
324 float scorerScore = scorer.score();
325 float scorerScore2 = scorer.score();
326 float scoreDiff = Math.abs(score - scorerScore);
327 float scorerDiff = Math.abs(scorerScore2 - scorerScore);
328 if (!more || doc != scorerDoc || scoreDiff > maxDiff
329 || scorerDiff > maxDiff) {
330 StringBuilder sbord = new StringBuilder();
331 for (int i = 0; i < order.length; i++)
332 sbord.append(order[i] == skip_op ? " skip()" : " next()");
333 throw new RuntimeException("ERROR matching docs:" + "\n\t"
334 + (doc != scorerDoc ? "--> " : "") + "doc=" + doc + ", scorerDoc=" + scorerDoc
335 + "\n\t" + (!more ? "--> " : "") + "tscorer.more=" + more
336 + "\n\t" + (scoreDiff > maxDiff ? "--> " : "")
337 + "scorerScore=" + scorerScore + " scoreDiff=" + scoreDiff
338 + " maxDiff=" + maxDiff + "\n\t"
339 + (scorerDiff > maxDiff ? "--> " : "") + "scorerScore2="
340 + scorerScore2 + " scorerDiff=" + scorerDiff
341 + "\n\thitCollector.doc=" + doc + " score=" + score
342 + "\n\t Scorer=" + scorer + "\n\t Query=" + q + " "
343 + q.getClass().getName() + "\n\t Searcher=" + s
344 + "\n\t Order=" + sbord + "\n\t Op="
345 + (op == skip_op ? " skip()" : " next()"));
347 } catch (IOException e) {
348 throw new RuntimeException(e);
353 public void setNextReader(IndexReader reader, int docBase) throws IOException {
354 // confirm that skipping beyond the last doc, on the
355 // previous reader, hits NO_MORE_DOCS
356 if (lastReader[0] != null) {
357 final IndexReader previousReader = lastReader[0];
358 Weight w = new IndexSearcher(previousReader).createNormalizedWeight(q);
359 Scorer scorer = w.scorer(previousReader, true, false);
360 if (scorer != null) {
361 boolean more = scorer.advance(lastDoc[0] + 1) != DocIdSetIterator.NO_MORE_DOCS;
362 Assert.assertFalse("query's last doc was "+ lastDoc[0] +" but skipTo("+(lastDoc[0]+1)+") got to "+scorer.docID(),more);
365 this.reader = lastReader[0] = reader;
371 public boolean acceptsDocsOutOfOrder() {
376 if (lastReader[0] != null) {
377 // confirm that skipping beyond the last doc, on the
378 // previous reader, hits NO_MORE_DOCS
379 final IndexReader previousReader = lastReader[0];
380 Weight w = new IndexSearcher(previousReader).createNormalizedWeight(q);
381 Scorer scorer = w.scorer(previousReader, true, false);
382 if (scorer != null) {
383 boolean more = scorer.advance(lastDoc[0] + 1) != DocIdSetIterator.NO_MORE_DOCS;
384 Assert.assertFalse("query's last doc was "+ lastDoc[0] +" but skipTo("+(lastDoc[0]+1)+") got to "+scorer.docID(),more);
390 // check that first skip on just created scorers always goes to the right doc
391 private static void checkFirstSkipTo(final Query q, final IndexSearcher s) throws IOException {
392 //System.out.println("checkFirstSkipTo: "+q);
393 final float maxDiff = 1e-3f;
394 final int lastDoc[] = {-1};
395 final IndexReader lastReader[] = {null};
397 s.search(q,new Collector() {
398 private Scorer scorer;
399 private IndexReader reader;
401 public void setScorer(Scorer scorer) throws IOException {
402 this.scorer = scorer;
405 public void collect(int doc) throws IOException {
406 //System.out.println("doc="+doc);
407 float score = scorer.score();
410 for (int i=lastDoc[0]+1; i<=doc; i++) {
411 Weight w = s.createNormalizedWeight(q);
412 Scorer scorer = w.scorer(reader, true, false);
413 Assert.assertTrue("query collected "+doc+" but skipTo("+i+") says no more docs!",scorer.advance(i) != DocIdSetIterator.NO_MORE_DOCS);
414 Assert.assertEquals("query collected "+doc+" but skipTo("+i+") got to "+scorer.docID(),doc,scorer.docID());
415 float skipToScore = scorer.score();
416 Assert.assertEquals("unstable skipTo("+i+") score!",skipToScore,scorer.score(),maxDiff);
417 Assert.assertEquals("query assigned doc "+doc+" a score of <"+score+"> but skipTo("+i+") has <"+skipToScore+">!",score,skipToScore,maxDiff);
420 } catch (IOException e) {
421 throw new RuntimeException(e);
426 public void setNextReader(IndexReader reader, int docBase) throws IOException {
427 // confirm that skipping beyond the last doc, on the
428 // previous reader, hits NO_MORE_DOCS
429 if (lastReader[0] != null) {
430 final IndexReader previousReader = lastReader[0];
431 Weight w = new IndexSearcher(previousReader).createNormalizedWeight(q);
432 Scorer scorer = w.scorer(previousReader, true, false);
434 if (scorer != null) {
435 boolean more = scorer.advance(lastDoc[0] + 1) != DocIdSetIterator.NO_MORE_DOCS;
436 Assert.assertFalse("query's last doc was "+ lastDoc[0] +" but skipTo("+(lastDoc[0]+1)+") got to "+scorer.docID(),more);
440 this.reader = lastReader[0] = reader;
444 public boolean acceptsDocsOutOfOrder() {
449 if (lastReader[0] != null) {
450 // confirm that skipping beyond the last doc, on the
451 // previous reader, hits NO_MORE_DOCS
452 final IndexReader previousReader = lastReader[0];
453 Weight w = new IndexSearcher(previousReader).createNormalizedWeight(q);
454 Scorer scorer = w.scorer(previousReader, true, false);
455 if (scorer != null) {
456 boolean more = scorer.advance(lastDoc[0] + 1) != DocIdSetIterator.NO_MORE_DOCS;
457 Assert.assertFalse("query's last doc was "+ lastDoc[0] +" but skipTo("+(lastDoc[0]+1)+") got to "+scorer.docID(),more);