1 package org.apache.lucene.benchmark.quality;
4 * Licensed to the Apache Software Foundation (ASF) under one or more
5 * contributor license agreements. See the NOTICE file distributed with
6 * this work for additional information regarding copyright ownership.
7 * The ASF licenses this file to You under the Apache License, Version 2.0
8 * (the "License"); you may not use this file except in compliance with
9 * the License. You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
20 import org.apache.lucene.benchmark.BenchmarkTestCase;
21 import org.apache.lucene.benchmark.quality.trec.TrecJudge;
22 import org.apache.lucene.benchmark.quality.trec.TrecTopicsReader;
23 import org.apache.lucene.benchmark.quality.utils.SimpleQQParser;
24 import org.apache.lucene.benchmark.quality.utils.SubmissionReport;
25 import org.apache.lucene.search.IndexSearcher;
26 import org.apache.lucene.store.Directory;
28 import java.io.BufferedReader;
30 import java.io.InputStream;
31 import java.io.InputStreamReader;
32 import java.io.PrintWriter;
35 * Test that quality run does its job.
37 * NOTE: if the default scoring or StandardAnalyzer is changed, then
38 * this test will not work correctly, as it does not dynamically
39 * generate its test trec topics/qrels!
41 public class TestQualityRun extends BenchmarkTestCase {
44 public void setUp() throws Exception {
46 copyToWorkDir("reuters.578.lines.txt.bz2");
49 public void testTrecQuality() throws Exception {
50 // first create the partial reuters index
53 int maxResults = 1000;
54 String docNameField = "doctitle"; // orig docID is in the linedoc format title
56 PrintWriter logger = VERBOSE ? new PrintWriter(System.out,true) : null;
59 InputStream topics = getClass().getResourceAsStream("trecTopics.txt");
60 TrecTopicsReader qReader = new TrecTopicsReader();
61 QualityQuery qqs[] = qReader.readQueries(new BufferedReader(new InputStreamReader(topics, "UTF-8")));
64 InputStream qrels = getClass().getResourceAsStream("trecQRels.txt");
65 Judge judge = new TrecJudge(new BufferedReader(new InputStreamReader(qrels, "UTF-8")));
67 // validate topics & judgments match each other
68 judge.validateData(qqs, logger);
70 Directory dir = newFSDirectory(new File(getWorkDir(),"index"));
71 IndexSearcher searcher = new IndexSearcher(dir, true);
73 QualityQueryParser qqParser = new SimpleQQParser("title","body");
74 QualityBenchmark qrun = new QualityBenchmark(qqs, qqParser, searcher, docNameField);
76 SubmissionReport submitLog = VERBOSE ? new SubmissionReport(logger, "TestRun") : null;
77 qrun.setMaxResults(maxResults);
78 QualityStats stats[] = qrun.execute(judge, submitLog, logger);
80 // --------- verify by the way judgments were altered for this test:
81 // for some queries, depending on m = qnum % 8
82 // m==0: avg_precision and recall are hurt, by marking fake docs as relevant
83 // m==1: precision_at_n and avg_precision are hurt, by unmarking relevant docs
84 // m==2: all precision, precision_at_n and recall are hurt.
85 // m>=3: these queries remain perfect
86 for (int i = 0; i < stats.length; i++) {
87 QualityStats s = stats[i];
91 assertTrue("avg-p should be hurt: "+s.getAvp(), 1.0 > s.getAvp());
92 assertTrue("recall should be hurt: "+s.getRecall(), 1.0 > s.getRecall());
93 for (int j = 1; j <= QualityStats.MAX_POINTS; j++) {
94 assertEquals("p_at_"+j+" should be perfect: "+s.getPrecisionAt(j), 1.0, s.getPrecisionAt(j), 1E-2);
99 assertTrue("avg-p should be hurt", 1.0 > s.getAvp());
100 assertEquals("recall should be perfect: "+s.getRecall(), 1.0, s.getRecall(), 1E-2);
101 for (int j = 1; j <= QualityStats.MAX_POINTS; j++) {
102 assertTrue("p_at_"+j+" should be hurt: "+s.getPrecisionAt(j), 1.0 > s.getPrecisionAt(j));
107 assertTrue("avg-p should be hurt: "+s.getAvp(), 1.0 > s.getAvp());
108 assertTrue("recall should be hurt: "+s.getRecall(), 1.0 > s.getRecall());
109 for (int j = 1; j <= QualityStats.MAX_POINTS; j++) {
110 assertTrue("p_at_"+j+" should be hurt: "+s.getPrecisionAt(j), 1.0 > s.getPrecisionAt(j));
115 assertEquals("avg-p should be perfect: "+s.getAvp(), 1.0, s.getAvp(), 1E-2);
116 assertEquals("recall should be perfect: "+s.getRecall(), 1.0, s.getRecall(), 1E-2);
117 for (int j = 1; j <= QualityStats.MAX_POINTS; j++) {
118 assertEquals("p_at_"+j+" should be perfect: "+s.getPrecisionAt(j), 1.0, s.getPrecisionAt(j), 1E-2);
125 QualityStats avg = QualityStats.average(stats);
127 avg.log("Average statistis:",1,logger," ");
130 assertTrue("mean avg-p should be hurt: "+avg.getAvp(), 1.0 > avg.getAvp());
131 assertTrue("avg recall should be hurt: "+avg.getRecall(), 1.0 > avg.getRecall());
132 for (int j = 1; j <= QualityStats.MAX_POINTS; j++) {
133 assertTrue("avg p_at_"+j+" should be hurt: "+avg.getPrecisionAt(j), 1.0 > avg.getPrecisionAt(j));
140 public void testTrecTopicsReader() throws Exception {
142 InputStream topicsFile = getClass().getResourceAsStream("trecTopics.txt");
143 TrecTopicsReader qReader = new TrecTopicsReader();
144 QualityQuery qqs[] = qReader.readQueries(
145 new BufferedReader(new InputStreamReader(topicsFile, "UTF-8")));
147 assertEquals(20, qqs.length);
149 QualityQuery qq = qqs[0];
150 assertEquals("statement months total 1987", qq.getValue("title"));
151 assertEquals("Topic 0 Description Line 1 Topic 0 Description Line 2",
152 qq.getValue("description"));
153 assertEquals("Topic 0 Narrative Line 1 Topic 0 Narrative Line 2",
154 qq.getValue("narrative"));
157 assertEquals("agreed 15 against five", qq.getValue("title"));
158 assertEquals("Topic 1 Description Line 1 Topic 1 Description Line 2",
159 qq.getValue("description"));
160 assertEquals("Topic 1 Narrative Line 1 Topic 1 Narrative Line 2",
161 qq.getValue("narrative"));
164 assertEquals("20 while common week", qq.getValue("title"));
165 assertEquals("Topic 19 Description Line 1 Topic 19 Description Line 2",
166 qq.getValue("description"));
167 assertEquals("Topic 19 Narrative Line 1 Topic 19 Narrative Line 2",
168 qq.getValue("narrative"));
171 // use benchmark logic to create the mini Reuters index
172 private void createReutersIndex() throws Exception {
174 String algLines[] = {
175 "# ----- properties ",
176 "content.source=org.apache.lucene.benchmark.byTask.feeds.LineDocSource",
177 "analyzer=org.apache.lucene.analysis.standard.ClassicAnalyzer",
178 "docs.file=" + getWorkDirResourcePath("reuters.578.lines.txt.bz2"),
179 "content.source.log.step=2500",
180 "doc.term.vector=false",
181 "content.source.forever=false",
182 "directory=FSDirectory",
184 "doc.tokenized=true",
192 // 2. execute the algorithm (required in every "logic" test)
193 execBenchmark(algLines);