--- /dev/null
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.benchmark.quality;
+
+import java.io.PrintWriter;
+import java.text.NumberFormat;
+import java.util.ArrayList;
+
+/**
+ * Results of quality benchmark run for a single query or for a set of queries.
+ */
+public class QualityStats {
+
+ /** Number of points for which precision is computed. */
+ public static final int MAX_POINTS = 20;
+
+ private double maxGoodPoints;
+ private double recall;
+ private double pAt[];
+ private double pReleventSum = 0;
+ private double numPoints = 0;
+ private double numGoodPoints = 0;
+ private double mrr = 0;
+ private long searchTime;
+ private long docNamesExtractTime;
+
+ /**
+ * A certain rank in which a relevant doc was found.
+ */
+ public static class RecallPoint {
+ private int rank;
+ private double recall;
+ private RecallPoint(int rank, double recall) {
+ this.rank = rank;
+ this.recall = recall;
+ }
+ /** Returns the rank: where on the list of returned docs this relevant doc appeared. */
+ public int getRank() {
+ return rank;
+ }
+ /** Returns the recall: how many relevant docs were returned up to this point, inclusive. */
+ public double getRecall() {
+ return recall;
+ }
+ }
+
+ private ArrayList<RecallPoint> recallPoints;
+
+ /**
+ * Construct a QualityStats object with anticipated maximal number of relevant hits.
+ * @param maxGoodPoints maximal possible relevant hits.
+ */
+ public QualityStats(double maxGoodPoints, long searchTime) {
+ this.maxGoodPoints = maxGoodPoints;
+ this.searchTime = searchTime;
+ this.recallPoints = new ArrayList<RecallPoint>();
+ pAt = new double[MAX_POINTS+1]; // pAt[0] unused.
+ }
+
+ /**
+ * Add a (possibly relevant) doc.
+ * @param n rank of the added doc (its ordinal position within the query results).
+ * @param isRelevant true if the added doc is relevant, false otherwise.
+ */
+ public void addResult(int n, boolean isRelevant, long docNameExtractTime) {
+ if (Math.abs(numPoints+1 - n) > 1E-6) {
+ throw new IllegalArgumentException("point "+n+" illegal after "+numPoints+" points!");
+ }
+ if (isRelevant) {
+ numGoodPoints+=1;
+ recallPoints.add(new RecallPoint(n,numGoodPoints));
+ if (recallPoints.size()==1 && n<=5) { // first point, but only within 5 top scores.
+ mrr = 1.0 / n;
+ }
+ }
+ numPoints = n;
+ double p = numGoodPoints / numPoints;
+ if (isRelevant) {
+ pReleventSum += p;
+ }
+ if (n<pAt.length) {
+ pAt[n] = p;
+ }
+ recall = maxGoodPoints<=0 ? p : numGoodPoints/maxGoodPoints;
+ docNamesExtractTime += docNameExtractTime;
+ }
+
+ /**
+ * Return the precision at rank n:
+ * |{relevant hits within first <code>n</code> hits}| / <code>n</code>.
+ * @param n requested precision point, must be at least 1 and at most {@link #MAX_POINTS}.
+ */
+ public double getPrecisionAt(int n) {
+ if (n<1 || n>MAX_POINTS) {
+ throw new IllegalArgumentException("n="+n+" - but it must be in [1,"+MAX_POINTS+"] range!");
+ }
+ if (n>numPoints) {
+ return (numPoints * pAt[(int)numPoints])/n;
+ }
+ return pAt[n];
+ }
+
+ /**
+ * Return the average precision at recall points.
+ */
+ public double getAvp() {
+ return maxGoodPoints==0 ? 0 : pReleventSum/maxGoodPoints;
+ }
+
+ /**
+ * Return the recall: |{relevant hits found}| / |{relevant hits existing}|.
+ */
+ public double getRecall() {
+ return recall;
+ }
+
+ /**
+ * Log information on this QualityStats object.
+ * @param logger Logger.
+ * @param prefix prefix before each log line.
+ */
+ public void log(String title, int paddLines, PrintWriter logger, String prefix) {
+ for (int i=0; i<paddLines; i++) {
+ logger.println();
+ }
+ if (title!=null && title.trim().length()>0) {
+ logger.println(title);
+ }
+ prefix = prefix==null ? "" : prefix;
+ NumberFormat nf = NumberFormat.getInstance();
+ nf.setMaximumFractionDigits(3);
+ nf.setMinimumFractionDigits(3);
+ nf.setGroupingUsed(true);
+ int M = 19;
+ logger.println(prefix+format("Search Seconds: ",M)+
+ fracFormat(nf.format((double)searchTime/1000)));
+ logger.println(prefix+format("DocName Seconds: ",M)+
+ fracFormat(nf.format((double)docNamesExtractTime/1000)));
+ logger.println(prefix+format("Num Points: ",M)+
+ fracFormat(nf.format(numPoints)));
+ logger.println(prefix+format("Num Good Points: ",M)+
+ fracFormat(nf.format(numGoodPoints)));
+ logger.println(prefix+format("Max Good Points: ",M)+
+ fracFormat(nf.format(maxGoodPoints)));
+ logger.println(prefix+format("Average Precision: ",M)+
+ fracFormat(nf.format(getAvp())));
+ logger.println(prefix+format("MRR: ",M)+
+ fracFormat(nf.format(getMRR())));
+ logger.println(prefix+format("Recall: ",M)+
+ fracFormat(nf.format(getRecall())));
+ for (int i=1; i<(int)numPoints && i<pAt.length; i++) {
+ logger.println(prefix+format("Precision At "+i+": ",M)+
+ fracFormat(nf.format(getPrecisionAt(i))));
+ }
+ for (int i=0; i<paddLines; i++) {
+ logger.println();
+ }
+ }
+
+ private static String padd = " ";
+ private String format(String s, int minLen) {
+ s = (s==null ? "" : s);
+ int n = Math.max(minLen,s.length());
+ return (s+padd).substring(0,n);
+ }
+ private String fracFormat(String frac) {
+ int k = frac.indexOf('.');
+ String s1 = padd+frac.substring(0,k);
+ int n = Math.max(k,6);
+ s1 = s1.substring(s1.length()-n);
+ return s1 + frac.substring(k);
+ }
+
+ /**
+ * Create a QualityStats object that is the average of the input QualityStats objects.
+ * @param stats array of input stats to be averaged.
+ * @return an average over the input stats.
+ */
+ public static QualityStats average(QualityStats[] stats) {
+ QualityStats avg = new QualityStats(0,0);
+ if (stats.length==0) {
+ // weired, no stats to average!
+ return avg;
+ }
+ int m = 0; // queries with positive judgements
+ // aggregate
+ for (int i=0; i<stats.length; i++) {
+ avg.searchTime += stats[i].searchTime;
+ avg.docNamesExtractTime += stats[i].docNamesExtractTime;
+ if (stats[i].maxGoodPoints>0) {
+ m++;
+ avg.numGoodPoints += stats[i].numGoodPoints;
+ avg.numPoints += stats[i].numPoints;
+ avg.pReleventSum += stats[i].getAvp();
+ avg.recall += stats[i].recall;
+ avg.mrr += stats[i].getMRR();
+ avg.maxGoodPoints += stats[i].maxGoodPoints;
+ for (int j=1; j<avg.pAt.length; j++) {
+ avg.pAt[j] += stats[i].getPrecisionAt(j);
+ }
+ }
+ }
+ assert m>0 : "Fishy: no \"good\" queries!";
+ // take average: times go by all queries, other measures go by "good" queries only.
+ avg.searchTime /= stats.length;
+ avg.docNamesExtractTime /= stats.length;
+ avg.numGoodPoints /= m;
+ avg.numPoints /= m;
+ avg.recall /= m;
+ avg.mrr /= m;
+ avg.maxGoodPoints /= m;
+ for (int j=1; j<avg.pAt.length; j++) {
+ avg.pAt[j] /= m;
+ }
+ avg.pReleventSum /= m; // this is actually avgp now
+ avg.pReleventSum *= avg.maxGoodPoints; // so that getAvgP() would be correct
+
+ return avg;
+ }
+
+ /**
+ * Returns the time it took to extract doc names for judging the measured query, in milliseconds.
+ */
+ public long getDocNamesExtractTime() {
+ return docNamesExtractTime;
+ }
+
+ /**
+ * Returns the maximal number of good points.
+ * This is the number of relevant docs known by the judge for the measured query.
+ */
+ public double getMaxGoodPoints() {
+ return maxGoodPoints;
+ }
+
+ /**
+ * Returns the number of good points (only relevant points).
+ */
+ public double getNumGoodPoints() {
+ return numGoodPoints;
+ }
+
+ /**
+ * Returns the number of points (both relevant and irrelevant points).
+ */
+ public double getNumPoints() {
+ return numPoints;
+ }
+
+ /**
+ * Returns the recallPoints.
+ */
+ public RecallPoint [] getRecallPoints() {
+ return recallPoints.toArray(new RecallPoint[0]);
+ }
+
+ /**
+ * Returns the Mean reciprocal rank over the queries or RR for a single query.
+ * <p>
+ * Reciprocal rank is defined as <code>1/r</code> where <code>r</code> is the
+ * rank of the first correct result, or <code>0</code> if there are no correct
+ * results within the top 5 results.
+ * <p>
+ * This follows the definition in
+ * <a href="http://www.cnlp.org/publications/02cnlptrec10.pdf">
+ * Question Answering - CNLP at the TREC-10 Question Answering Track</a>.
+ */
+ public double getMRR() {
+ return mrr;
+ }
+
+
+ /**
+ * Returns the search time in milliseconds for the measured query.
+ */
+ public long getSearchTime() {
+ return searchTime;
+ }
+
+}
fnp / pylucene.git / blobdiff