+++ /dev/null
-package org.apache.lucene.search;
-
-import org.apache.lucene.util.DocIdBitSet;
-import org.apache.lucene.util.LuceneTestCase;
-
-import java.util.BitSet;
-import java.io.IOException;
-
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.index.IndexWriterConfig.OpenMode;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.analysis.WhitespaceAnalyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-public class TestScorerPerf extends LuceneTestCase {
- boolean validate = true; // set to false when doing performance testing
-
- BitSet[] sets;
- Term[] terms;
- IndexSearcher s;
- Directory d;
-
- public void createDummySearcher() throws Exception {
- // Create a dummy index with nothing in it.
- // This could possibly fail if Lucene starts checking for docid ranges...
- d = newDirectory();
- IndexWriter iw = new IndexWriter(d, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
- iw.addDocument(new Document());
- iw.close();
- s = new IndexSearcher(d, true);
- }
-
- public void createRandomTerms(int nDocs, int nTerms, double power, Directory dir) throws Exception {
- int[] freq = new int[nTerms];
- terms = new Term[nTerms];
- for (int i=0; i<nTerms; i++) {
- int f = (nTerms+1)-i; // make first terms less frequent
- freq[i] = (int)Math.ceil(Math.pow(f,power));
- terms[i] = new Term("f",Character.toString((char)('A'+i)));
- }
-
- IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE));
- for (int i=0; i<nDocs; i++) {
- Document d = new Document();
- for (int j=0; j<nTerms; j++) {
- if (random.nextInt(freq[j]) == 0) {
- d.add(newField("f", terms[j].text(), Field.Store.NO, Field.Index.NOT_ANALYZED));
- //System.out.println(d);
- }
- }
- iw.addDocument(d);
- }
- iw.optimize();
- iw.close();
- }
-
-
- public BitSet randBitSet(int sz, int numBitsToSet) {
- BitSet set = new BitSet(sz);
- for (int i=0; i<numBitsToSet; i++) {
- set.set(random.nextInt(sz));
- }
- return set;
- }
-
- public BitSet[] randBitSets(int numSets, int setSize) {
- BitSet[] sets = new BitSet[numSets];
- for (int i=0; i<sets.length; i++) {
- sets[i] = randBitSet(setSize, random.nextInt(setSize));
- }
- return sets;
- }
-
- public static class CountingHitCollector extends Collector {
- int count=0;
- int sum=0;
- protected int docBase = 0;
-
- @Override
- public void setScorer(Scorer scorer) throws IOException {}
-
- @Override
- public void collect(int doc) {
- count++;
- sum += docBase + doc; // use it to avoid any possibility of being optimized away
- }
-
- public int getCount() { return count; }
- public int getSum() { return sum; }
-
- @Override
- public void setNextReader(IndexReader reader, int base) {
- docBase = base;
- }
- @Override
- public boolean acceptsDocsOutOfOrder() {
- return true;
- }
- }
-
-
- public static class MatchingHitCollector extends CountingHitCollector {
- BitSet answer;
- int pos=-1;
- public MatchingHitCollector(BitSet answer) {
- this.answer = answer;
- }
-
- public void collect(int doc, float score) {
-
- pos = answer.nextSetBit(pos+1);
- if (pos != doc + docBase) {
- throw new RuntimeException("Expected doc " + pos + " but got " + doc + docBase);
- }
- super.collect(doc);
- }
- }
-
-
- BitSet addClause(BooleanQuery bq, BitSet result) {
- final BitSet rnd = sets[random.nextInt(sets.length)];
- Query q = new ConstantScoreQuery(new Filter() {
- @Override
- public DocIdSet getDocIdSet(IndexReader reader) {
- return new DocIdBitSet(rnd);
- }
- });
- bq.add(q, BooleanClause.Occur.MUST);
- if (validate) {
- if (result==null) result = (BitSet)rnd.clone();
- else result.and(rnd);
- }
- return result;
- }
-
-
- public int doConjunctions(int iter, int maxClauses) throws IOException {
- int ret=0;
-
- for (int i=0; i<iter; i++) {
- int nClauses = random.nextInt(maxClauses-1)+2; // min 2 clauses
- BooleanQuery bq = new BooleanQuery();
- BitSet result=null;
- for (int j=0; j<nClauses; j++) {
- result = addClause(bq,result);
- }
-
- CountingHitCollector hc = validate ? new MatchingHitCollector(result)
- : new CountingHitCollector();
- s.search(bq, hc);
- ret += hc.getSum();
-
- if (validate) assertEquals(result.cardinality(), hc.getCount());
- // System.out.println(hc.getCount());
- }
-
- return ret;
- }
-
- public int doNestedConjunctions(int iter, int maxOuterClauses, int maxClauses) throws IOException {
- int ret=0;
- long nMatches=0;
-
- for (int i=0; i<iter; i++) {
- int oClauses = random.nextInt(maxOuterClauses-1)+2;
- BooleanQuery oq = new BooleanQuery();
- BitSet result=null;
-
- for (int o=0; o<oClauses; o++) {
-
- int nClauses = random.nextInt(maxClauses-1)+2; // min 2 clauses
- BooleanQuery bq = new BooleanQuery();
- for (int j=0; j<nClauses; j++) {
- result = addClause(bq,result);
- }
-
- oq.add(bq, BooleanClause.Occur.MUST);
- } // outer
-
- CountingHitCollector hc = validate ? new MatchingHitCollector(result)
- : new CountingHitCollector();
- s.search(oq, hc);
- nMatches += hc.getCount();
- ret += hc.getSum();
- if (validate) assertEquals(result.cardinality(), hc.getCount());
- // System.out.println(hc.getCount());
- }
- if (VERBOSE) System.out.println("Average number of matches="+(nMatches/iter));
- return ret;
- }
-
-
- public int doTermConjunctions(IndexSearcher s,
- int termsInIndex,
- int maxClauses,
- int iter
- ) throws IOException {
- int ret=0;
-
- long nMatches=0;
- for (int i=0; i<iter; i++) {
- int nClauses = random.nextInt(maxClauses-1)+2; // min 2 clauses
- BooleanQuery bq = new BooleanQuery();
- BitSet termflag = new BitSet(termsInIndex);
- for (int j=0; j<nClauses; j++) {
- int tnum;
- // don't pick same clause twice
- tnum = random.nextInt(termsInIndex);
- if (termflag.get(tnum)) tnum=termflag.nextClearBit(tnum);
- if (tnum<0 || tnum>=termsInIndex) tnum=termflag.nextClearBit(0);
- termflag.set(tnum);
- Query tq = new TermQuery(terms[tnum]);
- bq.add(tq, BooleanClause.Occur.MUST);
- }
-
- CountingHitCollector hc = new CountingHitCollector();
- s.search(bq, hc);
- nMatches += hc.getCount();
- ret += hc.getSum();
- }
- if (VERBOSE) System.out.println("Average number of matches="+(nMatches/iter));
-
- return ret;
- }
-
-
- public int doNestedTermConjunctions(IndexSearcher s,
- int termsInIndex,
- int maxOuterClauses,
- int maxClauses,
- int iter
- ) throws IOException {
- int ret=0;
- long nMatches=0;
- for (int i=0; i<iter; i++) {
- int oClauses = random.nextInt(maxOuterClauses-1)+2;
- BooleanQuery oq = new BooleanQuery();
- for (int o=0; o<oClauses; o++) {
-
- int nClauses = random.nextInt(maxClauses-1)+2; // min 2 clauses
- BooleanQuery bq = new BooleanQuery();
- BitSet termflag = new BitSet(termsInIndex);
- for (int j=0; j<nClauses; j++) {
- int tnum;
- // don't pick same clause twice
- tnum = random.nextInt(termsInIndex);
- if (termflag.get(tnum)) tnum=termflag.nextClearBit(tnum);
- if (tnum<0 || tnum>=25) tnum=termflag.nextClearBit(0);
- termflag.set(tnum);
- Query tq = new TermQuery(terms[tnum]);
- bq.add(tq, BooleanClause.Occur.MUST);
- } // inner
-
- oq.add(bq, BooleanClause.Occur.MUST);
- } // outer
-
-
- CountingHitCollector hc = new CountingHitCollector();
- s.search(oq, hc);
- nMatches += hc.getCount();
- ret += hc.getSum();
- }
- if (VERBOSE) System.out.println("Average number of matches="+(nMatches/iter));
- return ret;
- }
-
-
- public int doSloppyPhrase(IndexSearcher s,
- int termsInIndex,
- int maxClauses,
- int iter
- ) throws IOException {
- int ret=0;
-
- for (int i=0; i<iter; i++) {
- int nClauses = random.nextInt(maxClauses-1)+2; // min 2 clauses
- PhraseQuery q = new PhraseQuery();
- for (int j=0; j<nClauses; j++) {
- int tnum = random.nextInt(termsInIndex);
- q.add(new Term("f",Character.toString((char)(tnum+'A'))), j);
- }
- q.setSlop(termsInIndex); // this could be random too
-
- CountingHitCollector hc = new CountingHitCollector();
- s.search(q, hc);
- ret += hc.getSum();
- }
-
- return ret;
- }
-
-
- public void testConjunctions() throws Exception {
- // test many small sets... the bugs will be found on boundary conditions
- createDummySearcher();
- validate=true;
- sets=randBitSets(atLeast(1000), atLeast(10));
- doConjunctions(atLeast(10000), atLeast(5));
- doNestedConjunctions(atLeast(10000), atLeast(3), atLeast(3));
- s.close();
- d.close();
- }
-
- /***
- int bigIter=10;
-
- public void testConjunctionPerf() throws Exception {
- r = newRandom();
- createDummySearcher();
- validate=false;
- sets=randBitSets(32,1000000);
- for (int i=0; i<bigIter; i++) {
- long start = System.currentTimeMillis();
- doConjunctions(500,6);
- long end = System.currentTimeMillis();
- if (VERBOSE) System.out.println("milliseconds="+(end-start));
- }
- s.close();
- }
-
- public void testNestedConjunctionPerf() throws Exception {
- r = newRandom();
- createDummySearcher();
- validate=false;
- sets=randBitSets(32,1000000);
- for (int i=0; i<bigIter; i++) {
- long start = System.currentTimeMillis();
- doNestedConjunctions(500,3,3);
- long end = System.currentTimeMillis();
- if (VERBOSE) System.out.println("milliseconds="+(end-start));
- }
- s.close();
- }
-
-
- public void testConjunctionTerms() throws Exception {
- r = newRandom();
- validate=false;
- RAMDirectory dir = new RAMDirectory();
- if (VERBOSE) System.out.println("Creating index");
- createRandomTerms(100000,25,.5, dir);
- s = new IndexSearcher(dir, true);
- if (VERBOSE) System.out.println("Starting performance test");
- for (int i=0; i<bigIter; i++) {
- long start = System.currentTimeMillis();
- doTermConjunctions(s,25,5,1000);
- long end = System.currentTimeMillis();
- if (VERBOSE) System.out.println("milliseconds="+(end-start));
- }
- s.close();
- }
-
- public void testNestedConjunctionTerms() throws Exception {
- r = newRandom();
- validate=false;
- RAMDirectory dir = new RAMDirectory();
- if (VERBOSE) System.out.println("Creating index");
- createRandomTerms(100000,25,.2, dir);
- s = new IndexSearcher(dir, true);
- if (VERBOSE) System.out.println("Starting performance test");
- for (int i=0; i<bigIter; i++) {
- long start = System.currentTimeMillis();
- doNestedTermConjunctions(s,25,3,3,200);
- long end = System.currentTimeMillis();
- if (VERBOSE) System.out.println("milliseconds="+(end-start));
- }
- s.close();
- }
-
-
- public void testSloppyPhrasePerf() throws Exception {
- r = newRandom();
- validate=false;
- RAMDirectory dir = new RAMDirectory();
- if (VERBOSE) System.out.println("Creating index");
- createRandomTerms(100000,25,2,dir);
- s = new IndexSearcher(dir, true);
- if (VERBOSE) System.out.println("Starting performance test");
- for (int i=0; i<bigIter; i++) {
- long start = System.currentTimeMillis();
- doSloppyPhrase(s,25,2,1000);
- long end = System.currentTimeMillis();
- if (VERBOSE) System.out.println("milliseconds="+(end-start));
- }
- s.close();
- }
- ***/
-
-
-}
fnp / pylucene.git / blobdiff