+++ /dev/null
-package org.apache.lucene.index;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Random;
-
-import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.Field.Index;
-import org.apache.lucene.document.Field.Store;
-import org.apache.lucene.index.IndexWriterConfig.OpenMode;
-import org.apache.lucene.index.SegmentNorms;
-import org.apache.lucene.search.DefaultSimilarity;
-import org.apache.lucene.search.Similarity;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.LuceneTestCase;
-
-/**
- * Tests cloning IndexReader norms
- */
-public class TestIndexReaderCloneNorms extends LuceneTestCase {
-
- private class SimilarityOne extends DefaultSimilarity {
- @Override
- public float computeNorm(String fieldName, FieldInvertState state) {
- // diable length norm
- return state.getBoost();
- }
- }
-
- private static final int NUM_FIELDS = 10;
-
- private Similarity similarityOne;
-
- private Analyzer anlzr;
-
- private int numDocNorms;
-
- private ArrayList<Float> norms;
-
- private ArrayList<Float> modifiedNorms;
-
- private float lastNorm = 0;
-
- private float normDelta = (float) 0.001;
-
- @Override
- public void setUp() throws Exception {
- super.setUp();
- similarityOne = new SimilarityOne();
- anlzr = new MockAnalyzer(random);
- }
-
- /**
- * Test that norms values are preserved as the index is maintained. Including
- * separate norms. Including merging indexes with seprate norms. Including
- * optimize.
- */
- public void testNorms() throws IOException {
- // test with a single index: index1
- Directory dir1 = newDirectory();
- IndexWriter.unlock(dir1);
-
- norms = new ArrayList<Float>();
- modifiedNorms = new ArrayList<Float>();
-
- createIndex(random, dir1);
- doTestNorms(random, dir1);
-
- // test with a single index: index2
- ArrayList<Float> norms1 = norms;
- ArrayList<Float> modifiedNorms1 = modifiedNorms;
- int numDocNorms1 = numDocNorms;
-
- norms = new ArrayList<Float>();
- modifiedNorms = new ArrayList<Float>();
- numDocNorms = 0;
-
- Directory dir2 = newDirectory();
-
- createIndex(random, dir2);
- doTestNorms(random, dir2);
-
- // add index1 and index2 to a third index: index3
- Directory dir3 = newDirectory();
-
- createIndex(random, dir3);
- IndexWriter iw = new IndexWriter(dir3, newIndexWriterConfig(
- TEST_VERSION_CURRENT, anlzr).setOpenMode(OpenMode.APPEND)
- .setMaxBufferedDocs(5).setMergePolicy(newLogMergePolicy(3)));
- iw.addIndexes(new Directory[] { dir1, dir2 });
- iw.optimize();
- iw.close();
-
- norms1.addAll(norms);
- norms = norms1;
- modifiedNorms1.addAll(modifiedNorms);
- modifiedNorms = modifiedNorms1;
- numDocNorms += numDocNorms1;
-
- // test with index3
- verifyIndex(dir3);
- doTestNorms(random, dir3);
-
- // now with optimize
- iw = new IndexWriter(dir3, newIndexWriterConfig( TEST_VERSION_CURRENT,
- anlzr).setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(5).setMergePolicy(newLogMergePolicy(3)));
- iw.optimize();
- iw.close();
- verifyIndex(dir3);
-
- dir1.close();
- dir2.close();
- dir3.close();
- }
-
- // try cloning and reopening the norms
- private void doTestNorms(Random random, Directory dir) throws IOException {
- addDocs(random, dir, 12, true);
- IndexReader ir = IndexReader.open(dir, false);
- verifyIndex(ir);
- modifyNormsForF1(ir);
- IndexReader irc = (IndexReader) ir.clone();// IndexReader.open(dir, false);//ir.clone();
- verifyIndex(irc);
-
- modifyNormsForF1(irc);
-
- IndexReader irc3 = (IndexReader) irc.clone();
- verifyIndex(irc3);
- modifyNormsForF1(irc3);
- verifyIndex(irc3);
- irc3.flush();
- irc3.close();
-
- irc.close();
- ir.close();
- }
-
- public void testNormsClose() throws IOException {
- Directory dir1 = newDirectory();
- TestIndexReaderReopen.createIndex(random, dir1, false);
- SegmentReader reader1 = SegmentReader.getOnlySegmentReader(dir1);
- reader1.norms("field1");
- SegmentNorms r1norm = reader1.norms.get("field1");
- AtomicInteger r1BytesRef = r1norm.bytesRef();
- SegmentReader reader2 = (SegmentReader)reader1.clone();
- assertEquals(2, r1norm.bytesRef().get());
- reader1.close();
- assertEquals(1, r1BytesRef.get());
- reader2.norms("field1");
- reader2.close();
- dir1.close();
- }
-
- public void testNormsRefCounting() throws IOException {
- Directory dir1 = newDirectory();
- TestIndexReaderReopen.createIndex(random, dir1, false);
- IndexReader reader1 = IndexReader.open(dir1, false);
-
- IndexReader reader2C = (IndexReader) reader1.clone();
- SegmentReader segmentReader2C = SegmentReader.getOnlySegmentReader(reader2C);
- segmentReader2C.norms("field1"); // load the norms for the field
- SegmentNorms reader2CNorm = segmentReader2C.norms.get("field1");
- assertTrue("reader2CNorm.bytesRef()=" + reader2CNorm.bytesRef(), reader2CNorm.bytesRef().get() == 2);
-
-
-
- IndexReader reader3C = (IndexReader) reader2C.clone();
- SegmentReader segmentReader3C = SegmentReader.getOnlySegmentReader(reader3C);
- SegmentNorms reader3CCNorm = segmentReader3C.norms.get("field1");
- assertEquals(3, reader3CCNorm.bytesRef().get());
-
- // edit a norm and the refcount should be 1
- IndexReader reader4C = (IndexReader) reader3C.clone();
- SegmentReader segmentReader4C = SegmentReader.getOnlySegmentReader(reader4C);
- assertEquals(4, reader3CCNorm.bytesRef().get());
- reader4C.setNorm(5, "field1", 0.33f);
-
- // generate a cannot update exception in reader1
- try {
- reader3C.setNorm(1, "field1", 0.99f);
- fail("did not hit expected exception");
- } catch (Exception ex) {
- // expected
- }
-
- // norm values should be different
- assertTrue(Similarity.getDefault().decodeNormValue(segmentReader3C.norms("field1")[5])
- != Similarity.getDefault().decodeNormValue(segmentReader4C.norms("field1")[5]));
- SegmentNorms reader4CCNorm = segmentReader4C.norms.get("field1");
- assertEquals(3, reader3CCNorm.bytesRef().get());
- assertEquals(1, reader4CCNorm.bytesRef().get());
-
- IndexReader reader5C = (IndexReader) reader4C.clone();
- SegmentReader segmentReader5C = SegmentReader.getOnlySegmentReader(reader5C);
- SegmentNorms reader5CCNorm = segmentReader5C.norms.get("field1");
- reader5C.setNorm(5, "field1", 0.7f);
- assertEquals(1, reader5CCNorm.bytesRef().get());
-
- reader5C.close();
- reader4C.close();
- reader3C.close();
- reader2C.close();
- reader1.close();
- dir1.close();
- }
-
- private void createIndex(Random random, Directory dir) throws IOException {
- IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(
- TEST_VERSION_CURRENT, anlzr).setOpenMode(OpenMode.CREATE)
- .setMaxBufferedDocs(5).setSimilarity(similarityOne).setMergePolicy(newLogMergePolicy()));
- setUseCompoundFile(iw.getConfig().getMergePolicy(), true);
- setMergeFactor(iw.getConfig().getMergePolicy(), 3);
- iw.close();
- }
-
- private void modifyNormsForF1(IndexReader ir) throws IOException {
- int n = ir.maxDoc();
- // System.out.println("modifyNormsForF1 maxDoc: "+n);
- for (int i = 0; i < n; i += 3) { // modify for every third doc
- int k = (i * 3) % modifiedNorms.size();
- float origNorm = modifiedNorms.get(i).floatValue();
- float newNorm = modifiedNorms.get(k).floatValue();
- // System.out.println("Modifying: for "+i+" from "+origNorm+" to
- // "+newNorm);
- // System.out.println(" and: for "+k+" from "+newNorm+" to "+origNorm);
- modifiedNorms.set(i, Float.valueOf(newNorm));
- modifiedNorms.set(k, Float.valueOf(origNorm));
- ir.setNorm(i, "f" + 1, newNorm);
- ir.setNorm(k, "f" + 1, origNorm);
- // System.out.println("setNorm i: "+i);
- // break;
- }
- // ir.close();
- }
-
- private void verifyIndex(Directory dir) throws IOException {
- IndexReader ir = IndexReader.open(dir, false);
- verifyIndex(ir);
- ir.close();
- }
-
- private void verifyIndex(IndexReader ir) throws IOException {
- for (int i = 0; i < NUM_FIELDS; i++) {
- String field = "f" + i;
- byte b[] = ir.norms(field);
- assertEquals("number of norms mismatches", numDocNorms, b.length);
- ArrayList<Float> storedNorms = (i == 1 ? modifiedNorms : norms);
- for (int j = 0; j < b.length; j++) {
- float norm = Similarity.getDefault().decodeNormValue(b[j]);
- float norm1 = storedNorms.get(j).floatValue();
- assertEquals("stored norm value of " + field + " for doc " + j + " is "
- + norm + " - a mismatch!", norm, norm1, 0.000001);
- }
- }
- }
-
- private void addDocs(Random random, Directory dir, int ndocs, boolean compound)
- throws IOException {
- IndexWriterConfig conf = newIndexWriterConfig(
- TEST_VERSION_CURRENT, anlzr).setOpenMode(OpenMode.APPEND)
- .setMaxBufferedDocs(5).setSimilarity(similarityOne).setMergePolicy(newLogMergePolicy());
- LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
- lmp.setMergeFactor(3);
- lmp.setUseCompoundFile(compound);
- IndexWriter iw = new IndexWriter(dir, conf);
- for (int i = 0; i < ndocs; i++) {
- iw.addDocument(newDoc());
- }
- iw.close();
- }
-
- // create the next document
- private Document newDoc() {
- Document d = new Document();
- float boost = nextNorm();
- for (int i = 0; i < 10; i++) {
- Field f = newField("f" + i, "v" + i, Store.NO, Index.NOT_ANALYZED);
- f.setBoost(boost);
- d.add(f);
- }
- return d;
- }
-
- // return unique norm values that are unchanged by encoding/decoding
- private float nextNorm() {
- float norm = lastNorm + normDelta;
- do {
- float norm1 = Similarity.getDefault().decodeNormValue(
- Similarity.getDefault().encodeNormValue(norm));
- if (norm1 > lastNorm) {
- // System.out.println(norm1+" > "+lastNorm);
- norm = norm1;
- break;
- }
- norm += normDelta;
- } while (true);
- norms.add(numDocNorms, Float.valueOf(norm));
- modifiedNorms.add(numDocNorms, Float.valueOf(norm));
- // System.out.println("creating norm("+numDocNorms+"): "+norm);
- numDocNorms++;
- lastNorm = (norm > 10 ? 0 : norm); // there's a limit to how many distinct
- // values can be stored in a ingle byte
- return norm;
- }
-}
fnp / pylucene.git / blobdiff