--- /dev/null
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field.Index;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.IndexWriterConfig.OpenMode;
+import org.apache.lucene.search.FieldCache;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.store.AlreadyClosedException;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BitVector;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
+
+public class TestIndexReaderReopen extends LuceneTestCase {
+
+ public void testReopen() throws Exception {
+ final Directory dir1 = newDirectory();
+
+ createIndex(random, dir1, false);
+ performDefaultTests(new TestReopen() {
+
+ @Override
+ protected void modifyIndex(int i) throws IOException {
+ TestIndexReaderReopen.modifyIndex(i, dir1);
+ }
+
+ @Override
+ protected IndexReader openReader() throws IOException {
+ return IndexReader.open(dir1, false);
+ }
+
+ });
+ dir1.close();
+
+ final Directory dir2 = newDirectory();
+
+ createIndex(random, dir2, true);
+ performDefaultTests(new TestReopen() {
+
+ @Override
+ protected void modifyIndex(int i) throws IOException {
+ TestIndexReaderReopen.modifyIndex(i, dir2);
+ }
+
+ @Override
+ protected IndexReader openReader() throws IOException {
+ return IndexReader.open(dir2, false);
+ }
+
+ });
+ dir2.close();
+ }
+
+ public void testParallelReaderReopen() throws Exception {
+ final Directory dir1 = newDirectory();
+ createIndex(random, dir1, true);
+ final Directory dir2 = newDirectory();
+ createIndex(random, dir2, true);
+
+ performDefaultTests(new TestReopen() {
+
+ @Override
+ protected void modifyIndex(int i) throws IOException {
+ TestIndexReaderReopen.modifyIndex(i, dir1);
+ TestIndexReaderReopen.modifyIndex(i, dir2);
+ }
+
+ @Override
+ protected IndexReader openReader() throws IOException {
+ ParallelReader pr = new ParallelReader();
+ pr.add(IndexReader.open(dir1, false));
+ pr.add(IndexReader.open(dir2, false));
+ return pr;
+ }
+
+ });
+ dir1.close();
+ dir2.close();
+
+ final Directory dir3 = newDirectory();
+ createIndex(random, dir3, true);
+ final Directory dir4 = newDirectory();
+ createIndex(random, dir4, true);
+
+ performTestsWithExceptionInReopen(new TestReopen() {
+
+ @Override
+ protected void modifyIndex(int i) throws IOException {
+ TestIndexReaderReopen.modifyIndex(i, dir3);
+ TestIndexReaderReopen.modifyIndex(i, dir4);
+ }
+
+ @Override
+ protected IndexReader openReader() throws IOException {
+ ParallelReader pr = new ParallelReader();
+ pr.add(IndexReader.open(dir3, false));
+ pr.add(IndexReader.open(dir4, false));
+ // Does not implement reopen, so
+ // hits exception:
+ pr.add(new FilterIndexReader(IndexReader.open(dir3, false)));
+ return pr;
+ }
+
+ });
+ dir3.close();
+ dir4.close();
+ }
+
+ // LUCENE-1228: IndexWriter.commit() does not update the index version
+ // populate an index in iterations.
+ // at the end of every iteration, commit the index and reopen/recreate the reader.
+ // in each iteration verify the work of previous iteration.
+ // try this once with reopen once recreate, on both RAMDir and FSDir.
+ public void testCommitReopen () throws IOException {
+ Directory dir = newDirectory();
+ doTestReopenWithCommit(random, dir, true);
+ dir.close();
+ }
+ public void testCommitRecreate () throws IOException {
+ Directory dir = newDirectory();
+ doTestReopenWithCommit(random, dir, false);
+ dir.close();
+ }
+
+ private void doTestReopenWithCommit (Random random, Directory dir, boolean withReopen) throws IOException {
+ IndexWriter iwriter = new IndexWriter(dir, newIndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(
+ OpenMode.CREATE).setMergeScheduler(new SerialMergeScheduler()).setMergePolicy(newLogMergePolicy()));
+ iwriter.commit();
+ IndexReader reader = IndexReader.open(dir, false);
+ try {
+ int M = 3;
+ for (int i=0; i<4; i++) {
+ for (int j=0; j<M; j++) {
+ Document doc = new Document();
+ doc.add(newField("id", i+"_"+j, Store.YES, Index.NOT_ANALYZED));
+ doc.add(newField("id2", i+"_"+j, Store.YES, Index.NOT_ANALYZED_NO_NORMS));
+ doc.add(newField("id3", i+"_"+j, Store.YES, Index.NO));
+ iwriter.addDocument(doc);
+ if (i>0) {
+ int k = i-1;
+ int n = j + k*M;
+ Document prevItereationDoc = reader.document(n);
+ assertNotNull(prevItereationDoc);
+ String id = prevItereationDoc.get("id");
+ assertEquals(k+"_"+j, id);
+ }
+ }
+ iwriter.commit();
+ if (withReopen) {
+ // reopen
+ IndexReader r2 = reader.reopen();
+ if (reader != r2) {
+ reader.close();
+ reader = r2;
+ }
+ } else {
+ // recreate
+ reader.close();
+ reader = IndexReader.open(dir, false);
+ }
+ }
+ } finally {
+ iwriter.close();
+ reader.close();
+ }
+ }
+
+ public void testMultiReaderReopen() throws Exception {
+ final Directory dir1 = newDirectory();
+ createIndex(random, dir1, true);
+
+ final Directory dir2 = newDirectory();
+ createIndex(random, dir2, true);
+
+ performDefaultTests(new TestReopen() {
+
+ @Override
+ protected void modifyIndex(int i) throws IOException {
+ TestIndexReaderReopen.modifyIndex(i, dir1);
+ TestIndexReaderReopen.modifyIndex(i, dir2);
+ }
+
+ @Override
+ protected IndexReader openReader() throws IOException {
+ return new MultiReader(new IndexReader[]
+ {IndexReader.open(dir1, false),
+ IndexReader.open(dir2, false)});
+ }
+
+ });
+
+ dir1.close();
+ dir2.close();
+
+ final Directory dir3 = newDirectory();
+ createIndex(random, dir3, true);
+
+ final Directory dir4 = newDirectory();
+ createIndex(random, dir4, true);
+
+ performTestsWithExceptionInReopen(new TestReopen() {
+
+ @Override
+ protected void modifyIndex(int i) throws IOException {
+ TestIndexReaderReopen.modifyIndex(i, dir3);
+ TestIndexReaderReopen.modifyIndex(i, dir4);
+ }
+
+ @Override
+ protected IndexReader openReader() throws IOException {
+ return new MultiReader(new IndexReader[]
+ {IndexReader.open(dir3, false),
+ IndexReader.open(dir4, false),
+ // Does not implement reopen, so
+ // hits exception:
+ new FilterIndexReader(IndexReader.open(dir3, false))});
+ }
+
+ });
+ dir3.close();
+ dir4.close();
+ }
+
+ public void testMixedReaders() throws Exception {
+ final Directory dir1 = newDirectory();
+ createIndex(random, dir1, true);
+ final Directory dir2 = newDirectory();
+ createIndex(random, dir2, true);
+ final Directory dir3 = newDirectory();
+ createIndex(random, dir3, false);
+ final Directory dir4 = newDirectory();
+ createIndex(random, dir4, true);
+ final Directory dir5 = newDirectory();
+ createIndex(random, dir5, false);
+
+ performDefaultTests(new TestReopen() {
+
+ @Override
+ protected void modifyIndex(int i) throws IOException {
+ // only change norms in this index to maintain the same number of docs for each of ParallelReader's subreaders
+ if (i == 1) TestIndexReaderReopen.modifyIndex(i, dir1);
+
+ TestIndexReaderReopen.modifyIndex(i, dir4);
+ TestIndexReaderReopen.modifyIndex(i, dir5);
+ }
+
+ @Override
+ protected IndexReader openReader() throws IOException {
+ ParallelReader pr = new ParallelReader();
+ pr.add(IndexReader.open(dir1, false));
+ pr.add(IndexReader.open(dir2, false));
+ MultiReader mr = new MultiReader(new IndexReader[] {
+ IndexReader.open(dir3, false), IndexReader.open(dir4, false)});
+ return new MultiReader(new IndexReader[] {
+ pr, mr, IndexReader.open(dir5, false)});
+ }
+ });
+ dir1.close();
+ dir2.close();
+ dir3.close();
+ dir4.close();
+ dir5.close();
+ }
+
+ private void performDefaultTests(TestReopen test) throws Exception {
+
+ IndexReader index1 = test.openReader();
+ IndexReader index2 = test.openReader();
+
+ TestIndexReader.assertIndexEquals(index1, index2);
+
+ // verify that reopen() does not return a new reader instance
+ // in case the index has no changes
+ ReaderCouple couple = refreshReader(index2, false);
+ assertTrue(couple.refreshedReader == index2);
+
+ couple = refreshReader(index2, test, 0, true);
+ index1.close();
+ index1 = couple.newReader;
+
+ IndexReader index2_refreshed = couple.refreshedReader;
+ index2.close();
+
+ // test if refreshed reader and newly opened reader return equal results
+ TestIndexReader.assertIndexEquals(index1, index2_refreshed);
+
+ index2_refreshed.close();
+ assertReaderClosed(index2, true, true);
+ assertReaderClosed(index2_refreshed, true, true);
+
+ index2 = test.openReader();
+
+ for (int i = 1; i < 4; i++) {
+
+ index1.close();
+ couple = refreshReader(index2, test, i, true);
+ // refresh IndexReader
+ index2.close();
+
+ index2 = couple.refreshedReader;
+ index1 = couple.newReader;
+ TestIndexReader.assertIndexEquals(index1, index2);
+ }
+
+ index1.close();
+ index2.close();
+ assertReaderClosed(index1, true, true);
+ assertReaderClosed(index2, true, true);
+ }
+
+ public void testReferenceCounting() throws IOException {
+ for (int mode = 0; mode < 4; mode++) {
+ Directory dir1 = newDirectory();
+ createIndex(random, dir1, true);
+
+ IndexReader reader0 = IndexReader.open(dir1, false);
+ assertRefCountEquals(1, reader0);
+
+ assertTrue(reader0 instanceof DirectoryReader);
+ IndexReader[] subReaders0 = reader0.getSequentialSubReaders();
+ for (int i = 0; i < subReaders0.length; i++) {
+ assertRefCountEquals(1, subReaders0[i]);
+ }
+
+ // delete first document, so that only one of the subReaders have to be re-opened
+ IndexReader modifier = IndexReader.open(dir1, false);
+ modifier.deleteDocument(0);
+ modifier.close();
+
+ IndexReader reader1 = refreshReader(reader0, true).refreshedReader;
+ assertTrue(reader1 instanceof DirectoryReader);
+ IndexReader[] subReaders1 = reader1.getSequentialSubReaders();
+ assertEquals(subReaders0.length, subReaders1.length);
+
+ for (int i = 0; i < subReaders0.length; i++) {
+ if (subReaders0[i] != subReaders1[i]) {
+ assertRefCountEquals(1, subReaders0[i]);
+ assertRefCountEquals(1, subReaders1[i]);
+ } else {
+ assertRefCountEquals(2, subReaders0[i]);
+ }
+ }
+
+ // delete first document, so that only one of the subReaders have to be re-opened
+ modifier = IndexReader.open(dir1, false);
+ modifier.deleteDocument(1);
+ modifier.close();
+
+ IndexReader reader2 = refreshReader(reader1, true).refreshedReader;
+ assertTrue(reader2 instanceof DirectoryReader);
+ IndexReader[] subReaders2 = reader2.getSequentialSubReaders();
+ assertEquals(subReaders1.length, subReaders2.length);
+
+ for (int i = 0; i < subReaders2.length; i++) {
+ if (subReaders2[i] == subReaders1[i]) {
+ if (subReaders1[i] == subReaders0[i]) {
+ assertRefCountEquals(3, subReaders2[i]);
+ } else {
+ assertRefCountEquals(2, subReaders2[i]);
+ }
+ } else {
+ assertRefCountEquals(1, subReaders2[i]);
+ if (subReaders0[i] == subReaders1[i]) {
+ assertRefCountEquals(2, subReaders2[i]);
+ assertRefCountEquals(2, subReaders0[i]);
+ } else {
+ assertRefCountEquals(1, subReaders0[i]);
+ assertRefCountEquals(1, subReaders1[i]);
+ }
+ }
+ }
+
+ IndexReader reader3 = refreshReader(reader0, true).refreshedReader;
+ assertTrue(reader3 instanceof DirectoryReader);
+ IndexReader[] subReaders3 = reader3.getSequentialSubReaders();
+ assertEquals(subReaders3.length, subReaders0.length);
+
+ // try some permutations
+ switch (mode) {
+ case 0:
+ reader0.close();
+ reader1.close();
+ reader2.close();
+ reader3.close();
+ break;
+ case 1:
+ reader3.close();
+ reader2.close();
+ reader1.close();
+ reader0.close();
+ break;
+ case 2:
+ reader2.close();
+ reader3.close();
+ reader0.close();
+ reader1.close();
+ break;
+ case 3:
+ reader1.close();
+ reader3.close();
+ reader2.close();
+ reader0.close();
+ break;
+ }
+
+ assertReaderClosed(reader0, true, true);
+ assertReaderClosed(reader1, true, true);
+ assertReaderClosed(reader2, true, true);
+ assertReaderClosed(reader3, true, true);
+
+ dir1.close();
+ }
+ }
+
+
+ public void testReferenceCountingMultiReader() throws IOException {
+ for (int mode = 0; mode <=1; mode++) {
+ Directory dir1 = newDirectory();
+ createIndex(random, dir1, false);
+ Directory dir2 = newDirectory();
+ createIndex(random, dir2, true);
+
+ IndexReader reader1 = IndexReader.open(dir1, false);
+ assertRefCountEquals(1, reader1);
+
+ IndexReader initReader2 = IndexReader.open(dir2, false);
+ IndexReader multiReader1 = new MultiReader(new IndexReader[] {reader1, initReader2}, (mode == 0));
+ modifyIndex(0, dir2);
+ assertRefCountEquals(1 + mode, reader1);
+
+ IndexReader multiReader2 = multiReader1.reopen();
+ // index1 hasn't changed, so multiReader2 should share reader1 now with multiReader1
+ assertRefCountEquals(2 + mode, reader1);
+
+ modifyIndex(0, dir1);
+ IndexReader reader2 = reader1.reopen();
+ assertRefCountEquals(2 + mode, reader1);
+
+ if (mode == 1) {
+ initReader2.close();
+ }
+
+ modifyIndex(1, dir1);
+ IndexReader reader3 = reader2.reopen();
+ assertRefCountEquals(2 + mode, reader1);
+ assertRefCountEquals(1, reader2);
+
+ multiReader1.close();
+ assertRefCountEquals(1 + mode, reader1);
+
+ multiReader1.close();
+ assertRefCountEquals(1 + mode, reader1);
+
+ if (mode == 1) {
+ initReader2.close();
+ }
+
+ reader1.close();
+ assertRefCountEquals(1, reader1);
+
+ multiReader2.close();
+ assertRefCountEquals(0, reader1);
+
+ multiReader2.close();
+ assertRefCountEquals(0, reader1);
+
+ reader3.close();
+ assertRefCountEquals(0, reader1);
+ assertReaderClosed(reader1, true, false);
+
+ reader2.close();
+ assertRefCountEquals(0, reader1);
+ assertReaderClosed(reader1, true, false);
+
+ reader2.close();
+ assertRefCountEquals(0, reader1);
+
+ reader3.close();
+ assertRefCountEquals(0, reader1);
+ assertReaderClosed(reader1, true, true);
+ dir1.close();
+ dir2.close();
+ }
+
+ }
+
+ public void testReferenceCountingParallelReader() throws IOException {
+ for (int mode = 0; mode <=1; mode++) {
+ Directory dir1 = newDirectory();
+ createIndex(random, dir1, false);
+ Directory dir2 = newDirectory();
+ createIndex(random, dir2, true);
+
+ IndexReader reader1 = IndexReader.open(dir1, false);
+ assertRefCountEquals(1, reader1);
+
+ ParallelReader parallelReader1 = new ParallelReader(mode == 0);
+ parallelReader1.add(reader1);
+ IndexReader initReader2 = IndexReader.open(dir2, false);
+ parallelReader1.add(initReader2);
+ modifyIndex(1, dir2);
+ assertRefCountEquals(1 + mode, reader1);
+
+ IndexReader parallelReader2 = parallelReader1.reopen();
+ // index1 hasn't changed, so parallelReader2 should share reader1 now with multiReader1
+ assertRefCountEquals(2 + mode, reader1);
+
+ modifyIndex(0, dir1);
+ modifyIndex(0, dir2);
+ IndexReader reader2 = reader1.reopen();
+ assertRefCountEquals(2 + mode, reader1);
+
+ if (mode == 1) {
+ initReader2.close();
+ }
+
+ modifyIndex(4, dir1);
+ IndexReader reader3 = reader2.reopen();
+ assertRefCountEquals(2 + mode, reader1);
+ assertRefCountEquals(1, reader2);
+
+ parallelReader1.close();
+ assertRefCountEquals(1 + mode, reader1);
+
+ parallelReader1.close();
+ assertRefCountEquals(1 + mode, reader1);
+
+ if (mode == 1) {
+ initReader2.close();
+ }
+
+ reader1.close();
+ assertRefCountEquals(1, reader1);
+
+ parallelReader2.close();
+ assertRefCountEquals(0, reader1);
+
+ parallelReader2.close();
+ assertRefCountEquals(0, reader1);
+
+ reader3.close();
+ assertRefCountEquals(0, reader1);
+ assertReaderClosed(reader1, true, false);
+
+ reader2.close();
+ assertRefCountEquals(0, reader1);
+ assertReaderClosed(reader1, true, false);
+
+ reader2.close();
+ assertRefCountEquals(0, reader1);
+
+ reader3.close();
+ assertRefCountEquals(0, reader1);
+ assertReaderClosed(reader1, true, true);
+
+ dir1.close();
+ dir2.close();
+ }
+
+ }
+
+ public void testNormsRefCounting() throws IOException {
+ Directory dir1 = newDirectory();
+ createIndex(random, dir1, false);
+
+ IndexReader reader1 = IndexReader.open(dir1, false);
+ SegmentReader segmentReader1 = SegmentReader.getOnlySegmentReader(reader1);
+ IndexReader modifier = IndexReader.open(dir1, false);
+ modifier.deleteDocument(0);
+ modifier.close();
+
+ IndexReader reader2 = reader1.reopen();
+ modifier = IndexReader.open(dir1, false);
+ modifier.setNorm(1, "field1", 50);
+ modifier.setNorm(1, "field2", 50);
+ modifier.close();
+
+ IndexReader reader3 = reader2.reopen();
+ SegmentReader segmentReader3 = SegmentReader.getOnlySegmentReader(reader3);
+ modifier = IndexReader.open(dir1, false);
+ modifier.deleteDocument(2);
+ modifier.close();
+
+ IndexReader reader4 = reader3.reopen();
+ modifier = IndexReader.open(dir1, false);
+ modifier.deleteDocument(3);
+ modifier.close();
+
+ IndexReader reader5 = reader3.reopen();
+
+ // Now reader2-reader5 references reader1. reader1 and reader2
+ // share the same norms. reader3, reader4, reader5 also share norms.
+ assertRefCountEquals(1, reader1);
+ assertFalse(segmentReader1.normsClosed());
+
+ reader1.close();
+
+ assertRefCountEquals(0, reader1);
+ assertFalse(segmentReader1.normsClosed());
+
+ reader2.close();
+ assertRefCountEquals(0, reader1);
+
+ // now the norms for field1 and field2 should be closed
+ assertTrue(segmentReader1.normsClosed("field1"));
+ assertTrue(segmentReader1.normsClosed("field2"));
+
+ // but the norms for field3 and field4 should still be open
+ assertFalse(segmentReader1.normsClosed("field3"));
+ assertFalse(segmentReader1.normsClosed("field4"));
+
+ reader3.close();
+ assertRefCountEquals(0, reader1);
+ assertFalse(segmentReader3.normsClosed());
+ reader5.close();
+ assertRefCountEquals(0, reader1);
+ assertFalse(segmentReader3.normsClosed());
+ reader4.close();
+ assertRefCountEquals(0, reader1);
+
+ // and now all norms that reader1 used should be closed
+ assertTrue(segmentReader1.normsClosed());
+
+ // now that reader3, reader4 and reader5 are closed,
+ // the norms that those three readers shared should be
+ // closed as well
+ assertTrue(segmentReader3.normsClosed());
+
+ dir1.close();
+ }
+
+ private void performTestsWithExceptionInReopen(TestReopen test) throws Exception {
+ IndexReader index1 = test.openReader();
+ IndexReader index2 = test.openReader();
+
+ TestIndexReader.assertIndexEquals(index1, index2);
+
+ try {
+ refreshReader(index1, test, 0, true);
+ fail("Expected exception not thrown.");
+ } catch (Exception e) {
+ // expected exception
+ }
+
+ // index2 should still be usable and unaffected by the failed reopen() call
+ TestIndexReader.assertIndexEquals(index1, index2);
+
+ index1.close();
+ index2.close();
+ }
+
+ public void testThreadSafety() throws Exception {
+ final Directory dir = newDirectory();
+ // NOTE: this also controls the number of threads!
+ final int n = _TestUtil.nextInt(random, 20, 40);
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer(random)));
+ for (int i = 0; i < n; i++) {
+ writer.addDocument(createDocument(i, 3));
+ }
+ writer.optimize();
+ writer.close();
+
+ final TestReopen test = new TestReopen() {
+ @Override
+ protected void modifyIndex(int i) throws IOException {
+ if (i % 3 == 0) {
+ IndexReader modifier = IndexReader.open(dir, false);
+ modifier.setNorm(i, "field1", 50);
+ modifier.close();
+ } else if (i % 3 == 1) {
+ IndexReader modifier = IndexReader.open(dir, false);
+ modifier.deleteDocument(i % modifier.maxDoc());
+ modifier.close();
+ } else {
+ IndexWriter modifier = new IndexWriter(dir, new IndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer(random)));
+ modifier.addDocument(createDocument(n + i, 6));
+ modifier.close();
+ }
+ }
+
+ @Override
+ protected IndexReader openReader() throws IOException {
+ return IndexReader.open(dir, false);
+ }
+ };
+
+ final List<ReaderCouple> readers = Collections.synchronizedList(new ArrayList<ReaderCouple>());
+ IndexReader firstReader = IndexReader.open(dir, false);
+ IndexReader reader = firstReader;
+ final Random rnd = random;
+
+ ReaderThread[] threads = new ReaderThread[n];
+ final Set<IndexReader> readersToClose = Collections.synchronizedSet(new HashSet<IndexReader>());
+
+ for (int i = 0; i < n; i++) {
+ if (i % 2 == 0) {
+ IndexReader refreshed = reader.reopen();
+ if (refreshed != reader) {
+ readersToClose.add(reader);
+ }
+ reader = refreshed;
+ }
+ final IndexReader r = reader;
+
+ final int index = i;
+
+ ReaderThreadTask task;
+
+ if (i < 4 || (i >=10 && i < 14) || i > 18) {
+ task = new ReaderThreadTask() {
+
+ @Override
+ public void run() throws Exception {
+ while (!stopped) {
+ if (index % 2 == 0) {
+ // refresh reader synchronized
+ ReaderCouple c = (refreshReader(r, test, index, true));
+ readersToClose.add(c.newReader);
+ readersToClose.add(c.refreshedReader);
+ readers.add(c);
+ // prevent too many readers
+ break;
+ } else {
+ // not synchronized
+ IndexReader refreshed = r.reopen();
+
+ IndexSearcher searcher = newSearcher(refreshed);
+ ScoreDoc[] hits = searcher.search(
+ new TermQuery(new Term("field1", "a" + rnd.nextInt(refreshed.maxDoc()))),
+ null, 1000).scoreDocs;
+ if (hits.length > 0) {
+ searcher.doc(hits[0].doc);
+ }
+ searcher.close();
+ if (refreshed != r) {
+ refreshed.close();
+ }
+ }
+ synchronized(this) {
+ wait(_TestUtil.nextInt(random, 1, 100));
+ }
+ }
+ }
+
+ };
+ } else {
+ task = new ReaderThreadTask() {
+ @Override
+ public void run() throws Exception {
+ while (!stopped) {
+ int numReaders = readers.size();
+ if (numReaders > 0) {
+ ReaderCouple c = readers.get(rnd.nextInt(numReaders));
+ TestIndexReader.assertIndexEquals(c.newReader, c.refreshedReader);
+ }
+
+ synchronized(this) {
+ wait(_TestUtil.nextInt(random, 1, 100));
+ }
+ }
+ }
+ };
+ }
+
+ threads[i] = new ReaderThread(task);
+ threads[i].start();
+ }
+
+ synchronized(this) {
+ wait(1000);
+ }
+
+ for (int i = 0; i < n; i++) {
+ if (threads[i] != null) {
+ threads[i].stopThread();
+ }
+ }
+
+ for (int i = 0; i < n; i++) {
+ if (threads[i] != null) {
+ threads[i].join();
+ if (threads[i].error != null) {
+ String msg = "Error occurred in thread " + threads[i].getName() + ":\n" + threads[i].error.getMessage();
+ fail(msg);
+ }
+ }
+
+ }
+
+ for (final IndexReader readerToClose : readersToClose) {
+ readerToClose.close();
+ }
+
+ firstReader.close();
+ reader.close();
+
+ for (final IndexReader readerToClose : readersToClose) {
+ assertReaderClosed(readerToClose, true, true);
+ }
+
+ assertReaderClosed(reader, true, true);
+ assertReaderClosed(firstReader, true, true);
+
+ dir.close();
+ }
+
+ private static class ReaderCouple {
+ ReaderCouple(IndexReader r1, IndexReader r2) {
+ newReader = r1;
+ refreshedReader = r2;
+ }
+
+ IndexReader newReader;
+ IndexReader refreshedReader;
+ }
+
+ private abstract static class ReaderThreadTask {
+ protected volatile boolean stopped;
+ public void stop() {
+ this.stopped = true;
+ }
+
+ public abstract void run() throws Exception;
+ }
+
+ private static class ReaderThread extends Thread {
+ private ReaderThreadTask task;
+ private Throwable error;
+
+
+ ReaderThread(ReaderThreadTask task) {
+ this.task = task;
+ }
+
+ public void stopThread() {
+ this.task.stop();
+ }
+
+ @Override
+ public void run() {
+ try {
+ this.task.run();
+ } catch (Throwable r) {
+ r.printStackTrace(System.out);
+ this.error = r;
+ }
+ }
+ }
+
+ private Object createReaderMutex = new Object();
+
+ private ReaderCouple refreshReader(IndexReader reader, boolean hasChanges) throws IOException {
+ return refreshReader(reader, null, -1, hasChanges);
+ }
+
+ ReaderCouple refreshReader(IndexReader reader, TestReopen test, int modify, boolean hasChanges) throws IOException {
+ synchronized (createReaderMutex) {
+ IndexReader r = null;
+ if (test != null) {
+ test.modifyIndex(modify);
+ r = test.openReader();
+ }
+
+ IndexReader refreshed = null;
+ try {
+ refreshed = reader.reopen();
+ } finally {
+ if (refreshed == null && r != null) {
+ // Hit exception -- close opened reader
+ r.close();
+ }
+ }
+
+ if (hasChanges) {
+ if (refreshed == reader) {
+ fail("No new IndexReader instance created during refresh.");
+ }
+ } else {
+ if (refreshed != reader) {
+ fail("New IndexReader instance created during refresh even though index had no changes.");
+ }
+ }
+
+ return new ReaderCouple(r, refreshed);
+ }
+ }
+
+ public static void createIndex(Random random, Directory dir, boolean multiSegment) throws IOException {
+ IndexWriter.unlock(dir);
+ IndexWriter w = new IndexWriter(dir, LuceneTestCase.newIndexWriterConfig(random,
+ TEST_VERSION_CURRENT, new MockAnalyzer(random))
+ .setMergePolicy(new LogDocMergePolicy()));
+
+ for (int i = 0; i < 100; i++) {
+ w.addDocument(createDocument(i, 4));
+ if (multiSegment && (i % 10) == 0) {
+ w.commit();
+ }
+ }
+
+ if (!multiSegment) {
+ w.optimize();
+ }
+
+ w.close();
+
+ IndexReader r = IndexReader.open(dir, false);
+ if (multiSegment) {
+ assertTrue(r.getSequentialSubReaders().length > 1);
+ } else {
+ assertTrue(r.getSequentialSubReaders().length == 1);
+ }
+ r.close();
+ }
+
+ public static Document createDocument(int n, int numFields) {
+ StringBuilder sb = new StringBuilder();
+ Document doc = new Document();
+ sb.append("a");
+ sb.append(n);
+ doc.add(new Field("field1", sb.toString(), Store.YES, Index.ANALYZED));
+ doc.add(new Field("fielda", sb.toString(), Store.YES, Index.NOT_ANALYZED_NO_NORMS));
+ doc.add(new Field("fieldb", sb.toString(), Store.YES, Index.NO));
+ sb.append(" b");
+ sb.append(n);
+ for (int i = 1; i < numFields; i++) {
+ doc.add(new Field("field" + (i+1), sb.toString(), Store.YES, Index.ANALYZED));
+ }
+ return doc;
+ }
+
+ static void modifyIndex(int i, Directory dir) throws IOException {
+ switch (i) {
+ case 0: {
+ if (VERBOSE) {
+ System.out.println("TEST: modify index");
+ }
+ IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
+ w.setInfoStream(VERBOSE ? System.out : null);
+ w.deleteDocuments(new Term("field2", "a11"));
+ w.deleteDocuments(new Term("field2", "b30"));
+ w.close();
+ break;
+ }
+ case 1: {
+ IndexReader reader = IndexReader.open(dir, false);
+ reader.setNorm(4, "field1", 123);
+ reader.setNorm(44, "field2", 222);
+ reader.setNorm(44, "field4", 22);
+ reader.close();
+ break;
+ }
+ case 2: {
+ IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
+ w.optimize();
+ w.close();
+ break;
+ }
+ case 3: {
+ IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
+ w.addDocument(createDocument(101, 4));
+ w.optimize();
+ w.addDocument(createDocument(102, 4));
+ w.addDocument(createDocument(103, 4));
+ w.close();
+ break;
+ }
+ case 4: {
+ IndexReader reader = IndexReader.open(dir, false);
+ reader.setNorm(5, "field1", 123);
+ reader.setNorm(55, "field2", 222);
+ reader.close();
+ break;
+ }
+ case 5: {
+ IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
+ w.addDocument(createDocument(101, 4));
+ w.close();
+ break;
+ }
+ }
+ }
+
+ private void assertReaderClosed(IndexReader reader, boolean checkSubReaders, boolean checkNormsClosed) {
+ assertEquals(0, reader.getRefCount());
+
+ if (checkNormsClosed && reader instanceof SegmentReader) {
+ assertTrue(((SegmentReader) reader).normsClosed());
+ }
+
+ if (checkSubReaders) {
+ if (reader instanceof DirectoryReader) {
+ IndexReader[] subReaders = reader.getSequentialSubReaders();
+ for (int i = 0; i < subReaders.length; i++) {
+ assertReaderClosed(subReaders[i], checkSubReaders, checkNormsClosed);
+ }
+ }
+
+ if (reader instanceof MultiReader) {
+ IndexReader[] subReaders = reader.getSequentialSubReaders();
+ for (int i = 0; i < subReaders.length; i++) {
+ assertReaderClosed(subReaders[i], checkSubReaders, checkNormsClosed);
+ }
+ }
+
+ if (reader instanceof ParallelReader) {
+ IndexReader[] subReaders = ((ParallelReader) reader).getSubReaders();
+ for (int i = 0; i < subReaders.length; i++) {
+ assertReaderClosed(subReaders[i], checkSubReaders, checkNormsClosed);
+ }
+ }
+ }
+ }
+
+ /*
+ private void assertReaderOpen(IndexReader reader) {
+ reader.ensureOpen();
+
+ if (reader instanceof DirectoryReader) {
+ IndexReader[] subReaders = reader.getSequentialSubReaders();
+ for (int i = 0; i < subReaders.length; i++) {
+ assertReaderOpen(subReaders[i]);
+ }
+ }
+ }
+ */
+
+ private void assertRefCountEquals(int refCount, IndexReader reader) {
+ assertEquals("Reader has wrong refCount value.", refCount, reader.getRefCount());
+ }
+
+
+ private abstract static class TestReopen {
+ protected abstract IndexReader openReader() throws IOException;
+ protected abstract void modifyIndex(int i) throws IOException;
+ }
+
+ public void testCloseOrig() throws Throwable {
+ Directory dir = newDirectory();
+ createIndex(random, dir, false);
+ IndexReader r1 = IndexReader.open(dir, false);
+ IndexReader r2 = IndexReader.open(dir, false);
+ r2.deleteDocument(0);
+ r2.close();
+
+ IndexReader r3 = r1.reopen();
+ assertTrue(r1 != r3);
+ r1.close();
+ try {
+ r1.document(2);
+ fail("did not hit exception");
+ } catch (AlreadyClosedException ace) {
+ // expected
+ }
+ r3.close();
+ dir.close();
+ }
+
+ public void testDeletes() throws Throwable {
+ Directory dir = newDirectory();
+ createIndex(random, dir, false); // Create an index with a bunch of docs (1 segment)
+
+ modifyIndex(0, dir); // Get delete bitVector on 1st segment
+ modifyIndex(5, dir); // Add a doc (2 segments)
+
+ IndexReader r1 = IndexReader.open(dir, false); // MSR
+
+ modifyIndex(5, dir); // Add another doc (3 segments)
+
+ IndexReader r2 = r1.reopen(); // MSR
+ assertTrue(r1 != r2);
+
+ SegmentReader sr1 = (SegmentReader) r1.getSequentialSubReaders()[0]; // Get SRs for the first segment from original
+ SegmentReader sr2 = (SegmentReader) r2.getSequentialSubReaders()[0]; // and reopened IRs
+
+ // At this point they share the same BitVector
+ assertTrue(sr1.deletedDocs==sr2.deletedDocs);
+
+ r2.deleteDocument(0);
+
+ // r1 should not see the delete
+ assertFalse(r1.isDeleted(0));
+
+ // Now r2 should have made a private copy of deleted docs:
+ assertTrue(sr1.deletedDocs!=sr2.deletedDocs);
+
+ r1.close();
+ r2.close();
+ dir.close();
+ }
+
+ public void testDeletes2() throws Throwable {
+ Directory dir = newDirectory();
+ createIndex(random, dir, false);
+ // Get delete bitVector
+ modifyIndex(0, dir);
+ IndexReader r1 = IndexReader.open(dir, false);
+
+ // Add doc:
+ modifyIndex(5, dir);
+
+ IndexReader r2 = r1.reopen();
+ assertTrue(r1 != r2);
+
+ IndexReader[] rs2 = r2.getSequentialSubReaders();
+
+ SegmentReader sr1 = SegmentReader.getOnlySegmentReader(r1);
+ SegmentReader sr2 = (SegmentReader) rs2[0];
+
+ // At this point they share the same BitVector
+ assertTrue(sr1.deletedDocs==sr2.deletedDocs);
+ final BitVector delDocs = sr1.deletedDocs;
+ r1.close();
+
+ r2.deleteDocument(0);
+ assertTrue(delDocs==sr2.deletedDocs);
+ r2.close();
+ dir.close();
+ }
+
+ private static class KeepAllCommits implements IndexDeletionPolicy {
+ public void onInit(List<? extends IndexCommit> commits) {
+ }
+ public void onCommit(List<? extends IndexCommit> commits) {
+ }
+ }
+
+ public void testReopenOnCommit() throws Throwable {
+ Directory dir = newDirectory();
+ IndexWriter writer = new IndexWriter(
+ dir,
+ newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
+ setIndexDeletionPolicy(new KeepAllCommits()).
+ setMaxBufferedDocs(-1).
+ setMergePolicy(newLogMergePolicy(10))
+ );
+ for(int i=0;i<4;i++) {
+ Document doc = new Document();
+ doc.add(newField("id", ""+i, Field.Store.NO, Field.Index.NOT_ANALYZED));
+ writer.addDocument(doc);
+ Map<String,String> data = new HashMap<String,String>();
+ data.put("index", i+"");
+ writer.commit(data);
+ }
+ for(int i=0;i<4;i++) {
+ writer.deleteDocuments(new Term("id", ""+i));
+ Map<String,String> data = new HashMap<String,String>();
+ data.put("index", (4+i)+"");
+ writer.commit(data);
+ }
+ writer.close();
+
+ IndexReader r = IndexReader.open(dir, false);
+ assertEquals(0, r.numDocs());
+
+ Collection<IndexCommit> commits = IndexReader.listCommits(dir);
+ for (final IndexCommit commit : commits) {
+ IndexReader r2 = r.reopen(commit);
+ assertTrue(r2 != r);
+
+ // Reader should be readOnly
+ try {
+ r2.deleteDocument(0);
+ fail("no exception hit");
+ } catch (UnsupportedOperationException uoe) {
+ // expected
+ }
+
+ final Map<String,String> s = commit.getUserData();
+ final int v;
+ if (s.size() == 0) {
+ // First commit created by IW
+ v = -1;
+ } else {
+ v = Integer.parseInt(s.get("index"));
+ }
+ if (v < 4) {
+ assertEquals(1+v, r2.numDocs());
+ } else {
+ assertEquals(7-v, r2.numDocs());
+ }
+ r.close();
+ r = r2;
+ }
+ r.close();
+ dir.close();
+ }
+
+ // LUCENE-1579: Make sure all SegmentReaders are new when
+ // reopen switches readOnly
+ public void testReopenChangeReadonly() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriter writer = new IndexWriter(
+ dir,
+ newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
+ setMaxBufferedDocs(-1).
+ setMergePolicy(newLogMergePolicy(10))
+ );
+ Document doc = new Document();
+ doc.add(newField("number", "17", Field.Store.NO, Field.Index.NOT_ANALYZED));
+ writer.addDocument(doc);
+ writer.commit();
+
+ // Open reader1
+ IndexReader r = IndexReader.open(dir, false);
+ assertTrue(r instanceof DirectoryReader);
+ IndexReader r1 = SegmentReader.getOnlySegmentReader(r);
+ final int[] ints = FieldCache.DEFAULT.getInts(r1, "number");
+ assertEquals(1, ints.length);
+ assertEquals(17, ints[0]);
+
+ // Reopen to readonly w/ no chnages
+ IndexReader r3 = r.reopen(true);
+ assertTrue(r3 instanceof ReadOnlyDirectoryReader);
+ r3.close();
+
+ // Add new segment
+ writer.addDocument(doc);
+ writer.commit();
+
+ // Reopen reader1 --> reader2
+ IndexReader r2 = r.reopen(true);
+ r.close();
+ assertTrue(r2 instanceof ReadOnlyDirectoryReader);
+ IndexReader[] subs = r2.getSequentialSubReaders();
+ final int[] ints2 = FieldCache.DEFAULT.getInts(subs[0], "number");
+ r2.close();
+
+ assertTrue(subs[0] instanceof ReadOnlySegmentReader);
+ assertTrue(subs[1] instanceof ReadOnlySegmentReader);
+ assertTrue(ints == ints2);
+
+ writer.close();
+ dir.close();
+ }
+}
fnp / pylucene.git / blobdiff