X-Git-Url: https://git.mdrn.pl/pylucene.git/blobdiff_plain/a2e61f0c04805cfcb8706176758d1283c7e3a55c..aaeed5504b982cf3545252ab528713250aa33eed:/lucene-java-3.5.0/lucene/backwards/src/test/org/apache/lucene/index/TestIndexReaderReopen.java diff --git a/lucene-java-3.5.0/lucene/backwards/src/test/org/apache/lucene/index/TestIndexReaderReopen.java b/lucene-java-3.5.0/lucene/backwards/src/test/org/apache/lucene/index/TestIndexReaderReopen.java new file mode 100644 index 0000000..1cd5142 --- /dev/null +++ b/lucene-java-3.5.0/lucene/backwards/src/test/org/apache/lucene/index/TestIndexReaderReopen.java @@ -0,0 +1,1273 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; + +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field.Index; +import org.apache.lucene.document.Field.Store; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexWriterConfig.OpenMode; +import org.apache.lucene.search.FieldCache; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.store.AlreadyClosedException; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BitVector; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util._TestUtil; + +public class TestIndexReaderReopen extends LuceneTestCase { + + public void testReopen() throws Exception { + final Directory dir1 = newDirectory(); + + createIndex(random, dir1, false); + performDefaultTests(new TestReopen() { + + @Override + protected void modifyIndex(int i) throws IOException { + TestIndexReaderReopen.modifyIndex(i, dir1); + } + + @Override + protected IndexReader openReader() throws IOException { + return IndexReader.open(dir1, false); + } + + }); + dir1.close(); + + final Directory dir2 = newDirectory(); + + createIndex(random, dir2, true); + performDefaultTests(new TestReopen() { + + @Override + protected void modifyIndex(int i) throws IOException { + TestIndexReaderReopen.modifyIndex(i, dir2); + } + + @Override + protected IndexReader openReader() throws IOException { + return IndexReader.open(dir2, false); + } + + }); + dir2.close(); + } + + public void testParallelReaderReopen() throws Exception { + final Directory dir1 = newDirectory(); + createIndex(random, dir1, true); + final Directory dir2 = newDirectory(); + createIndex(random, dir2, true); + + performDefaultTests(new TestReopen() { + + @Override + protected void modifyIndex(int i) throws IOException { + TestIndexReaderReopen.modifyIndex(i, dir1); + TestIndexReaderReopen.modifyIndex(i, dir2); + } + + @Override + protected IndexReader openReader() throws IOException { + ParallelReader pr = new ParallelReader(); + pr.add(IndexReader.open(dir1, false)); + pr.add(IndexReader.open(dir2, false)); + return pr; + } + + }); + dir1.close(); + dir2.close(); + + final Directory dir3 = newDirectory(); + createIndex(random, dir3, true); + final Directory dir4 = newDirectory(); + createIndex(random, dir4, true); + + performTestsWithExceptionInReopen(new TestReopen() { + + @Override + protected void modifyIndex(int i) throws IOException { + TestIndexReaderReopen.modifyIndex(i, dir3); + TestIndexReaderReopen.modifyIndex(i, dir4); + } + + @Override + protected IndexReader openReader() throws IOException { + ParallelReader pr = new ParallelReader(); + pr.add(IndexReader.open(dir3, false)); + pr.add(IndexReader.open(dir4, false)); + // Does not implement reopen, so + // hits exception: + pr.add(new FilterIndexReader(IndexReader.open(dir3, false))); + return pr; + } + + }); + dir3.close(); + dir4.close(); + } + + // LUCENE-1228: IndexWriter.commit() does not update the index version + // populate an index in iterations. + // at the end of every iteration, commit the index and reopen/recreate the reader. + // in each iteration verify the work of previous iteration. + // try this once with reopen once recreate, on both RAMDir and FSDir. + public void testCommitReopen () throws IOException { + Directory dir = newDirectory(); + doTestReopenWithCommit(random, dir, true); + dir.close(); + } + public void testCommitRecreate () throws IOException { + Directory dir = newDirectory(); + doTestReopenWithCommit(random, dir, false); + dir.close(); + } + + private void doTestReopenWithCommit (Random random, Directory dir, boolean withReopen) throws IOException { + IndexWriter iwriter = new IndexWriter(dir, newIndexWriterConfig( + TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode( + OpenMode.CREATE).setMergeScheduler(new SerialMergeScheduler()).setMergePolicy(newLogMergePolicy())); + iwriter.commit(); + IndexReader reader = IndexReader.open(dir, false); + try { + int M = 3; + for (int i=0; i<4; i++) { + for (int j=0; j0) { + int k = i-1; + int n = j + k*M; + Document prevItereationDoc = reader.document(n); + assertNotNull(prevItereationDoc); + String id = prevItereationDoc.get("id"); + assertEquals(k+"_"+j, id); + } + } + iwriter.commit(); + if (withReopen) { + // reopen + IndexReader r2 = reader.reopen(); + if (reader != r2) { + reader.close(); + reader = r2; + } + } else { + // recreate + reader.close(); + reader = IndexReader.open(dir, false); + } + } + } finally { + iwriter.close(); + reader.close(); + } + } + + public void testMultiReaderReopen() throws Exception { + final Directory dir1 = newDirectory(); + createIndex(random, dir1, true); + + final Directory dir2 = newDirectory(); + createIndex(random, dir2, true); + + performDefaultTests(new TestReopen() { + + @Override + protected void modifyIndex(int i) throws IOException { + TestIndexReaderReopen.modifyIndex(i, dir1); + TestIndexReaderReopen.modifyIndex(i, dir2); + } + + @Override + protected IndexReader openReader() throws IOException { + return new MultiReader(new IndexReader[] + {IndexReader.open(dir1, false), + IndexReader.open(dir2, false)}); + } + + }); + + dir1.close(); + dir2.close(); + + final Directory dir3 = newDirectory(); + createIndex(random, dir3, true); + + final Directory dir4 = newDirectory(); + createIndex(random, dir4, true); + + performTestsWithExceptionInReopen(new TestReopen() { + + @Override + protected void modifyIndex(int i) throws IOException { + TestIndexReaderReopen.modifyIndex(i, dir3); + TestIndexReaderReopen.modifyIndex(i, dir4); + } + + @Override + protected IndexReader openReader() throws IOException { + return new MultiReader(new IndexReader[] + {IndexReader.open(dir3, false), + IndexReader.open(dir4, false), + // Does not implement reopen, so + // hits exception: + new FilterIndexReader(IndexReader.open(dir3, false))}); + } + + }); + dir3.close(); + dir4.close(); + } + + public void testMixedReaders() throws Exception { + final Directory dir1 = newDirectory(); + createIndex(random, dir1, true); + final Directory dir2 = newDirectory(); + createIndex(random, dir2, true); + final Directory dir3 = newDirectory(); + createIndex(random, dir3, false); + final Directory dir4 = newDirectory(); + createIndex(random, dir4, true); + final Directory dir5 = newDirectory(); + createIndex(random, dir5, false); + + performDefaultTests(new TestReopen() { + + @Override + protected void modifyIndex(int i) throws IOException { + // only change norms in this index to maintain the same number of docs for each of ParallelReader's subreaders + if (i == 1) TestIndexReaderReopen.modifyIndex(i, dir1); + + TestIndexReaderReopen.modifyIndex(i, dir4); + TestIndexReaderReopen.modifyIndex(i, dir5); + } + + @Override + protected IndexReader openReader() throws IOException { + ParallelReader pr = new ParallelReader(); + pr.add(IndexReader.open(dir1, false)); + pr.add(IndexReader.open(dir2, false)); + MultiReader mr = new MultiReader(new IndexReader[] { + IndexReader.open(dir3, false), IndexReader.open(dir4, false)}); + return new MultiReader(new IndexReader[] { + pr, mr, IndexReader.open(dir5, false)}); + } + }); + dir1.close(); + dir2.close(); + dir3.close(); + dir4.close(); + dir5.close(); + } + + private void performDefaultTests(TestReopen test) throws Exception { + + IndexReader index1 = test.openReader(); + IndexReader index2 = test.openReader(); + + TestIndexReader.assertIndexEquals(index1, index2); + + // verify that reopen() does not return a new reader instance + // in case the index has no changes + ReaderCouple couple = refreshReader(index2, false); + assertTrue(couple.refreshedReader == index2); + + couple = refreshReader(index2, test, 0, true); + index1.close(); + index1 = couple.newReader; + + IndexReader index2_refreshed = couple.refreshedReader; + index2.close(); + + // test if refreshed reader and newly opened reader return equal results + TestIndexReader.assertIndexEquals(index1, index2_refreshed); + + index2_refreshed.close(); + assertReaderClosed(index2, true, true); + assertReaderClosed(index2_refreshed, true, true); + + index2 = test.openReader(); + + for (int i = 1; i < 4; i++) { + + index1.close(); + couple = refreshReader(index2, test, i, true); + // refresh IndexReader + index2.close(); + + index2 = couple.refreshedReader; + index1 = couple.newReader; + TestIndexReader.assertIndexEquals(index1, index2); + } + + index1.close(); + index2.close(); + assertReaderClosed(index1, true, true); + assertReaderClosed(index2, true, true); + } + + public void testReferenceCounting() throws IOException { + for (int mode = 0; mode < 4; mode++) { + Directory dir1 = newDirectory(); + createIndex(random, dir1, true); + + IndexReader reader0 = IndexReader.open(dir1, false); + assertRefCountEquals(1, reader0); + + assertTrue(reader0 instanceof DirectoryReader); + IndexReader[] subReaders0 = reader0.getSequentialSubReaders(); + for (int i = 0; i < subReaders0.length; i++) { + assertRefCountEquals(1, subReaders0[i]); + } + + // delete first document, so that only one of the subReaders have to be re-opened + IndexReader modifier = IndexReader.open(dir1, false); + modifier.deleteDocument(0); + modifier.close(); + + IndexReader reader1 = refreshReader(reader0, true).refreshedReader; + assertTrue(reader1 instanceof DirectoryReader); + IndexReader[] subReaders1 = reader1.getSequentialSubReaders(); + assertEquals(subReaders0.length, subReaders1.length); + + for (int i = 0; i < subReaders0.length; i++) { + if (subReaders0[i] != subReaders1[i]) { + assertRefCountEquals(1, subReaders0[i]); + assertRefCountEquals(1, subReaders1[i]); + } else { + assertRefCountEquals(2, subReaders0[i]); + } + } + + // delete first document, so that only one of the subReaders have to be re-opened + modifier = IndexReader.open(dir1, false); + modifier.deleteDocument(1); + modifier.close(); + + IndexReader reader2 = refreshReader(reader1, true).refreshedReader; + assertTrue(reader2 instanceof DirectoryReader); + IndexReader[] subReaders2 = reader2.getSequentialSubReaders(); + assertEquals(subReaders1.length, subReaders2.length); + + for (int i = 0; i < subReaders2.length; i++) { + if (subReaders2[i] == subReaders1[i]) { + if (subReaders1[i] == subReaders0[i]) { + assertRefCountEquals(3, subReaders2[i]); + } else { + assertRefCountEquals(2, subReaders2[i]); + } + } else { + assertRefCountEquals(1, subReaders2[i]); + if (subReaders0[i] == subReaders1[i]) { + assertRefCountEquals(2, subReaders2[i]); + assertRefCountEquals(2, subReaders0[i]); + } else { + assertRefCountEquals(1, subReaders0[i]); + assertRefCountEquals(1, subReaders1[i]); + } + } + } + + IndexReader reader3 = refreshReader(reader0, true).refreshedReader; + assertTrue(reader3 instanceof DirectoryReader); + IndexReader[] subReaders3 = reader3.getSequentialSubReaders(); + assertEquals(subReaders3.length, subReaders0.length); + + // try some permutations + switch (mode) { + case 0: + reader0.close(); + reader1.close(); + reader2.close(); + reader3.close(); + break; + case 1: + reader3.close(); + reader2.close(); + reader1.close(); + reader0.close(); + break; + case 2: + reader2.close(); + reader3.close(); + reader0.close(); + reader1.close(); + break; + case 3: + reader1.close(); + reader3.close(); + reader2.close(); + reader0.close(); + break; + } + + assertReaderClosed(reader0, true, true); + assertReaderClosed(reader1, true, true); + assertReaderClosed(reader2, true, true); + assertReaderClosed(reader3, true, true); + + dir1.close(); + } + } + + + public void testReferenceCountingMultiReader() throws IOException { + for (int mode = 0; mode <=1; mode++) { + Directory dir1 = newDirectory(); + createIndex(random, dir1, false); + Directory dir2 = newDirectory(); + createIndex(random, dir2, true); + + IndexReader reader1 = IndexReader.open(dir1, false); + assertRefCountEquals(1, reader1); + + IndexReader initReader2 = IndexReader.open(dir2, false); + IndexReader multiReader1 = new MultiReader(new IndexReader[] {reader1, initReader2}, (mode == 0)); + modifyIndex(0, dir2); + assertRefCountEquals(1 + mode, reader1); + + IndexReader multiReader2 = multiReader1.reopen(); + // index1 hasn't changed, so multiReader2 should share reader1 now with multiReader1 + assertRefCountEquals(2 + mode, reader1); + + modifyIndex(0, dir1); + IndexReader reader2 = reader1.reopen(); + assertRefCountEquals(2 + mode, reader1); + + if (mode == 1) { + initReader2.close(); + } + + modifyIndex(1, dir1); + IndexReader reader3 = reader2.reopen(); + assertRefCountEquals(2 + mode, reader1); + assertRefCountEquals(1, reader2); + + multiReader1.close(); + assertRefCountEquals(1 + mode, reader1); + + multiReader1.close(); + assertRefCountEquals(1 + mode, reader1); + + if (mode == 1) { + initReader2.close(); + } + + reader1.close(); + assertRefCountEquals(1, reader1); + + multiReader2.close(); + assertRefCountEquals(0, reader1); + + multiReader2.close(); + assertRefCountEquals(0, reader1); + + reader3.close(); + assertRefCountEquals(0, reader1); + assertReaderClosed(reader1, true, false); + + reader2.close(); + assertRefCountEquals(0, reader1); + assertReaderClosed(reader1, true, false); + + reader2.close(); + assertRefCountEquals(0, reader1); + + reader3.close(); + assertRefCountEquals(0, reader1); + assertReaderClosed(reader1, true, true); + dir1.close(); + dir2.close(); + } + + } + + public void testReferenceCountingParallelReader() throws IOException { + for (int mode = 0; mode <=1; mode++) { + Directory dir1 = newDirectory(); + createIndex(random, dir1, false); + Directory dir2 = newDirectory(); + createIndex(random, dir2, true); + + IndexReader reader1 = IndexReader.open(dir1, false); + assertRefCountEquals(1, reader1); + + ParallelReader parallelReader1 = new ParallelReader(mode == 0); + parallelReader1.add(reader1); + IndexReader initReader2 = IndexReader.open(dir2, false); + parallelReader1.add(initReader2); + modifyIndex(1, dir2); + assertRefCountEquals(1 + mode, reader1); + + IndexReader parallelReader2 = parallelReader1.reopen(); + // index1 hasn't changed, so parallelReader2 should share reader1 now with multiReader1 + assertRefCountEquals(2 + mode, reader1); + + modifyIndex(0, dir1); + modifyIndex(0, dir2); + IndexReader reader2 = reader1.reopen(); + assertRefCountEquals(2 + mode, reader1); + + if (mode == 1) { + initReader2.close(); + } + + modifyIndex(4, dir1); + IndexReader reader3 = reader2.reopen(); + assertRefCountEquals(2 + mode, reader1); + assertRefCountEquals(1, reader2); + + parallelReader1.close(); + assertRefCountEquals(1 + mode, reader1); + + parallelReader1.close(); + assertRefCountEquals(1 + mode, reader1); + + if (mode == 1) { + initReader2.close(); + } + + reader1.close(); + assertRefCountEquals(1, reader1); + + parallelReader2.close(); + assertRefCountEquals(0, reader1); + + parallelReader2.close(); + assertRefCountEquals(0, reader1); + + reader3.close(); + assertRefCountEquals(0, reader1); + assertReaderClosed(reader1, true, false); + + reader2.close(); + assertRefCountEquals(0, reader1); + assertReaderClosed(reader1, true, false); + + reader2.close(); + assertRefCountEquals(0, reader1); + + reader3.close(); + assertRefCountEquals(0, reader1); + assertReaderClosed(reader1, true, true); + + dir1.close(); + dir2.close(); + } + + } + + public void testNormsRefCounting() throws IOException { + Directory dir1 = newDirectory(); + createIndex(random, dir1, false); + + IndexReader reader1 = IndexReader.open(dir1, false); + SegmentReader segmentReader1 = SegmentReader.getOnlySegmentReader(reader1); + IndexReader modifier = IndexReader.open(dir1, false); + modifier.deleteDocument(0); + modifier.close(); + + IndexReader reader2 = reader1.reopen(); + modifier = IndexReader.open(dir1, false); + modifier.setNorm(1, "field1", 50); + modifier.setNorm(1, "field2", 50); + modifier.close(); + + IndexReader reader3 = reader2.reopen(); + SegmentReader segmentReader3 = SegmentReader.getOnlySegmentReader(reader3); + modifier = IndexReader.open(dir1, false); + modifier.deleteDocument(2); + modifier.close(); + + IndexReader reader4 = reader3.reopen(); + modifier = IndexReader.open(dir1, false); + modifier.deleteDocument(3); + modifier.close(); + + IndexReader reader5 = reader3.reopen(); + + // Now reader2-reader5 references reader1. reader1 and reader2 + // share the same norms. reader3, reader4, reader5 also share norms. + assertRefCountEquals(1, reader1); + assertFalse(segmentReader1.normsClosed()); + + reader1.close(); + + assertRefCountEquals(0, reader1); + assertFalse(segmentReader1.normsClosed()); + + reader2.close(); + assertRefCountEquals(0, reader1); + + // now the norms for field1 and field2 should be closed + assertTrue(segmentReader1.normsClosed("field1")); + assertTrue(segmentReader1.normsClosed("field2")); + + // but the norms for field3 and field4 should still be open + assertFalse(segmentReader1.normsClosed("field3")); + assertFalse(segmentReader1.normsClosed("field4")); + + reader3.close(); + assertRefCountEquals(0, reader1); + assertFalse(segmentReader3.normsClosed()); + reader5.close(); + assertRefCountEquals(0, reader1); + assertFalse(segmentReader3.normsClosed()); + reader4.close(); + assertRefCountEquals(0, reader1); + + // and now all norms that reader1 used should be closed + assertTrue(segmentReader1.normsClosed()); + + // now that reader3, reader4 and reader5 are closed, + // the norms that those three readers shared should be + // closed as well + assertTrue(segmentReader3.normsClosed()); + + dir1.close(); + } + + private void performTestsWithExceptionInReopen(TestReopen test) throws Exception { + IndexReader index1 = test.openReader(); + IndexReader index2 = test.openReader(); + + TestIndexReader.assertIndexEquals(index1, index2); + + try { + refreshReader(index1, test, 0, true); + fail("Expected exception not thrown."); + } catch (Exception e) { + // expected exception + } + + // index2 should still be usable and unaffected by the failed reopen() call + TestIndexReader.assertIndexEquals(index1, index2); + + index1.close(); + index2.close(); + } + + public void testThreadSafety() throws Exception { + final Directory dir = newDirectory(); + // NOTE: this also controls the number of threads! + final int n = _TestUtil.nextInt(random, 20, 40); + IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( + TEST_VERSION_CURRENT, new MockAnalyzer(random))); + for (int i = 0; i < n; i++) { + writer.addDocument(createDocument(i, 3)); + } + writer.optimize(); + writer.close(); + + final TestReopen test = new TestReopen() { + @Override + protected void modifyIndex(int i) throws IOException { + if (i % 3 == 0) { + IndexReader modifier = IndexReader.open(dir, false); + modifier.setNorm(i, "field1", 50); + modifier.close(); + } else if (i % 3 == 1) { + IndexReader modifier = IndexReader.open(dir, false); + modifier.deleteDocument(i % modifier.maxDoc()); + modifier.close(); + } else { + IndexWriter modifier = new IndexWriter(dir, new IndexWriterConfig( + TEST_VERSION_CURRENT, new MockAnalyzer(random))); + modifier.addDocument(createDocument(n + i, 6)); + modifier.close(); + } + } + + @Override + protected IndexReader openReader() throws IOException { + return IndexReader.open(dir, false); + } + }; + + final List readers = Collections.synchronizedList(new ArrayList()); + IndexReader firstReader = IndexReader.open(dir, false); + IndexReader reader = firstReader; + final Random rnd = random; + + ReaderThread[] threads = new ReaderThread[n]; + final Set readersToClose = Collections.synchronizedSet(new HashSet()); + + for (int i = 0; i < n; i++) { + if (i % 2 == 0) { + IndexReader refreshed = reader.reopen(); + if (refreshed != reader) { + readersToClose.add(reader); + } + reader = refreshed; + } + final IndexReader r = reader; + + final int index = i; + + ReaderThreadTask task; + + if (i < 4 || (i >=10 && i < 14) || i > 18) { + task = new ReaderThreadTask() { + + @Override + public void run() throws Exception { + while (!stopped) { + if (index % 2 == 0) { + // refresh reader synchronized + ReaderCouple c = (refreshReader(r, test, index, true)); + readersToClose.add(c.newReader); + readersToClose.add(c.refreshedReader); + readers.add(c); + // prevent too many readers + break; + } else { + // not synchronized + IndexReader refreshed = r.reopen(); + + IndexSearcher searcher = newSearcher(refreshed); + ScoreDoc[] hits = searcher.search( + new TermQuery(new Term("field1", "a" + rnd.nextInt(refreshed.maxDoc()))), + null, 1000).scoreDocs; + if (hits.length > 0) { + searcher.doc(hits[0].doc); + } + searcher.close(); + if (refreshed != r) { + refreshed.close(); + } + } + synchronized(this) { + wait(_TestUtil.nextInt(random, 1, 100)); + } + } + } + + }; + } else { + task = new ReaderThreadTask() { + @Override + public void run() throws Exception { + while (!stopped) { + int numReaders = readers.size(); + if (numReaders > 0) { + ReaderCouple c = readers.get(rnd.nextInt(numReaders)); + TestIndexReader.assertIndexEquals(c.newReader, c.refreshedReader); + } + + synchronized(this) { + wait(_TestUtil.nextInt(random, 1, 100)); + } + } + } + }; + } + + threads[i] = new ReaderThread(task); + threads[i].start(); + } + + synchronized(this) { + wait(1000); + } + + for (int i = 0; i < n; i++) { + if (threads[i] != null) { + threads[i].stopThread(); + } + } + + for (int i = 0; i < n; i++) { + if (threads[i] != null) { + threads[i].join(); + if (threads[i].error != null) { + String msg = "Error occurred in thread " + threads[i].getName() + ":\n" + threads[i].error.getMessage(); + fail(msg); + } + } + + } + + for (final IndexReader readerToClose : readersToClose) { + readerToClose.close(); + } + + firstReader.close(); + reader.close(); + + for (final IndexReader readerToClose : readersToClose) { + assertReaderClosed(readerToClose, true, true); + } + + assertReaderClosed(reader, true, true); + assertReaderClosed(firstReader, true, true); + + dir.close(); + } + + private static class ReaderCouple { + ReaderCouple(IndexReader r1, IndexReader r2) { + newReader = r1; + refreshedReader = r2; + } + + IndexReader newReader; + IndexReader refreshedReader; + } + + private abstract static class ReaderThreadTask { + protected volatile boolean stopped; + public void stop() { + this.stopped = true; + } + + public abstract void run() throws Exception; + } + + private static class ReaderThread extends Thread { + private ReaderThreadTask task; + private Throwable error; + + + ReaderThread(ReaderThreadTask task) { + this.task = task; + } + + public void stopThread() { + this.task.stop(); + } + + @Override + public void run() { + try { + this.task.run(); + } catch (Throwable r) { + r.printStackTrace(System.out); + this.error = r; + } + } + } + + private Object createReaderMutex = new Object(); + + private ReaderCouple refreshReader(IndexReader reader, boolean hasChanges) throws IOException { + return refreshReader(reader, null, -1, hasChanges); + } + + ReaderCouple refreshReader(IndexReader reader, TestReopen test, int modify, boolean hasChanges) throws IOException { + synchronized (createReaderMutex) { + IndexReader r = null; + if (test != null) { + test.modifyIndex(modify); + r = test.openReader(); + } + + IndexReader refreshed = null; + try { + refreshed = reader.reopen(); + } finally { + if (refreshed == null && r != null) { + // Hit exception -- close opened reader + r.close(); + } + } + + if (hasChanges) { + if (refreshed == reader) { + fail("No new IndexReader instance created during refresh."); + } + } else { + if (refreshed != reader) { + fail("New IndexReader instance created during refresh even though index had no changes."); + } + } + + return new ReaderCouple(r, refreshed); + } + } + + public static void createIndex(Random random, Directory dir, boolean multiSegment) throws IOException { + IndexWriter.unlock(dir); + IndexWriter w = new IndexWriter(dir, LuceneTestCase.newIndexWriterConfig(random, + TEST_VERSION_CURRENT, new MockAnalyzer(random)) + .setMergePolicy(new LogDocMergePolicy())); + + for (int i = 0; i < 100; i++) { + w.addDocument(createDocument(i, 4)); + if (multiSegment && (i % 10) == 0) { + w.commit(); + } + } + + if (!multiSegment) { + w.optimize(); + } + + w.close(); + + IndexReader r = IndexReader.open(dir, false); + if (multiSegment) { + assertTrue(r.getSequentialSubReaders().length > 1); + } else { + assertTrue(r.getSequentialSubReaders().length == 1); + } + r.close(); + } + + public static Document createDocument(int n, int numFields) { + StringBuilder sb = new StringBuilder(); + Document doc = new Document(); + sb.append("a"); + sb.append(n); + doc.add(new Field("field1", sb.toString(), Store.YES, Index.ANALYZED)); + doc.add(new Field("fielda", sb.toString(), Store.YES, Index.NOT_ANALYZED_NO_NORMS)); + doc.add(new Field("fieldb", sb.toString(), Store.YES, Index.NO)); + sb.append(" b"); + sb.append(n); + for (int i = 1; i < numFields; i++) { + doc.add(new Field("field" + (i+1), sb.toString(), Store.YES, Index.ANALYZED)); + } + return doc; + } + + static void modifyIndex(int i, Directory dir) throws IOException { + switch (i) { + case 0: { + if (VERBOSE) { + System.out.println("TEST: modify index"); + } + IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); + w.setInfoStream(VERBOSE ? System.out : null); + w.deleteDocuments(new Term("field2", "a11")); + w.deleteDocuments(new Term("field2", "b30")); + w.close(); + break; + } + case 1: { + IndexReader reader = IndexReader.open(dir, false); + reader.setNorm(4, "field1", 123); + reader.setNorm(44, "field2", 222); + reader.setNorm(44, "field4", 22); + reader.close(); + break; + } + case 2: { + IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); + w.optimize(); + w.close(); + break; + } + case 3: { + IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); + w.addDocument(createDocument(101, 4)); + w.optimize(); + w.addDocument(createDocument(102, 4)); + w.addDocument(createDocument(103, 4)); + w.close(); + break; + } + case 4: { + IndexReader reader = IndexReader.open(dir, false); + reader.setNorm(5, "field1", 123); + reader.setNorm(55, "field2", 222); + reader.close(); + break; + } + case 5: { + IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); + w.addDocument(createDocument(101, 4)); + w.close(); + break; + } + } + } + + private void assertReaderClosed(IndexReader reader, boolean checkSubReaders, boolean checkNormsClosed) { + assertEquals(0, reader.getRefCount()); + + if (checkNormsClosed && reader instanceof SegmentReader) { + assertTrue(((SegmentReader) reader).normsClosed()); + } + + if (checkSubReaders) { + if (reader instanceof DirectoryReader) { + IndexReader[] subReaders = reader.getSequentialSubReaders(); + for (int i = 0; i < subReaders.length; i++) { + assertReaderClosed(subReaders[i], checkSubReaders, checkNormsClosed); + } + } + + if (reader instanceof MultiReader) { + IndexReader[] subReaders = reader.getSequentialSubReaders(); + for (int i = 0; i < subReaders.length; i++) { + assertReaderClosed(subReaders[i], checkSubReaders, checkNormsClosed); + } + } + + if (reader instanceof ParallelReader) { + IndexReader[] subReaders = ((ParallelReader) reader).getSubReaders(); + for (int i = 0; i < subReaders.length; i++) { + assertReaderClosed(subReaders[i], checkSubReaders, checkNormsClosed); + } + } + } + } + + /* + private void assertReaderOpen(IndexReader reader) { + reader.ensureOpen(); + + if (reader instanceof DirectoryReader) { + IndexReader[] subReaders = reader.getSequentialSubReaders(); + for (int i = 0; i < subReaders.length; i++) { + assertReaderOpen(subReaders[i]); + } + } + } + */ + + private void assertRefCountEquals(int refCount, IndexReader reader) { + assertEquals("Reader has wrong refCount value.", refCount, reader.getRefCount()); + } + + + private abstract static class TestReopen { + protected abstract IndexReader openReader() throws IOException; + protected abstract void modifyIndex(int i) throws IOException; + } + + public void testCloseOrig() throws Throwable { + Directory dir = newDirectory(); + createIndex(random, dir, false); + IndexReader r1 = IndexReader.open(dir, false); + IndexReader r2 = IndexReader.open(dir, false); + r2.deleteDocument(0); + r2.close(); + + IndexReader r3 = r1.reopen(); + assertTrue(r1 != r3); + r1.close(); + try { + r1.document(2); + fail("did not hit exception"); + } catch (AlreadyClosedException ace) { + // expected + } + r3.close(); + dir.close(); + } + + public void testDeletes() throws Throwable { + Directory dir = newDirectory(); + createIndex(random, dir, false); // Create an index with a bunch of docs (1 segment) + + modifyIndex(0, dir); // Get delete bitVector on 1st segment + modifyIndex(5, dir); // Add a doc (2 segments) + + IndexReader r1 = IndexReader.open(dir, false); // MSR + + modifyIndex(5, dir); // Add another doc (3 segments) + + IndexReader r2 = r1.reopen(); // MSR + assertTrue(r1 != r2); + + SegmentReader sr1 = (SegmentReader) r1.getSequentialSubReaders()[0]; // Get SRs for the first segment from original + SegmentReader sr2 = (SegmentReader) r2.getSequentialSubReaders()[0]; // and reopened IRs + + // At this point they share the same BitVector + assertTrue(sr1.deletedDocs==sr2.deletedDocs); + + r2.deleteDocument(0); + + // r1 should not see the delete + assertFalse(r1.isDeleted(0)); + + // Now r2 should have made a private copy of deleted docs: + assertTrue(sr1.deletedDocs!=sr2.deletedDocs); + + r1.close(); + r2.close(); + dir.close(); + } + + public void testDeletes2() throws Throwable { + Directory dir = newDirectory(); + createIndex(random, dir, false); + // Get delete bitVector + modifyIndex(0, dir); + IndexReader r1 = IndexReader.open(dir, false); + + // Add doc: + modifyIndex(5, dir); + + IndexReader r2 = r1.reopen(); + assertTrue(r1 != r2); + + IndexReader[] rs2 = r2.getSequentialSubReaders(); + + SegmentReader sr1 = SegmentReader.getOnlySegmentReader(r1); + SegmentReader sr2 = (SegmentReader) rs2[0]; + + // At this point they share the same BitVector + assertTrue(sr1.deletedDocs==sr2.deletedDocs); + final BitVector delDocs = sr1.deletedDocs; + r1.close(); + + r2.deleteDocument(0); + assertTrue(delDocs==sr2.deletedDocs); + r2.close(); + dir.close(); + } + + private static class KeepAllCommits implements IndexDeletionPolicy { + public void onInit(List commits) { + } + public void onCommit(List commits) { + } + } + + public void testReopenOnCommit() throws Throwable { + Directory dir = newDirectory(); + IndexWriter writer = new IndexWriter( + dir, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). + setIndexDeletionPolicy(new KeepAllCommits()). + setMaxBufferedDocs(-1). + setMergePolicy(newLogMergePolicy(10)) + ); + for(int i=0;i<4;i++) { + Document doc = new Document(); + doc.add(newField("id", ""+i, Field.Store.NO, Field.Index.NOT_ANALYZED)); + writer.addDocument(doc); + Map data = new HashMap(); + data.put("index", i+""); + writer.commit(data); + } + for(int i=0;i<4;i++) { + writer.deleteDocuments(new Term("id", ""+i)); + Map data = new HashMap(); + data.put("index", (4+i)+""); + writer.commit(data); + } + writer.close(); + + IndexReader r = IndexReader.open(dir, false); + assertEquals(0, r.numDocs()); + + Collection commits = IndexReader.listCommits(dir); + for (final IndexCommit commit : commits) { + IndexReader r2 = r.reopen(commit); + assertTrue(r2 != r); + + // Reader should be readOnly + try { + r2.deleteDocument(0); + fail("no exception hit"); + } catch (UnsupportedOperationException uoe) { + // expected + } + + final Map s = commit.getUserData(); + final int v; + if (s.size() == 0) { + // First commit created by IW + v = -1; + } else { + v = Integer.parseInt(s.get("index")); + } + if (v < 4) { + assertEquals(1+v, r2.numDocs()); + } else { + assertEquals(7-v, r2.numDocs()); + } + r.close(); + r = r2; + } + r.close(); + dir.close(); + } + + // LUCENE-1579: Make sure all SegmentReaders are new when + // reopen switches readOnly + public void testReopenChangeReadonly() throws Exception { + Directory dir = newDirectory(); + IndexWriter writer = new IndexWriter( + dir, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). + setMaxBufferedDocs(-1). + setMergePolicy(newLogMergePolicy(10)) + ); + Document doc = new Document(); + doc.add(newField("number", "17", Field.Store.NO, Field.Index.NOT_ANALYZED)); + writer.addDocument(doc); + writer.commit(); + + // Open reader1 + IndexReader r = IndexReader.open(dir, false); + assertTrue(r instanceof DirectoryReader); + IndexReader r1 = SegmentReader.getOnlySegmentReader(r); + final int[] ints = FieldCache.DEFAULT.getInts(r1, "number"); + assertEquals(1, ints.length); + assertEquals(17, ints[0]); + + // Reopen to readonly w/ no chnages + IndexReader r3 = r.reopen(true); + assertTrue(r3 instanceof ReadOnlyDirectoryReader); + r3.close(); + + // Add new segment + writer.addDocument(doc); + writer.commit(); + + // Reopen reader1 --> reader2 + IndexReader r2 = r.reopen(true); + r.close(); + assertTrue(r2 instanceof ReadOnlyDirectoryReader); + IndexReader[] subs = r2.getSequentialSubReaders(); + final int[] ints2 = FieldCache.DEFAULT.getInts(subs[0], "number"); + r2.close(); + + assertTrue(subs[0] instanceof ReadOnlySegmentReader); + assertTrue(subs[1] instanceof ReadOnlySegmentReader); + assertTrue(ints == ints2); + + writer.close(); + dir.close(); + } +}