package org.apache.lucene.index; /** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Random; import java.util.Set; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field.Index; import org.apache.lucene.document.Field.Store; import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.search.FieldCache; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BitVector; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util._TestUtil; public class TestIndexReaderReopen extends LuceneTestCase { public void testReopen() throws Exception { final Directory dir1 = newDirectory(); createIndex(random, dir1, false); performDefaultTests(new TestReopen() { @Override protected void modifyIndex(int i) throws IOException { TestIndexReaderReopen.modifyIndex(i, dir1); } @Override protected IndexReader openReader() throws IOException { return IndexReader.open(dir1, false); } }); dir1.close(); final Directory dir2 = newDirectory(); createIndex(random, dir2, true); performDefaultTests(new TestReopen() { @Override protected void modifyIndex(int i) throws IOException { TestIndexReaderReopen.modifyIndex(i, dir2); } @Override protected IndexReader openReader() throws IOException { return IndexReader.open(dir2, false); } }); dir2.close(); } public void testParallelReaderReopen() throws Exception { final Directory dir1 = newDirectory(); createIndex(random, dir1, true); final Directory dir2 = newDirectory(); createIndex(random, dir2, true); performDefaultTests(new TestReopen() { @Override protected void modifyIndex(int i) throws IOException { TestIndexReaderReopen.modifyIndex(i, dir1); TestIndexReaderReopen.modifyIndex(i, dir2); } @Override protected IndexReader openReader() throws IOException { ParallelReader pr = new ParallelReader(); pr.add(IndexReader.open(dir1, false)); pr.add(IndexReader.open(dir2, false)); return pr; } }); dir1.close(); dir2.close(); final Directory dir3 = newDirectory(); createIndex(random, dir3, true); final Directory dir4 = newDirectory(); createIndex(random, dir4, true); performTestsWithExceptionInReopen(new TestReopen() { @Override protected void modifyIndex(int i) throws IOException { TestIndexReaderReopen.modifyIndex(i, dir3); TestIndexReaderReopen.modifyIndex(i, dir4); } @Override protected IndexReader openReader() throws IOException { ParallelReader pr = new ParallelReader(); pr.add(IndexReader.open(dir3, false)); pr.add(IndexReader.open(dir4, false)); // Does not implement reopen, so // hits exception: pr.add(new FilterIndexReader(IndexReader.open(dir3, false))); return pr; } }); dir3.close(); dir4.close(); } // LUCENE-1228: IndexWriter.commit() does not update the index version // populate an index in iterations. // at the end of every iteration, commit the index and reopen/recreate the reader. // in each iteration verify the work of previous iteration. // try this once with reopen once recreate, on both RAMDir and FSDir. public void testCommitReopen () throws IOException { Directory dir = newDirectory(); doTestReopenWithCommit(random, dir, true); dir.close(); } public void testCommitRecreate () throws IOException { Directory dir = newDirectory(); doTestReopenWithCommit(random, dir, false); dir.close(); } private void doTestReopenWithCommit (Random random, Directory dir, boolean withReopen) throws IOException { IndexWriter iwriter = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode( OpenMode.CREATE).setMergeScheduler(new SerialMergeScheduler()).setMergePolicy(newLogMergePolicy())); iwriter.commit(); IndexReader reader = IndexReader.open(dir, false); try { int M = 3; for (int i=0; i<4; i++) { for (int j=0; j0) { int k = i-1; int n = j + k*M; Document prevItereationDoc = reader.document(n); assertNotNull(prevItereationDoc); String id = prevItereationDoc.get("id"); assertEquals(k+"_"+j, id); } } iwriter.commit(); if (withReopen) { // reopen IndexReader r2 = IndexReader.openIfChanged(reader); if (r2 != null) { reader.close(); reader = r2; } } else { // recreate reader.close(); reader = IndexReader.open(dir, false); } } } finally { iwriter.close(); reader.close(); } } public void testMultiReaderReopen() throws Exception { final Directory dir1 = newDirectory(); createIndex(random, dir1, true); final Directory dir2 = newDirectory(); createIndex(random, dir2, true); performDefaultTests(new TestReopen() { @Override protected void modifyIndex(int i) throws IOException { TestIndexReaderReopen.modifyIndex(i, dir1); TestIndexReaderReopen.modifyIndex(i, dir2); } @Override protected IndexReader openReader() throws IOException { return new MultiReader(new IndexReader[] {IndexReader.open(dir1, false), IndexReader.open(dir2, false)}); } }); dir1.close(); dir2.close(); final Directory dir3 = newDirectory(); createIndex(random, dir3, true); final Directory dir4 = newDirectory(); createIndex(random, dir4, true); performTestsWithExceptionInReopen(new TestReopen() { @Override protected void modifyIndex(int i) throws IOException { TestIndexReaderReopen.modifyIndex(i, dir3); TestIndexReaderReopen.modifyIndex(i, dir4); } @Override protected IndexReader openReader() throws IOException { return new MultiReader(new IndexReader[] {IndexReader.open(dir3, false), IndexReader.open(dir4, false), // Does not implement reopen, so // hits exception: new FilterIndexReader(IndexReader.open(dir3, false))}); } }); dir3.close(); dir4.close(); } public void testMixedReaders() throws Exception { final Directory dir1 = newDirectory(); createIndex(random, dir1, true); final Directory dir2 = newDirectory(); createIndex(random, dir2, true); final Directory dir3 = newDirectory(); createIndex(random, dir3, false); final Directory dir4 = newDirectory(); createIndex(random, dir4, true); final Directory dir5 = newDirectory(); createIndex(random, dir5, false); performDefaultTests(new TestReopen() { @Override protected void modifyIndex(int i) throws IOException { // only change norms in this index to maintain the same number of docs for each of ParallelReader's subreaders if (i == 1) TestIndexReaderReopen.modifyIndex(i, dir1); TestIndexReaderReopen.modifyIndex(i, dir4); TestIndexReaderReopen.modifyIndex(i, dir5); } @Override protected IndexReader openReader() throws IOException { ParallelReader pr = new ParallelReader(); pr.add(IndexReader.open(dir1, false)); pr.add(IndexReader.open(dir2, false)); MultiReader mr = new MultiReader(new IndexReader[] { IndexReader.open(dir3, false), IndexReader.open(dir4, false)}); return new MultiReader(new IndexReader[] { pr, mr, IndexReader.open(dir5, false)}); } }); dir1.close(); dir2.close(); dir3.close(); dir4.close(); dir5.close(); } private void performDefaultTests(TestReopen test) throws Exception { IndexReader index1 = test.openReader(); IndexReader index2 = test.openReader(); TestIndexReader.assertIndexEquals(index1, index2); // verify that reopen() does not return a new reader instance // in case the index has no changes ReaderCouple couple = refreshReader(index2, false); assertTrue(couple.refreshedReader == index2); couple = refreshReader(index2, test, 0, true); index1.close(); index1 = couple.newReader; IndexReader index2_refreshed = couple.refreshedReader; index2.close(); // test if refreshed reader and newly opened reader return equal results TestIndexReader.assertIndexEquals(index1, index2_refreshed); index2_refreshed.close(); assertReaderClosed(index2, true, true); assertReaderClosed(index2_refreshed, true, true); index2 = test.openReader(); for (int i = 1; i < 4; i++) { index1.close(); couple = refreshReader(index2, test, i, true); // refresh IndexReader index2.close(); index2 = couple.refreshedReader; index1 = couple.newReader; TestIndexReader.assertIndexEquals(index1, index2); } index1.close(); index2.close(); assertReaderClosed(index1, true, true); assertReaderClosed(index2, true, true); } public void testReferenceCounting() throws IOException { for (int mode = 0; mode < 4; mode++) { Directory dir1 = newDirectory(); createIndex(random, dir1, true); IndexReader reader0 = IndexReader.open(dir1, false); assertRefCountEquals(1, reader0); assertTrue(reader0 instanceof DirectoryReader); IndexReader[] subReaders0 = reader0.getSequentialSubReaders(); for (int i = 0; i < subReaders0.length; i++) { assertRefCountEquals(1, subReaders0[i]); } // delete first document, so that only one of the subReaders have to be re-opened IndexReader modifier = IndexReader.open(dir1, false); modifier.deleteDocument(0); modifier.close(); IndexReader reader1 = refreshReader(reader0, true).refreshedReader; assertTrue(reader1 instanceof DirectoryReader); IndexReader[] subReaders1 = reader1.getSequentialSubReaders(); assertEquals(subReaders0.length, subReaders1.length); for (int i = 0; i < subReaders0.length; i++) { if (subReaders0[i] != subReaders1[i]) { assertRefCountEquals(1, subReaders0[i]); assertRefCountEquals(1, subReaders1[i]); } else { assertRefCountEquals(2, subReaders0[i]); } } // delete first document, so that only one of the subReaders have to be re-opened modifier = IndexReader.open(dir1, false); modifier.deleteDocument(1); modifier.close(); IndexReader reader2 = refreshReader(reader1, true).refreshedReader; assertTrue(reader2 instanceof DirectoryReader); IndexReader[] subReaders2 = reader2.getSequentialSubReaders(); assertEquals(subReaders1.length, subReaders2.length); for (int i = 0; i < subReaders2.length; i++) { if (subReaders2[i] == subReaders1[i]) { if (subReaders1[i] == subReaders0[i]) { assertRefCountEquals(3, subReaders2[i]); } else { assertRefCountEquals(2, subReaders2[i]); } } else { assertRefCountEquals(1, subReaders2[i]); if (subReaders0[i] == subReaders1[i]) { assertRefCountEquals(2, subReaders2[i]); assertRefCountEquals(2, subReaders0[i]); } else { assertRefCountEquals(1, subReaders0[i]); assertRefCountEquals(1, subReaders1[i]); } } } IndexReader reader3 = refreshReader(reader0, true).refreshedReader; assertTrue(reader3 instanceof DirectoryReader); IndexReader[] subReaders3 = reader3.getSequentialSubReaders(); assertEquals(subReaders3.length, subReaders0.length); // try some permutations switch (mode) { case 0: reader0.close(); reader1.close(); reader2.close(); reader3.close(); break; case 1: reader3.close(); reader2.close(); reader1.close(); reader0.close(); break; case 2: reader2.close(); reader3.close(); reader0.close(); reader1.close(); break; case 3: reader1.close(); reader3.close(); reader2.close(); reader0.close(); break; } assertReaderClosed(reader0, true, true); assertReaderClosed(reader1, true, true); assertReaderClosed(reader2, true, true); assertReaderClosed(reader3, true, true); dir1.close(); } } public void testReferenceCountingMultiReader() throws IOException { for (int mode = 0; mode <=1; mode++) { Directory dir1 = newDirectory(); createIndex(random, dir1, false); Directory dir2 = newDirectory(); createIndex(random, dir2, true); IndexReader reader1 = IndexReader.open(dir1, false); assertRefCountEquals(1, reader1); IndexReader initReader2 = IndexReader.open(dir2, false); IndexReader multiReader1 = new MultiReader(new IndexReader[] {reader1, initReader2}, (mode == 0)); modifyIndex(0, dir2); assertRefCountEquals(1 + mode, reader1); IndexReader multiReader2 = IndexReader.openIfChanged(multiReader1); assertNotNull(multiReader2); // index1 hasn't changed, so multiReader2 should share reader1 now with multiReader1 assertRefCountEquals(2 + mode, reader1); modifyIndex(0, dir1); IndexReader reader2 = IndexReader.openIfChanged(reader1); assertNotNull(reader2); assertNull(IndexReader.openIfChanged(reader2)); assertRefCountEquals(2 + mode, reader1); if (mode == 1) { initReader2.close(); } modifyIndex(1, dir1); IndexReader reader3 = IndexReader.openIfChanged(reader2); assertNotNull(reader3); assertRefCountEquals(2 + mode, reader1); assertRefCountEquals(1, reader2); multiReader1.close(); assertRefCountEquals(1 + mode, reader1); multiReader1.close(); assertRefCountEquals(1 + mode, reader1); if (mode == 1) { initReader2.close(); } reader1.close(); assertRefCountEquals(1, reader1); multiReader2.close(); assertRefCountEquals(0, reader1); multiReader2.close(); assertRefCountEquals(0, reader1); reader3.close(); assertRefCountEquals(0, reader1); assertReaderClosed(reader1, true, false); reader2.close(); assertRefCountEquals(0, reader1); assertReaderClosed(reader1, true, false); reader2.close(); assertRefCountEquals(0, reader1); reader3.close(); assertRefCountEquals(0, reader1); assertReaderClosed(reader1, true, true); dir1.close(); dir2.close(); } } public void testReferenceCountingParallelReader() throws IOException { for (int mode = 0; mode <=1; mode++) { Directory dir1 = newDirectory(); createIndex(random, dir1, false); Directory dir2 = newDirectory(); createIndex(random, dir2, true); IndexReader reader1 = IndexReader.open(dir1, false); assertRefCountEquals(1, reader1); ParallelReader parallelReader1 = new ParallelReader(mode == 0); parallelReader1.add(reader1); IndexReader initReader2 = IndexReader.open(dir2, false); parallelReader1.add(initReader2); modifyIndex(1, dir2); assertRefCountEquals(1 + mode, reader1); IndexReader parallelReader2 = IndexReader.openIfChanged(parallelReader1); assertNotNull(parallelReader2); assertNull(IndexReader.openIfChanged(parallelReader2)); // index1 hasn't changed, so parallelReader2 should share reader1 now with multiReader1 assertRefCountEquals(2 + mode, reader1); modifyIndex(0, dir1); modifyIndex(0, dir2); IndexReader reader2 = IndexReader.openIfChanged(reader1); assertNotNull(reader2); assertRefCountEquals(2 + mode, reader1); if (mode == 1) { initReader2.close(); } modifyIndex(4, dir1); IndexReader reader3 = IndexReader.openIfChanged(reader2); assertNotNull(reader3); assertRefCountEquals(2 + mode, reader1); assertRefCountEquals(1, reader2); parallelReader1.close(); assertRefCountEquals(1 + mode, reader1); parallelReader1.close(); assertRefCountEquals(1 + mode, reader1); if (mode == 1) { initReader2.close(); } reader1.close(); assertRefCountEquals(1, reader1); parallelReader2.close(); assertRefCountEquals(0, reader1); parallelReader2.close(); assertRefCountEquals(0, reader1); reader3.close(); assertRefCountEquals(0, reader1); assertReaderClosed(reader1, true, false); reader2.close(); assertRefCountEquals(0, reader1); assertReaderClosed(reader1, true, false); reader2.close(); assertRefCountEquals(0, reader1); reader3.close(); assertRefCountEquals(0, reader1); assertReaderClosed(reader1, true, true); dir1.close(); dir2.close(); } } public void testNormsRefCounting() throws IOException { Directory dir1 = newDirectory(); createIndex(random, dir1, false); IndexReader reader1 = IndexReader.open(dir1, false); SegmentReader segmentReader1 = SegmentReader.getOnlySegmentReader(reader1); IndexReader modifier = IndexReader.open(dir1, false); modifier.deleteDocument(0); modifier.close(); IndexReader reader2 = IndexReader.openIfChanged(reader1); assertNotNull(reader2); modifier = IndexReader.open(dir1, false); modifier.setNorm(1, "field1", 50); modifier.setNorm(1, "field2", 50); modifier.close(); IndexReader reader3 = IndexReader.openIfChanged(reader2); assertNotNull(reader3); SegmentReader segmentReader3 = SegmentReader.getOnlySegmentReader(reader3); modifier = IndexReader.open(dir1, false); modifier.deleteDocument(2); modifier.close(); IndexReader reader4 = IndexReader.openIfChanged(reader3); assertNotNull(reader4); modifier = IndexReader.open(dir1, false); modifier.deleteDocument(3); modifier.close(); IndexReader reader5 = IndexReader.openIfChanged(reader3); assertNotNull(reader5); // Now reader2-reader5 references reader1. reader1 and reader2 // share the same norms. reader3, reader4, reader5 also share norms. assertRefCountEquals(1, reader1); assertFalse(segmentReader1.normsClosed()); reader1.close(); assertRefCountEquals(0, reader1); assertFalse(segmentReader1.normsClosed()); reader2.close(); assertRefCountEquals(0, reader1); // now the norms for field1 and field2 should be closed assertTrue(segmentReader1.normsClosed("field1")); assertTrue(segmentReader1.normsClosed("field2")); // but the norms for field3 and field4 should still be open assertFalse(segmentReader1.normsClosed("field3")); assertFalse(segmentReader1.normsClosed("field4")); reader3.close(); assertRefCountEquals(0, reader1); assertFalse(segmentReader3.normsClosed()); reader5.close(); assertRefCountEquals(0, reader1); assertFalse(segmentReader3.normsClosed()); reader4.close(); assertRefCountEquals(0, reader1); // and now all norms that reader1 used should be closed assertTrue(segmentReader1.normsClosed()); // now that reader3, reader4 and reader5 are closed, // the norms that those three readers shared should be // closed as well assertTrue(segmentReader3.normsClosed()); dir1.close(); } private void performTestsWithExceptionInReopen(TestReopen test) throws Exception { IndexReader index1 = test.openReader(); IndexReader index2 = test.openReader(); TestIndexReader.assertIndexEquals(index1, index2); try { refreshReader(index1, test, 0, true); fail("Expected exception not thrown."); } catch (Exception e) { // expected exception } // index2 should still be usable and unaffected by the failed reopen() call TestIndexReader.assertIndexEquals(index1, index2); index1.close(); index2.close(); } public void testThreadSafety() throws Exception { final Directory dir = newDirectory(); // NOTE: this also controls the number of threads! final int n = _TestUtil.nextInt(random, 20, 40); IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))); for (int i = 0; i < n; i++) { writer.addDocument(createDocument(i, 3)); } writer.forceMerge(1); writer.close(); final TestReopen test = new TestReopen() { @Override protected void modifyIndex(int i) throws IOException { if (i % 3 == 0) { IndexReader modifier = IndexReader.open(dir, false); modifier.setNorm(i, "field1", 50); modifier.close(); } else if (i % 3 == 1) { IndexReader modifier = IndexReader.open(dir, false); modifier.deleteDocument(i % modifier.maxDoc()); modifier.close(); } else { IndexWriter modifier = new IndexWriter(dir, new IndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))); modifier.addDocument(createDocument(n + i, 6)); modifier.close(); } } @Override protected IndexReader openReader() throws IOException { return IndexReader.open(dir, false); } }; final List readers = Collections.synchronizedList(new ArrayList()); IndexReader firstReader = IndexReader.open(dir, false); IndexReader reader = firstReader; final Random rnd = random; ReaderThread[] threads = new ReaderThread[n]; final Set readersToClose = Collections.synchronizedSet(new HashSet()); for (int i = 0; i < n; i++) { if (i % 2 == 0) { IndexReader refreshed = IndexReader.openIfChanged(reader); if (refreshed != null) { readersToClose.add(reader); reader = refreshed; } } final IndexReader r = reader; final int index = i; ReaderThreadTask task; if (i < 4 || (i >=10 && i < 14) || i > 18) { task = new ReaderThreadTask() { @Override public void run() throws Exception { while (!stopped) { if (index % 2 == 0) { // refresh reader synchronized ReaderCouple c = (refreshReader(r, test, index, true)); readersToClose.add(c.newReader); readersToClose.add(c.refreshedReader); readers.add(c); // prevent too many readers break; } else { // not synchronized IndexReader refreshed = IndexReader.openIfChanged(r); if (refreshed == null) { refreshed = r; } IndexSearcher searcher = newSearcher(refreshed); ScoreDoc[] hits = searcher.search( new TermQuery(new Term("field1", "a" + rnd.nextInt(refreshed.maxDoc()))), null, 1000).scoreDocs; if (hits.length > 0) { searcher.doc(hits[0].doc); } searcher.close(); if (refreshed != r) { refreshed.close(); } } synchronized(this) { wait(_TestUtil.nextInt(random, 1, 100)); } } } }; } else { task = new ReaderThreadTask() { @Override public void run() throws Exception { while (!stopped) { int numReaders = readers.size(); if (numReaders > 0) { ReaderCouple c = readers.get(rnd.nextInt(numReaders)); TestIndexReader.assertIndexEquals(c.newReader, c.refreshedReader); } synchronized(this) { wait(_TestUtil.nextInt(random, 1, 100)); } } } }; } threads[i] = new ReaderThread(task); threads[i].start(); } synchronized(this) { wait(1000); } for (int i = 0; i < n; i++) { if (threads[i] != null) { threads[i].stopThread(); } } for (int i = 0; i < n; i++) { if (threads[i] != null) { threads[i].join(); if (threads[i].error != null) { String msg = "Error occurred in thread " + threads[i].getName() + ":\n" + threads[i].error.getMessage(); fail(msg); } } } for (final IndexReader readerToClose : readersToClose) { readerToClose.close(); } firstReader.close(); reader.close(); for (final IndexReader readerToClose : readersToClose) { assertReaderClosed(readerToClose, true, true); } assertReaderClosed(reader, true, true); assertReaderClosed(firstReader, true, true); dir.close(); } private static class ReaderCouple { ReaderCouple(IndexReader r1, IndexReader r2) { newReader = r1; refreshedReader = r2; } IndexReader newReader; IndexReader refreshedReader; } private abstract static class ReaderThreadTask { protected volatile boolean stopped; public void stop() { this.stopped = true; } public abstract void run() throws Exception; } private static class ReaderThread extends Thread { private ReaderThreadTask task; private Throwable error; ReaderThread(ReaderThreadTask task) { this.task = task; } public void stopThread() { this.task.stop(); } @Override public void run() { try { this.task.run(); } catch (Throwable r) { r.printStackTrace(System.out); this.error = r; } } } private Object createReaderMutex = new Object(); private ReaderCouple refreshReader(IndexReader reader, boolean hasChanges) throws IOException { return refreshReader(reader, null, -1, hasChanges); } ReaderCouple refreshReader(IndexReader reader, TestReopen test, int modify, boolean hasChanges) throws IOException { synchronized (createReaderMutex) { IndexReader r = null; if (test != null) { test.modifyIndex(modify); r = test.openReader(); } IndexReader refreshed = null; try { refreshed = IndexReader.openIfChanged(reader); if (refreshed == null) { refreshed = reader; } } finally { if (refreshed == null && r != null) { // Hit exception -- close opened reader r.close(); } } if (hasChanges) { if (refreshed == reader) { fail("No new IndexReader instance created during refresh."); } } else { if (refreshed != reader) { fail("New IndexReader instance created during refresh even though index had no changes."); } } return new ReaderCouple(r, refreshed); } } public static void createIndex(Random random, Directory dir, boolean multiSegment) throws IOException { IndexWriter.unlock(dir); IndexWriter w = new IndexWriter(dir, LuceneTestCase.newIndexWriterConfig(random, TEST_VERSION_CURRENT, new MockAnalyzer(random)) .setMergePolicy(new LogDocMergePolicy())); for (int i = 0; i < 100; i++) { w.addDocument(createDocument(i, 4)); if (multiSegment && (i % 10) == 0) { w.commit(); } } if (!multiSegment) { w.forceMerge(1); } w.close(); IndexReader r = IndexReader.open(dir, false); if (multiSegment) { assertTrue(r.getSequentialSubReaders().length > 1); } else { assertTrue(r.getSequentialSubReaders().length == 1); } r.close(); } public static Document createDocument(int n, int numFields) { StringBuilder sb = new StringBuilder(); Document doc = new Document(); sb.append("a"); sb.append(n); doc.add(new Field("field1", sb.toString(), Store.YES, Index.ANALYZED)); doc.add(new Field("fielda", sb.toString(), Store.YES, Index.NOT_ANALYZED_NO_NORMS)); doc.add(new Field("fieldb", sb.toString(), Store.YES, Index.NO)); sb.append(" b"); sb.append(n); for (int i = 1; i < numFields; i++) { doc.add(new Field("field" + (i+1), sb.toString(), Store.YES, Index.ANALYZED)); } return doc; } static void modifyIndex(int i, Directory dir) throws IOException { switch (i) { case 0: { if (VERBOSE) { System.out.println("TEST: modify index"); } IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); w.setInfoStream(VERBOSE ? System.out : null); w.deleteDocuments(new Term("field2", "a11")); w.deleteDocuments(new Term("field2", "b30")); w.close(); break; } case 1: { IndexReader reader = IndexReader.open(dir, false); reader.setNorm(4, "field1", 123); reader.setNorm(44, "field2", 222); reader.setNorm(44, "field4", 22); reader.close(); break; } case 2: { IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); w.forceMerge(1); w.close(); break; } case 3: { IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); w.addDocument(createDocument(101, 4)); w.forceMerge(1); w.addDocument(createDocument(102, 4)); w.addDocument(createDocument(103, 4)); w.close(); break; } case 4: { IndexReader reader = IndexReader.open(dir, false); reader.setNorm(5, "field1", 123); reader.setNorm(55, "field2", 222); reader.close(); break; } case 5: { IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); w.addDocument(createDocument(101, 4)); w.close(); break; } } } private void assertReaderClosed(IndexReader reader, boolean checkSubReaders, boolean checkNormsClosed) { assertEquals(0, reader.getRefCount()); if (checkNormsClosed && reader instanceof SegmentReader) { assertTrue(((SegmentReader) reader).normsClosed()); } if (checkSubReaders) { if (reader instanceof DirectoryReader) { IndexReader[] subReaders = reader.getSequentialSubReaders(); for (int i = 0; i < subReaders.length; i++) { assertReaderClosed(subReaders[i], checkSubReaders, checkNormsClosed); } } if (reader instanceof MultiReader) { IndexReader[] subReaders = reader.getSequentialSubReaders(); for (int i = 0; i < subReaders.length; i++) { assertReaderClosed(subReaders[i], checkSubReaders, checkNormsClosed); } } if (reader instanceof ParallelReader) { IndexReader[] subReaders = ((ParallelReader) reader).getSubReaders(); for (int i = 0; i < subReaders.length; i++) { assertReaderClosed(subReaders[i], checkSubReaders, checkNormsClosed); } } } } /* private void assertReaderOpen(IndexReader reader) { reader.ensureOpen(); if (reader instanceof DirectoryReader) { IndexReader[] subReaders = reader.getSequentialSubReaders(); for (int i = 0; i < subReaders.length; i++) { assertReaderOpen(subReaders[i]); } } } */ private void assertRefCountEquals(int refCount, IndexReader reader) { assertEquals("Reader has wrong refCount value.", refCount, reader.getRefCount()); } private abstract static class TestReopen { protected abstract IndexReader openReader() throws IOException; protected abstract void modifyIndex(int i) throws IOException; } public void testCloseOrig() throws Throwable { Directory dir = newDirectory(); createIndex(random, dir, false); IndexReader r1 = IndexReader.open(dir, false); IndexReader r2 = IndexReader.open(dir, false); r2.deleteDocument(0); r2.close(); IndexReader r3 = IndexReader.openIfChanged(r1); assertNotNull(r3); assertTrue(r1 != r3); r1.close(); try { r1.document(2); fail("did not hit exception"); } catch (AlreadyClosedException ace) { // expected } r3.close(); dir.close(); } public void testDeletes() throws Throwable { Directory dir = newDirectory(); createIndex(random, dir, false); // Create an index with a bunch of docs (1 segment) modifyIndex(0, dir); // Get delete bitVector on 1st segment modifyIndex(5, dir); // Add a doc (2 segments) IndexReader r1 = IndexReader.open(dir, false); // MSR modifyIndex(5, dir); // Add another doc (3 segments) IndexReader r2 = IndexReader.openIfChanged(r1); // MSR assertNotNull(r2); assertNull(IndexReader.openIfChanged(r2)); assertTrue(r1 != r2); SegmentReader sr1 = (SegmentReader) r1.getSequentialSubReaders()[0]; // Get SRs for the first segment from original SegmentReader sr2 = (SegmentReader) r2.getSequentialSubReaders()[0]; // and reopened IRs // At this point they share the same BitVector assertTrue(sr1.deletedDocs==sr2.deletedDocs); r2.deleteDocument(0); // r1 should not see the delete assertFalse(r1.isDeleted(0)); // Now r2 should have made a private copy of deleted docs: assertTrue(sr1.deletedDocs!=sr2.deletedDocs); r1.close(); r2.close(); dir.close(); } public void testDeletes2() throws Throwable { Directory dir = newDirectory(); createIndex(random, dir, false); // Get delete bitVector modifyIndex(0, dir); IndexReader r1 = IndexReader.open(dir, false); // Add doc: modifyIndex(5, dir); IndexReader r2 = IndexReader.openIfChanged(r1); assertNotNull(r2); assertTrue(r1 != r2); IndexReader[] rs2 = r2.getSequentialSubReaders(); SegmentReader sr1 = SegmentReader.getOnlySegmentReader(r1); SegmentReader sr2 = (SegmentReader) rs2[0]; // At this point they share the same BitVector assertTrue(sr1.deletedDocs==sr2.deletedDocs); final BitVector delDocs = sr1.deletedDocs; r1.close(); r2.deleteDocument(0); assertTrue(delDocs==sr2.deletedDocs); r2.close(); dir.close(); } private static class KeepAllCommits implements IndexDeletionPolicy { public void onInit(List commits) { } public void onCommit(List commits) { } } public void testReopenOnCommit() throws Throwable { Directory dir = newDirectory(); IndexWriter writer = new IndexWriter( dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). setIndexDeletionPolicy(new KeepAllCommits()). setMaxBufferedDocs(-1). setMergePolicy(newLogMergePolicy(10)) ); for(int i=0;i<4;i++) { Document doc = new Document(); doc.add(newField("id", ""+i, Field.Store.NO, Field.Index.NOT_ANALYZED)); writer.addDocument(doc); Map data = new HashMap(); data.put("index", i+""); writer.commit(data); } for(int i=0;i<4;i++) { writer.deleteDocuments(new Term("id", ""+i)); Map data = new HashMap(); data.put("index", (4+i)+""); writer.commit(data); } writer.close(); IndexReader r = IndexReader.open(dir, false); assertEquals(0, r.numDocs()); Collection commits = IndexReader.listCommits(dir); for (final IndexCommit commit : commits) { IndexReader r2 = IndexReader.openIfChanged(r, commit); assertNotNull(r2); assertTrue(r2 != r); // Reader should be readOnly try { r2.deleteDocument(0); fail("no exception hit"); } catch (UnsupportedOperationException uoe) { // expected } final Map s = commit.getUserData(); final int v; if (s.size() == 0) { // First commit created by IW v = -1; } else { v = Integer.parseInt(s.get("index")); } if (v < 4) { assertEquals(1+v, r2.numDocs()); } else { assertEquals(7-v, r2.numDocs()); } r.close(); r = r2; } r.close(); dir.close(); } // LUCENE-1579: Make sure all SegmentReaders are new when // reopen switches readOnly public void testReopenChangeReadonly() throws Exception { Directory dir = newDirectory(); IndexWriter writer = new IndexWriter( dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). setMaxBufferedDocs(-1). setMergePolicy(newLogMergePolicy(10)) ); Document doc = new Document(); doc.add(newField("number", "17", Field.Store.NO, Field.Index.NOT_ANALYZED)); writer.addDocument(doc); writer.commit(); // Open reader1 IndexReader r = IndexReader.open(dir, false); assertTrue(r instanceof DirectoryReader); IndexReader r1 = SegmentReader.getOnlySegmentReader(r); final int[] ints = FieldCache.DEFAULT.getInts(r1, "number"); assertEquals(1, ints.length); assertEquals(17, ints[0]); // Reopen to readonly w/ no chnages IndexReader r3 = IndexReader.openIfChanged(r, true); assertNotNull(r3); assertTrue(r3 instanceof ReadOnlyDirectoryReader); r3.close(); // Add new segment writer.addDocument(doc); writer.commit(); // Reopen reader1 --> reader2 IndexReader r2 = IndexReader.openIfChanged(r, true); assertNotNull(r2); r.close(); assertTrue(r2 instanceof ReadOnlyDirectoryReader); IndexReader[] subs = r2.getSequentialSubReaders(); final int[] ints2 = FieldCache.DEFAULT.getInts(subs[0], "number"); r2.close(); assertTrue(subs[0] instanceof ReadOnlySegmentReader); assertTrue(subs[1] instanceof ReadOnlySegmentReader); assertTrue(ints == ints2); writer.close(); dir.close(); } }