1 package org.apache.lucene.index;
4 * Licensed to the Apache Software Foundation (ASF) under one or more
5 * contributor license agreements. See the NOTICE file distributed with
6 * this work for additional information regarding copyright ownership.
7 * The ASF licenses this file to You under the Apache License, Version 2.0
8 * (the "License"); you may not use this file except in compliance with
9 * the License. You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
20 import org.apache.lucene.index.SegmentNorms;
21 import org.apache.lucene.search.Similarity;
22 import org.apache.lucene.analysis.MockAnalyzer;
23 import org.apache.lucene.document.Document;
24 import org.apache.lucene.document.Field;
25 import org.apache.lucene.store.Directory;
26 import org.apache.lucene.store.LockObtainFailedException;
27 import org.apache.lucene.util.LuceneTestCase;
30 * Tests cloning multiple types of readers, modifying the deletedDocs and norms
31 * and verifies copy on write semantics of the deletedDocs and norms is
32 * implemented properly
34 public class TestIndexReaderClone extends LuceneTestCase {
36 public void testCloneReadOnlySegmentReader() throws Exception {
37 final Directory dir1 = newDirectory();
39 TestIndexReaderReopen.createIndex(random, dir1, false);
40 IndexReader reader = IndexReader.open(dir1, false);
41 IndexReader readOnlyReader = reader.clone(true);
42 if (!isReadOnly(readOnlyReader)) {
43 fail("reader isn't read only");
45 if (deleteWorked(1, readOnlyReader)) {
46 fail("deleting from the original should not have worked");
49 readOnlyReader.close();
53 // open non-readOnly reader1, clone to non-readOnly
54 // reader2, make sure we can change reader2
55 public void testCloneNoChangesStillReadOnly() throws Exception {
56 final Directory dir1 = newDirectory();
58 TestIndexReaderReopen.createIndex(random, dir1, true);
59 IndexReader r1 = IndexReader.open(dir1, false);
60 IndexReader r2 = r1.clone(false);
61 if (!deleteWorked(1, r2)) {
62 fail("deleting from the cloned should have worked");
69 // open non-readOnly reader1, clone to non-readOnly
70 // reader2, make sure we can change reader1
71 public void testCloneWriteToOrig() throws Exception {
72 final Directory dir1 = newDirectory();
74 TestIndexReaderReopen.createIndex(random, dir1, true);
75 IndexReader r1 = IndexReader.open(dir1, false);
76 IndexReader r2 = r1.clone(false);
77 if (!deleteWorked(1, r1)) {
78 fail("deleting from the original should have worked");
85 // open non-readOnly reader1, clone to non-readOnly
86 // reader2, make sure we can change reader2
87 public void testCloneWriteToClone() throws Exception {
88 final Directory dir1 = newDirectory();
90 TestIndexReaderReopen.createIndex(random, dir1, true);
91 IndexReader r1 = IndexReader.open(dir1, false);
92 IndexReader r2 = r1.clone(false);
93 if (!deleteWorked(1, r2)) {
94 fail("deleting from the original should have worked");
96 // should fail because reader1 holds the write lock
97 assertTrue("first reader should not be able to delete", !deleteWorked(1, r1));
99 // should fail because we are now stale (reader1
100 // committed changes)
101 assertTrue("first reader should not be able to delete", !deleteWorked(1, r1));
107 // create single-segment index, open non-readOnly
108 // SegmentReader, add docs, reopen to multireader, then do
110 public void testReopenSegmentReaderToMultiReader() throws Exception {
111 final Directory dir1 = newDirectory();
113 TestIndexReaderReopen.createIndex(random, dir1, false);
114 IndexReader reader1 = IndexReader.open(dir1, false);
116 TestIndexReaderReopen.modifyIndex(5, dir1);
118 IndexReader reader2 = reader1.reopen();
119 assertTrue(reader1 != reader2);
121 assertTrue(deleteWorked(1, reader2));
127 // open non-readOnly reader1, clone to readOnly reader2
128 public void testCloneWriteableToReadOnly() throws Exception {
129 final Directory dir1 = newDirectory();
131 TestIndexReaderReopen.createIndex(random, dir1, true);
132 IndexReader reader = IndexReader.open(dir1, false);
133 IndexReader readOnlyReader = reader.clone(true);
134 if (!isReadOnly(readOnlyReader)) {
135 fail("reader isn't read only");
137 if (deleteWorked(1, readOnlyReader)) {
138 fail("deleting from the original should not have worked");
140 // this readonly reader shouldn't have a write lock
141 if (readOnlyReader.hasChanges) {
142 fail("readOnlyReader has a write lock");
145 readOnlyReader.close();
149 // open non-readOnly reader1, reopen to readOnly reader2
150 public void testReopenWriteableToReadOnly() throws Exception {
151 final Directory dir1 = newDirectory();
153 TestIndexReaderReopen.createIndex(random, dir1, true);
154 IndexReader reader = IndexReader.open(dir1, false);
155 final int docCount = reader.numDocs();
156 assertTrue(deleteWorked(1, reader));
157 assertEquals(docCount-1, reader.numDocs());
159 IndexReader readOnlyReader = reader.reopen(true);
160 if (!isReadOnly(readOnlyReader)) {
161 fail("reader isn't read only");
163 assertFalse(deleteWorked(1, readOnlyReader));
164 assertEquals(docCount-1, readOnlyReader.numDocs());
166 readOnlyReader.close();
170 // open readOnly reader1, clone to non-readOnly reader2
171 public void testCloneReadOnlyToWriteable() throws Exception {
172 final Directory dir1 = newDirectory();
174 TestIndexReaderReopen.createIndex(random, dir1, true);
175 IndexReader reader1 = IndexReader.open(dir1, true);
177 IndexReader reader2 = reader1.clone(false);
178 if (isReadOnly(reader2)) {
179 fail("reader should not be read only");
181 assertFalse("deleting from the original reader should not have worked", deleteWorked(1, reader1));
182 // this readonly reader shouldn't yet have a write lock
183 if (reader2.hasChanges) {
184 fail("cloned reader should not have write lock");
186 assertTrue("deleting from the cloned reader should have worked", deleteWorked(1, reader2));
192 // open non-readOnly reader1 on multi-segment index, then
193 // optimize the index, then clone to readOnly reader2
194 public void testReadOnlyCloneAfterOptimize() throws Exception {
195 final Directory dir1 = newDirectory();
197 TestIndexReaderReopen.createIndex(random, dir1, true);
198 IndexReader reader1 = IndexReader.open(dir1, false);
199 IndexWriter w = new IndexWriter(dir1, newIndexWriterConfig(
200 TEST_VERSION_CURRENT, new MockAnalyzer(random)));
203 IndexReader reader2 = reader1.clone(true);
204 assertTrue(isReadOnly(reader2));
210 private static boolean deleteWorked(int doc, IndexReader r) {
211 boolean exception = false;
213 // trying to delete from the original reader should throw an exception
214 r.deleteDocument(doc);
215 } catch (Exception ex) {
221 public void testCloneReadOnlyDirectoryReader() throws Exception {
222 final Directory dir1 = newDirectory();
224 TestIndexReaderReopen.createIndex(random, dir1, true);
225 IndexReader reader = IndexReader.open(dir1, false);
226 IndexReader readOnlyReader = reader.clone(true);
227 if (!isReadOnly(readOnlyReader)) {
228 fail("reader isn't read only");
231 readOnlyReader.close();
235 public static boolean isReadOnly(IndexReader r) {
236 if (r instanceof ReadOnlySegmentReader
237 || r instanceof ReadOnlyDirectoryReader)
242 public void testParallelReader() throws Exception {
243 final Directory dir1 = newDirectory();
244 TestIndexReaderReopen.createIndex(random, dir1, true);
245 final Directory dir2 = newDirectory();
246 TestIndexReaderReopen.createIndex(random, dir2, true);
247 IndexReader r1 = IndexReader.open(dir1, false);
248 IndexReader r2 = IndexReader.open(dir2, false);
250 ParallelReader pr1 = new ParallelReader();
254 performDefaultTests(pr1);
261 * 1. Get a norm from the original reader 2. Clone the original reader 3.
262 * Delete a document and set the norm of the cloned reader 4. Verify the norms
263 * are not the same on each reader 5. Verify the doc deleted is only in the
264 * cloned reader 6. Try to delete a document in the original reader, an
265 * exception should be thrown
267 * @param r1 IndexReader to perform tests on
270 private void performDefaultTests(IndexReader r1) throws Exception {
271 float norm1 = Similarity.getDefault().decodeNormValue(r1.norms("field1")[4]);
273 IndexReader pr1Clone = (IndexReader) r1.clone();
274 pr1Clone.deleteDocument(10);
275 pr1Clone.setNorm(4, "field1", 0.5f);
276 assertTrue(Similarity.getDefault().decodeNormValue(r1.norms("field1")[4]) == norm1);
277 assertTrue(Similarity.getDefault().decodeNormValue(pr1Clone.norms("field1")[4]) != norm1);
279 assertTrue(!r1.isDeleted(10));
280 assertTrue(pr1Clone.isDeleted(10));
282 // try to update the original reader, which should throw an exception
284 r1.deleteDocument(11);
285 fail("Tried to delete doc 11 and an exception should have been thrown");
286 } catch (Exception exception) {
292 public void testMixedReaders() throws Exception {
293 final Directory dir1 = newDirectory();
294 TestIndexReaderReopen.createIndex(random, dir1, true);
295 final Directory dir2 = newDirectory();
296 TestIndexReaderReopen.createIndex(random, dir2, true);
297 IndexReader r1 = IndexReader.open(dir1, false);
298 IndexReader r2 = IndexReader.open(dir2, false);
300 MultiReader multiReader = new MultiReader(new IndexReader[] { r1, r2 });
301 performDefaultTests(multiReader);
307 public void testSegmentReaderUndeleteall() throws Exception {
308 final Directory dir1 = newDirectory();
309 TestIndexReaderReopen.createIndex(random, dir1, false);
310 SegmentReader origSegmentReader = SegmentReader.getOnlySegmentReader(dir1);
311 origSegmentReader.deleteDocument(10);
312 assertDelDocsRefCountEquals(1, origSegmentReader);
313 origSegmentReader.undeleteAll();
314 assertNull(origSegmentReader.deletedDocsRef);
315 origSegmentReader.close();
316 // need to test norms?
320 public void testSegmentReaderCloseReferencing() throws Exception {
321 final Directory dir1 = newDirectory();
322 TestIndexReaderReopen.createIndex(random, dir1, false);
323 SegmentReader origSegmentReader = SegmentReader.getOnlySegmentReader(dir1);
324 origSegmentReader.deleteDocument(1);
325 origSegmentReader.setNorm(4, "field1", 0.5f);
327 SegmentReader clonedSegmentReader = (SegmentReader) origSegmentReader
329 assertDelDocsRefCountEquals(2, origSegmentReader);
330 origSegmentReader.close();
331 assertDelDocsRefCountEquals(1, origSegmentReader);
332 // check the norm refs
333 SegmentNorms norm = clonedSegmentReader.norms.get("field1");
334 assertEquals(1, norm.bytesRef().get());
335 clonedSegmentReader.close();
339 public void testSegmentReaderDelDocsReferenceCounting() throws Exception {
340 final Directory dir1 = newDirectory();
341 TestIndexReaderReopen.createIndex(random, dir1, false);
343 IndexReader origReader = IndexReader.open(dir1, false);
344 SegmentReader origSegmentReader = SegmentReader.getOnlySegmentReader(origReader);
345 // deletedDocsRef should be null because nothing has updated yet
346 assertNull(origSegmentReader.deletedDocsRef);
348 // we deleted a document, so there is now a deletedDocs bitvector and a
350 origReader.deleteDocument(1);
351 assertDelDocsRefCountEquals(1, origSegmentReader);
353 // the cloned segmentreader should have 2 references, 1 to itself, and 1 to
354 // the original segmentreader
355 IndexReader clonedReader = (IndexReader) origReader.clone();
356 SegmentReader clonedSegmentReader = SegmentReader.getOnlySegmentReader(clonedReader);
357 assertDelDocsRefCountEquals(2, origSegmentReader);
358 // deleting a document creates a new deletedDocs bitvector, the refs goes to
360 clonedReader.deleteDocument(2);
361 assertDelDocsRefCountEquals(1, origSegmentReader);
362 assertDelDocsRefCountEquals(1, clonedSegmentReader);
364 // make sure the deletedocs objects are different (copy
366 assertTrue(origSegmentReader.deletedDocs != clonedSegmentReader.deletedDocs);
368 assertDocDeleted(origSegmentReader, clonedSegmentReader, 1);
369 assertTrue(!origSegmentReader.isDeleted(2)); // doc 2 should not be deleted
370 // in original segmentreader
371 assertTrue(clonedSegmentReader.isDeleted(2)); // doc 2 should be deleted in
372 // cloned segmentreader
374 // deleting a doc from the original segmentreader should throw an exception
376 origReader.deleteDocument(4);
377 fail("expected exception");
378 } catch (LockObtainFailedException lbfe) {
383 // try closing the original segment reader to see if it affects the
384 // clonedSegmentReader
385 clonedReader.deleteDocument(3);
386 clonedReader.flush();
387 assertDelDocsRefCountEquals(1, clonedSegmentReader);
389 // test a reopened reader
390 IndexReader reopenedReader = clonedReader.reopen();
391 IndexReader cloneReader2 = (IndexReader) reopenedReader.clone();
392 SegmentReader cloneSegmentReader2 = SegmentReader.getOnlySegmentReader(cloneReader2);
393 assertDelDocsRefCountEquals(2, cloneSegmentReader2);
394 clonedReader.close();
395 reopenedReader.close();
396 cloneReader2.close();
402 public void testCloneWithDeletes() throws Throwable {
403 final Directory dir1 = newDirectory();
404 TestIndexReaderReopen.createIndex(random, dir1, false);
405 IndexReader origReader = IndexReader.open(dir1, false);
406 origReader.deleteDocument(1);
408 IndexReader clonedReader = (IndexReader) origReader.clone();
410 clonedReader.close();
412 IndexReader r = IndexReader.open(dir1, false);
413 assertTrue(r.isDeleted(1));
419 public void testCloneWithSetNorm() throws Throwable {
420 final Directory dir1 = newDirectory();
421 TestIndexReaderReopen.createIndex(random, dir1, false);
422 IndexReader orig = IndexReader.open(dir1, false);
423 orig.setNorm(1, "field1", 17.0f);
424 final byte encoded = Similarity.getDefault().encodeNormValue(17.0f);
425 assertEquals(encoded, orig.norms("field1")[1]);
427 // the cloned segmentreader should have 2 references, 1 to itself, and 1 to
428 // the original segmentreader
429 IndexReader clonedReader = (IndexReader) orig.clone();
431 clonedReader.close();
433 IndexReader r = IndexReader.open(dir1, false);
434 assertEquals(encoded, r.norms("field1")[1]);
439 private void assertDocDeleted(SegmentReader reader, SegmentReader reader2,
441 assertEquals(reader.isDeleted(doc), reader2.isDeleted(doc));
444 private void assertDelDocsRefCountEquals(int refCount, SegmentReader reader) {
445 assertEquals(refCount, reader.deletedDocsRef.get());
448 public void testCloneSubreaders() throws Exception {
449 final Directory dir1 = newDirectory();
451 TestIndexReaderReopen.createIndex(random, dir1, true);
452 IndexReader reader = IndexReader.open(dir1, false);
453 reader.deleteDocument(1); // acquire write lock
454 IndexReader[] subs = reader.getSequentialSubReaders();
455 assert subs.length > 1;
457 IndexReader[] clones = new IndexReader[subs.length];
458 for (int x=0; x < subs.length; x++) {
459 clones[x] = (IndexReader) subs[x].clone();
462 for (int x=0; x < subs.length; x++) {
468 public void testLucene1516Bug() throws Exception {
469 final Directory dir1 = newDirectory();
470 TestIndexReaderReopen.createIndex(random, dir1, false);
471 IndexReader r1 = IndexReader.open(dir1, false);
473 IndexReader r2 = r1.clone(false);
474 r1.deleteDocument(5);
485 public void testCloseStoredFields() throws Exception {
486 final Directory dir = newDirectory();
487 IndexWriter w = new IndexWriter(
489 newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
490 setMergePolicy(newLogMergePolicy(false))
492 Document doc = new Document();
493 doc.add(newField("field", "yes it's stored", Field.Store.YES, Field.Index.ANALYZED));
496 IndexReader r1 = IndexReader.open(dir, false);
497 IndexReader r2 = r1.clone(false);