1 package org.apache.lucene.index;
4 * Licensed to the Apache Software Foundation (ASF) under one or more
5 * contributor license agreements. See the NOTICE file distributed with
6 * this work for additional information regarding copyright ownership.
7 * The ASF licenses this file to You under the Apache License, Version 2.0
8 * (the "License"); you may not use this file except in compliance with
9 * the License. You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
20 import java.io.IOException;
22 import org.apache.lucene.analysis.MockAnalyzer;
23 import org.apache.lucene.document.Document;
24 import org.apache.lucene.document.Field;
25 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
26 import org.apache.lucene.store.Directory;
27 import org.apache.lucene.store.MockDirectoryWrapper;
28 import org.apache.lucene.util.LuceneTestCase;
30 import static org.apache.lucene.index.TestIndexReader.addDoc;
31 import static org.apache.lucene.index.TestIndexReader.addDocumentWithFields;
32 import static org.apache.lucene.index.TestIndexReader.assertTermDocsCount;
33 import static org.apache.lucene.index.TestIndexReader.createDocument;
35 public class TestIndexReaderDelete extends LuceneTestCase {
36 private void deleteReaderReaderConflict(boolean optimize) throws IOException {
37 Directory dir = newDirectory();
39 Term searchTerm1 = new Term("content", "aaa");
40 Term searchTerm2 = new Term("content", "bbb");
41 Term searchTerm3 = new Term("content", "ccc");
43 // add 100 documents with term : aaa
44 // add 100 documents with term : bbb
45 // add 100 documents with term : ccc
46 IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE));
47 for (int i = 0; i < 100; i++) {
48 addDoc(writer, searchTerm1.text());
49 addDoc(writer, searchTerm2.text());
50 addDoc(writer, searchTerm3.text());
57 // Both readers get segment info as exists at this time
58 IndexReader reader1 = IndexReader.open(dir, false);
59 assertEquals("first opened", 100, reader1.docFreq(searchTerm1));
60 assertEquals("first opened", 100, reader1.docFreq(searchTerm2));
61 assertEquals("first opened", 100, reader1.docFreq(searchTerm3));
62 assertTermDocsCount("first opened", reader1, searchTerm1, 100);
63 assertTermDocsCount("first opened", reader1, searchTerm2, 100);
64 assertTermDocsCount("first opened", reader1, searchTerm3, 100);
66 IndexReader reader2 = IndexReader.open(dir, false);
67 assertEquals("first opened", 100, reader2.docFreq(searchTerm1));
68 assertEquals("first opened", 100, reader2.docFreq(searchTerm2));
69 assertEquals("first opened", 100, reader2.docFreq(searchTerm3));
70 assertTermDocsCount("first opened", reader2, searchTerm1, 100);
71 assertTermDocsCount("first opened", reader2, searchTerm2, 100);
72 assertTermDocsCount("first opened", reader2, searchTerm3, 100);
74 // DELETE DOCS FROM READER 2 and CLOSE IT
75 // delete documents containing term: aaa
76 // when the reader is closed, the segment info is updated and
77 // the first reader is now stale
78 reader2.deleteDocuments(searchTerm1);
79 assertEquals("after delete 1", 100, reader2.docFreq(searchTerm1));
80 assertEquals("after delete 1", 100, reader2.docFreq(searchTerm2));
81 assertEquals("after delete 1", 100, reader2.docFreq(searchTerm3));
82 assertTermDocsCount("after delete 1", reader2, searchTerm1, 0);
83 assertTermDocsCount("after delete 1", reader2, searchTerm2, 100);
84 assertTermDocsCount("after delete 1", reader2, searchTerm3, 100);
87 // Make sure reader 1 is unchanged since it was open earlier
88 assertEquals("after delete 1", 100, reader1.docFreq(searchTerm1));
89 assertEquals("after delete 1", 100, reader1.docFreq(searchTerm2));
90 assertEquals("after delete 1", 100, reader1.docFreq(searchTerm3));
91 assertTermDocsCount("after delete 1", reader1, searchTerm1, 100);
92 assertTermDocsCount("after delete 1", reader1, searchTerm2, 100);
93 assertTermDocsCount("after delete 1", reader1, searchTerm3, 100);
96 // ATTEMPT TO DELETE FROM STALE READER
97 // delete documents containing term: bbb
99 reader1.deleteDocuments(searchTerm2);
100 fail("Delete allowed from a stale index reader");
101 } catch (IOException e) {
105 // RECREATE READER AND TRY AGAIN
107 reader1 = IndexReader.open(dir, false);
108 assertEquals("reopened", 100, reader1.docFreq(searchTerm1));
109 assertEquals("reopened", 100, reader1.docFreq(searchTerm2));
110 assertEquals("reopened", 100, reader1.docFreq(searchTerm3));
111 assertTermDocsCount("reopened", reader1, searchTerm1, 0);
112 assertTermDocsCount("reopened", reader1, searchTerm2, 100);
113 assertTermDocsCount("reopened", reader1, searchTerm3, 100);
115 reader1.deleteDocuments(searchTerm2);
116 assertEquals("deleted 2", 100, reader1.docFreq(searchTerm1));
117 assertEquals("deleted 2", 100, reader1.docFreq(searchTerm2));
118 assertEquals("deleted 2", 100, reader1.docFreq(searchTerm3));
119 assertTermDocsCount("deleted 2", reader1, searchTerm1, 0);
120 assertTermDocsCount("deleted 2", reader1, searchTerm2, 0);
121 assertTermDocsCount("deleted 2", reader1, searchTerm3, 100);
124 // Open another reader to confirm that everything is deleted
125 reader2 = IndexReader.open(dir, false);
126 assertTermDocsCount("reopened 2", reader2, searchTerm1, 0);
127 assertTermDocsCount("reopened 2", reader2, searchTerm2, 0);
128 assertTermDocsCount("reopened 2", reader2, searchTerm3, 100);
134 private void deleteReaderWriterConflict(boolean optimize) throws IOException {
135 //Directory dir = new RAMDirectory();
136 Directory dir = newDirectory();
138 Term searchTerm = new Term("content", "aaa");
139 Term searchTerm2 = new Term("content", "bbb");
141 // add 100 documents with term : aaa
142 IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE));
143 for (int i = 0; i < 100; i++) {
144 addDoc(writer, searchTerm.text());
148 // OPEN READER AT THIS POINT - this should fix the view of the
149 // index at the point of having 100 "aaa" documents and 0 "bbb"
150 IndexReader reader = IndexReader.open(dir, false);
151 assertEquals("first docFreq", 100, reader.docFreq(searchTerm));
152 assertEquals("first docFreq", 0, reader.docFreq(searchTerm2));
153 assertTermDocsCount("first reader", reader, searchTerm, 100);
154 assertTermDocsCount("first reader", reader, searchTerm2, 0);
156 // add 100 documents with term : bbb
157 writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND));
158 for (int i = 0; i < 100; i++) {
159 addDoc(writer, searchTerm2.text());
162 // REQUEST OPTIMIZATION
163 // This causes a new segment to become current for all subsequent
164 // searchers. Because of this, deletions made via a previously open
165 // reader, which would be applied to that reader's segment, are lost
166 // for subsequent searchers/readers
171 // The reader should not see the new data
172 assertEquals("first docFreq", 100, reader.docFreq(searchTerm));
173 assertEquals("first docFreq", 0, reader.docFreq(searchTerm2));
174 assertTermDocsCount("first reader", reader, searchTerm, 100);
175 assertTermDocsCount("first reader", reader, searchTerm2, 0);
178 // DELETE DOCUMENTS CONTAINING TERM: aaa
179 // NOTE: the reader was created when only "aaa" documents were in
182 deleted = reader.deleteDocuments(searchTerm);
183 fail("Delete allowed on an index reader with stale segment information");
184 } catch (StaleReaderException e) {
188 // Re-open index reader and try again. This time it should see
191 reader = IndexReader.open(dir, false);
192 assertEquals("first docFreq", 100, reader.docFreq(searchTerm));
193 assertEquals("first docFreq", 100, reader.docFreq(searchTerm2));
194 assertTermDocsCount("first reader", reader, searchTerm, 100);
195 assertTermDocsCount("first reader", reader, searchTerm2, 100);
197 deleted = reader.deleteDocuments(searchTerm);
198 assertEquals("deleted count", 100, deleted);
199 assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm));
200 assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm2));
201 assertTermDocsCount("deleted termDocs", reader, searchTerm, 0);
202 assertTermDocsCount("deleted termDocs", reader, searchTerm2, 100);
205 // CREATE A NEW READER and re-test
206 reader = IndexReader.open(dir, false);
207 assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm2));
208 assertTermDocsCount("deleted termDocs", reader, searchTerm, 0);
209 assertTermDocsCount("deleted termDocs", reader, searchTerm2, 100);
214 public void testBasicDelete() throws IOException {
215 Directory dir = newDirectory();
217 IndexWriter writer = null;
218 IndexReader reader = null;
219 Term searchTerm = new Term("content", "aaa");
221 // add 100 documents with term : aaa
222 writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
223 writer.setInfoStream(VERBOSE ? System.out : null);
224 for (int i = 0; i < 100; i++) {
225 addDoc(writer, searchTerm.text());
229 // OPEN READER AT THIS POINT - this should fix the view of the
230 // index at the point of having 100 "aaa" documents and 0 "bbb"
231 reader = IndexReader.open(dir, false);
232 assertEquals("first docFreq", 100, reader.docFreq(searchTerm));
233 assertTermDocsCount("first reader", reader, searchTerm, 100);
236 // DELETE DOCUMENTS CONTAINING TERM: aaa
238 reader = IndexReader.open(dir, false);
239 deleted = reader.deleteDocuments(searchTerm);
240 assertEquals("deleted count", 100, deleted);
241 assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm));
242 assertTermDocsCount("deleted termDocs", reader, searchTerm, 0);
244 // open a 2nd reader to make sure first reader can
245 // commit its changes (.del) while second reader
247 IndexReader reader2 = IndexReader.open(dir, false);
250 // CREATE A NEW READER and re-test
251 reader = IndexReader.open(dir, false);
252 assertEquals("deleted docFreq", 0, reader.docFreq(searchTerm));
253 assertTermDocsCount("deleted termDocs", reader, searchTerm, 0);
259 public void testDeleteReaderReaderConflictUnoptimized() throws IOException {
260 deleteReaderReaderConflict(false);
263 public void testDeleteReaderReaderConflictOptimized() throws IOException {
264 deleteReaderReaderConflict(true);
267 public void testDeleteReaderWriterConflictUnoptimized() throws IOException {
268 deleteReaderWriterConflict(false);
271 public void testDeleteReaderWriterConflictOptimized() throws IOException {
272 deleteReaderWriterConflict(true);
275 public void testMultiReaderDeletes() throws Exception {
276 Directory dir = newDirectory();
277 RandomIndexWriter w= new RandomIndexWriter(random, dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy()));
278 Document doc = new Document();
279 doc.add(newField("f", "doctor", Field.Store.NO, Field.Index.NOT_ANALYZED));
281 doc = new Document();
283 doc.add(newField("f", "who", Field.Store.NO, Field.Index.NOT_ANALYZED));
285 IndexReader r = new SlowMultiReaderWrapper(w.getReader());
288 assertFalse(r.hasDeletions());
291 r = new SlowMultiReaderWrapper(IndexReader.open(dir, false));
293 assertFalse(r.hasDeletions());
294 assertEquals(1, r.deleteDocuments(new Term("f", "doctor")));
295 assertTrue(r.hasDeletions());
296 assertTrue(r.isDeleted(0));
297 assertEquals(1, r.deleteDocuments(new Term("f", "who")));
298 assertTrue(r.isDeleted(1));
303 public void testUndeleteAll() throws IOException {
304 Directory dir = newDirectory();
305 IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
306 addDocumentWithFields(writer);
307 addDocumentWithFields(writer);
309 IndexReader reader = IndexReader.open(dir, false);
310 reader.deleteDocument(0);
311 reader.deleteDocument(1);
312 reader.undeleteAll();
314 reader = IndexReader.open(dir, false);
315 assertEquals(2, reader.numDocs()); // nothing has really been deleted thanks to undeleteAll()
320 public void testUndeleteAllAfterClose() throws IOException {
321 Directory dir = newDirectory();
322 IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
323 addDocumentWithFields(writer);
324 addDocumentWithFields(writer);
326 IndexReader reader = IndexReader.open(dir, false);
327 reader.deleteDocument(0);
329 reader = IndexReader.open(dir, false);
330 reader.undeleteAll();
331 assertEquals(2, reader.numDocs()); // nothing has really been deleted thanks to undeleteAll()
336 public void testUndeleteAllAfterCloseThenReopen() throws IOException {
337 Directory dir = newDirectory();
338 IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
339 addDocumentWithFields(writer);
340 addDocumentWithFields(writer);
342 IndexReader reader = IndexReader.open(dir, false);
343 reader.deleteDocument(0);
345 reader = IndexReader.open(dir, false);
346 reader.undeleteAll();
348 reader = IndexReader.open(dir, false);
349 assertEquals(2, reader.numDocs()); // nothing has really been deleted thanks to undeleteAll()
355 public void testIndexReaderUnDeleteAll() throws Exception {
356 MockDirectoryWrapper dir = newDirectory();
357 dir.setPreventDoubleWrite(false);
358 IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
359 TEST_VERSION_CURRENT, new MockAnalyzer(random)));
360 writer.addDocument(createDocument("a"));
361 writer.addDocument(createDocument("b"));
362 writer.addDocument(createDocument("c"));
364 IndexReader reader = IndexReader.open(dir, false);
365 reader.deleteDocuments(new Term("id", "a"));
367 reader.deleteDocuments(new Term("id", "b"));
368 reader.undeleteAll();
369 reader.deleteDocuments(new Term("id", "b"));
371 IndexReader.open(dir,true).close();