lucene-java-3.5.0/lucene/src/test/org/apache/lucene/index/TestIndexReaderDelete.java

   1 package org.apache.lucene.index;
   2
   3 /**
   4  * Licensed to the Apache Software Foundation (ASF) under one or more
   5  * contributor license agreements.  See the NOTICE file distributed with
   6  * this work for additional information regarding copyright ownership.
   7  * The ASF licenses this file to You under the Apache License, Version 2.0
   8  * (the "License"); you may not use this file except in compliance with
   9  * the License.  You may obtain a copy of the License at
  10  *
  11  *     http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  */
  19
  20 import java.io.IOException;
  21
  22 import org.apache.lucene.analysis.MockAnalyzer;
  23 import org.apache.lucene.document.Document;
  24 import org.apache.lucene.document.Field;
  25 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
  26 import org.apache.lucene.store.Directory;
  27 import org.apache.lucene.store.MockDirectoryWrapper;
  28 import org.apache.lucene.util.LuceneTestCase;
  29
  30 import static org.apache.lucene.index.TestIndexReader.addDoc;
  31 import static org.apache.lucene.index.TestIndexReader.addDocumentWithFields;
  32 import static org.apache.lucene.index.TestIndexReader.assertTermDocsCount;
  33 import static org.apache.lucene.index.TestIndexReader.createDocument;
  34
  35 public class TestIndexReaderDelete extends LuceneTestCase {
  36   private void deleteReaderReaderConflict(boolean doFullMerge) throws IOException {
  37     Directory dir = newDirectory();
  38
  39     Term searchTerm1 = new Term("content", "aaa");
  40     Term searchTerm2 = new Term("content", "bbb");
  41     Term searchTerm3 = new Term("content", "ccc");
  42
  43     //  add 100 documents with term : aaa
  44     //  add 100 documents with term : bbb
  45     //  add 100 documents with term : ccc
  46     IndexWriter writer  = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE));
  47     for (int i = 0; i < 100; i++) {
  48         addDoc(writer, searchTerm1.text());
  49         addDoc(writer, searchTerm2.text());
  50         addDoc(writer, searchTerm3.text());
  51     }
  52     if (doFullMerge) {
  53       writer.forceMerge(1);
  54     }
  55     writer.close();
  56
  57     // OPEN TWO READERS
  58     // Both readers get segment info as exists at this time
  59     IndexReader reader1 = IndexReader.open(dir, false);
  60     assertEquals("first opened", 100, reader1.docFreq(searchTerm1));
  61     assertEquals("first opened", 100, reader1.docFreq(searchTerm2));
  62     assertEquals("first opened", 100, reader1.docFreq(searchTerm3));
  63     assertTermDocsCount("first opened", reader1, searchTerm1, 100);
  64     assertTermDocsCount("first opened", reader1, searchTerm2, 100);
  65     assertTermDocsCount("first opened", reader1, searchTerm3, 100);
  66
  67     IndexReader reader2 = IndexReader.open(dir, false);
  68     assertEquals("first opened", 100, reader2.docFreq(searchTerm1));
  69     assertEquals("first opened", 100, reader2.docFreq(searchTerm2));
  70     assertEquals("first opened", 100, reader2.docFreq(searchTerm3));
  71     assertTermDocsCount("first opened", reader2, searchTerm1, 100);
  72     assertTermDocsCount("first opened", reader2, searchTerm2, 100);
  73     assertTermDocsCount("first opened", reader2, searchTerm3, 100);
  74
  75     // DELETE DOCS FROM READER 2 and CLOSE IT
  76     // delete documents containing term: aaa
  77     // when the reader is closed, the segment info is updated and
  78     // the first reader is now stale
  79     reader2.deleteDocuments(searchTerm1);
  80     assertEquals("after delete 1", 100, reader2.docFreq(searchTerm1));
  81     assertEquals("after delete 1", 100, reader2.docFreq(searchTerm2));
  82     assertEquals("after delete 1", 100, reader2.docFreq(searchTerm3));
  83     assertTermDocsCount("after delete 1", reader2, searchTerm1, 0);
  84     assertTermDocsCount("after delete 1", reader2, searchTerm2, 100);
  85     assertTermDocsCount("after delete 1", reader2, searchTerm3, 100);
  86     reader2.close();
  87
  88     // Make sure reader 1 is unchanged since it was open earlier
  89     assertEquals("after delete 1", 100, reader1.docFreq(searchTerm1));
  90     assertEquals("after delete 1", 100, reader1.docFreq(searchTerm2));
  91     assertEquals("after delete 1", 100, reader1.docFreq(searchTerm3));
  92     assertTermDocsCount("after delete 1", reader1, searchTerm1, 100);
  93     assertTermDocsCount("after delete 1", reader1, searchTerm2, 100);
  94     assertTermDocsCount("after delete 1", reader1, searchTerm3, 100);
  95
  96
  97     // ATTEMPT TO DELETE FROM STALE READER
  98     // delete documents containing term: bbb
  99     try {
 100         reader1.deleteDocuments(searchTerm2);
 101         fail("Delete allowed from a stale index reader");
 102     } catch (IOException e) {
 103         /* success */
 104     }
 105
 106     // RECREATE READER AND TRY AGAIN
 107     reader1.close();
 108     reader1 = IndexReader.open(dir, false);
 109     assertEquals("reopened", 100, reader1.docFreq(searchTerm1));
 110     assertEquals("reopened", 100, reader1.docFreq(searchTerm2));
 111     assertEquals("reopened", 100, reader1.docFreq(searchTerm3));
 112     assertTermDocsCount("reopened", reader1, searchTerm1, 0);
 113     assertTermDocsCount("reopened", reader1, searchTerm2, 100);
 114     assertTermDocsCount("reopened", reader1, searchTerm3, 100);
 115
 116     reader1.deleteDocuments(searchTerm2);
 117     assertEquals("deleted 2", 100, reader1.docFreq(searchTerm1));
 118     assertEquals("deleted 2", 100, reader1.docFreq(searchTerm2));
 119     assertEquals("deleted 2", 100, reader1.docFreq(searchTerm3));
 120     assertTermDocsCount("deleted 2", reader1, searchTerm1, 0);
 121     assertTermDocsCount("deleted 2", reader1, searchTerm2, 0);
 122     assertTermDocsCount("deleted 2", reader1, searchTerm3, 100);
 123     reader1.close();
 124
 125     // Open another reader to confirm that everything is deleted
 126     reader2 = IndexReader.open(dir, false);
 127     assertTermDocsCount("reopened 2", reader2, searchTerm1, 0);
 128     assertTermDocsCount("reopened 2", reader2, searchTerm2, 0);
 129     assertTermDocsCount("reopened 2", reader2, searchTerm3, 100);
 130     reader2.close();
 131
 132     dir.close();
 133   }
 134
 135   private void deleteReaderWriterConflict(boolean doFullMerge) throws IOException {
 136     //Directory dir = new RAMDirectory();
 137     Directory dir = newDirectory();
 138
 139     Term searchTerm = new Term("content", "aaa");
 140     Term searchTerm2 = new Term("content", "bbb");
 141
 142     //  add 100 documents with term : aaa
 143     IndexWriter writer  = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE));
 144     for (int i = 0; i < 100; i++) {
 145         addDoc(writer, searchTerm.text());
 146     }
 147     writer.close();
 148
 149     // OPEN READER AT THIS POINT - this should fix the view of the
 150     // index at the point of having 100 "aaa" documents and 0 "bbb"
 151     IndexReader reader = IndexReader.open(dir, false);
 152     assertEquals("first docFreq", 100, reader.docFreq(searchTerm));
 153     assertEquals("first docFreq", 0, reader.docFreq(searchTerm2));
 154     assertTermDocsCount("first reader", reader, searchTerm, 100);
 155     assertTermDocsCount("first reader", reader, searchTerm2, 0);
 156
 157     // add 100 documents with term : bbb
 158     writer  = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND));
 159     for (int i = 0; i < 100; i++) {
 160         addDoc(writer, searchTerm2.text());
 161     }
 162
 163     // REQUEST full merge
 164     // This causes a new segment to become current for all subsequent
 165     // searchers. Because of this, deletions made via a previously open
 166     // reader, which would be applied to that reader's segment, are lost
 167     // for subsequent searchers/readers
 168     if (doFullMerge) {
 169       writer.forceMerge(1);
 170     }
 171     writer.close();
 172
 173     // The reader should not see the new data
 174     assertEquals("first docFreq", 100, reader.docFreq(searchTerm));
 175     assertEquals("first docFreq", 0, reader.docFreq(searchTerm2));
 176     assertTermDocsCount("first reader", reader, searchTerm, 100);
 177     assertTermDocsCount("first reader", reader, searchTerm2, 0);
 178
 179
 180     // DELETE DOCUMENTS CONTAINING TERM: aaa
 181     // NOTE: the reader was created when only "aaa" documents were in
 182     int deleted = 0;
 183     try {
 184         deleted = reader.deleteDocuments(searchTerm);
 185         fail("Delete allowed on an index reader with stale segment information");
 186     } catch (StaleReaderException e) {
 187         /* success */
 188     }
 189
 190     // Re-open index reader and try again. This time it should see
 191     // the new data.
 192     reader.close();
 193     reader = IndexReader.open(dir, false);
 194     assertEquals("first docFreq", 100, reader.docFreq(searchTerm));
 195     assertEquals("first docFreq", 100, reader.docFreq(searchTerm2));
 196     assertTermDocsCount("first reader", reader, searchTerm, 100);
 197     assertTermDocsCount("first reader", reader, searchTerm2, 100);
 198
 199     deleted = reader.deleteDocuments(searchTerm);
 200     assertEquals("deleted count", 100, deleted);
 201     assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm));
 202     assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm2));
 203     assertTermDocsCount("deleted termDocs", reader, searchTerm, 0);
 204     assertTermDocsCount("deleted termDocs", reader, searchTerm2, 100);
 205     reader.close();
 206
 207     // CREATE A NEW READER and re-test
 208     reader = IndexReader.open(dir, false);
 209     assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm2));
 210     assertTermDocsCount("deleted termDocs", reader, searchTerm, 0);
 211     assertTermDocsCount("deleted termDocs", reader, searchTerm2, 100);
 212     reader.close();
 213     dir.close();
 214   }
 215
 216   public void testBasicDelete() throws IOException {
 217     Directory dir = newDirectory();
 218
 219     IndexWriter writer = null;
 220     IndexReader reader = null;
 221     Term searchTerm = new Term("content", "aaa");
 222
 223     //  add 100 documents with term : aaa
 224     writer  = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
 225     writer.setInfoStream(VERBOSE ? System.out : null);
 226     for (int i = 0; i < 100; i++) {
 227         addDoc(writer, searchTerm.text());
 228     }
 229     writer.close();
 230
 231     // OPEN READER AT THIS POINT - this should fix the view of the
 232     // index at the point of having 100 "aaa" documents and 0 "bbb"
 233     reader = IndexReader.open(dir, false);
 234     assertEquals("first docFreq", 100, reader.docFreq(searchTerm));
 235     assertTermDocsCount("first reader", reader, searchTerm, 100);
 236     reader.close();
 237
 238     // DELETE DOCUMENTS CONTAINING TERM: aaa
 239     int deleted = 0;
 240     reader = IndexReader.open(dir, false);
 241     deleted = reader.deleteDocuments(searchTerm);
 242     assertEquals("deleted count", 100, deleted);
 243     assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm));
 244     assertTermDocsCount("deleted termDocs", reader, searchTerm, 0);
 245
 246     // open a 2nd reader to make sure first reader can
 247     // commit its changes (.del) while second reader
 248     // is open:
 249     IndexReader reader2 = IndexReader.open(dir, false);
 250     reader.close();
 251
 252     // CREATE A NEW READER and re-test
 253     reader = IndexReader.open(dir, false);
 254     assertEquals("deleted docFreq", 0, reader.docFreq(searchTerm));
 255     assertTermDocsCount("deleted termDocs", reader, searchTerm, 0);
 256     reader.close();
 257     reader2.close();
 258     dir.close();
 259   }
 260
 261   public void testDeleteReaderReaderConflictNoFullMerge() throws IOException {
 262     deleteReaderReaderConflict(false);
 263   }
 264
 265   public void testDeleteReaderReaderConflictFullMerge() throws IOException {
 266     deleteReaderReaderConflict(true);
 267   }
 268
 269   public void testDeleteReaderWriterConflictNoFullMerge() throws IOException {
 270     deleteReaderWriterConflict(false);
 271   }
 272
 273   public void testDeleteReaderWriterConflictFullMerge() throws IOException {
 274     deleteReaderWriterConflict(true);
 275   }
 276
 277   public void testMultiReaderDeletes() throws Exception {
 278     Directory dir = newDirectory();
 279     RandomIndexWriter w= new RandomIndexWriter(random, dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy()));
 280     Document doc = new Document();
 281     doc.add(newField("f", "doctor", Field.Store.NO, Field.Index.NOT_ANALYZED));
 282     w.addDocument(doc);
 283     doc = new Document();
 284     w.commit();
 285     doc.add(newField("f", "who", Field.Store.NO, Field.Index.NOT_ANALYZED));
 286     w.addDocument(doc);
 287     IndexReader r = new SlowMultiReaderWrapper(w.getReader());
 288     w.close();
 289
 290     assertFalse(r.hasDeletions());
 291     r.close();
 292
 293     r = new SlowMultiReaderWrapper(IndexReader.open(dir, false));
 294
 295     assertFalse(r.hasDeletions());
 296     assertEquals(1, r.deleteDocuments(new Term("f", "doctor")));
 297     assertTrue(r.hasDeletions());
 298     assertTrue(r.isDeleted(0));
 299     assertEquals(1, r.deleteDocuments(new Term("f", "who")));
 300     assertTrue(r.isDeleted(1));
 301     r.close();
 302     dir.close();
 303   }
 304
 305   public void testUndeleteAll() throws IOException {
 306     Directory dir = newDirectory();
 307     IndexWriter writer  = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
 308     addDocumentWithFields(writer);
 309     addDocumentWithFields(writer);
 310     writer.close();
 311     IndexReader reader = IndexReader.open(dir, false);
 312     reader.deleteDocument(0);
 313     reader.deleteDocument(1);
 314     reader.undeleteAll();
 315     reader.close();
 316     reader = IndexReader.open(dir, false);
 317     assertEquals(2, reader.numDocs());  // nothing has really been deleted thanks to undeleteAll()
 318     reader.close();
 319     dir.close();
 320   }
 321
 322   public void testUndeleteAllAfterClose() throws IOException {
 323     Directory dir = newDirectory();
 324     IndexWriter writer  = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
 325     addDocumentWithFields(writer);
 326     addDocumentWithFields(writer);
 327     writer.close();
 328     IndexReader reader = IndexReader.open(dir, false);
 329     reader.deleteDocument(0);
 330     reader.close();
 331     reader = IndexReader.open(dir, false);
 332     reader.undeleteAll();
 333     assertEquals(2, reader.numDocs());  // nothing has really been deleted thanks to undeleteAll()
 334     reader.close();
 335     dir.close();
 336   }
 337
 338   public void testUndeleteAllAfterCloseThenReopen() throws IOException {
 339     Directory dir = newDirectory();
 340     IndexWriter writer  = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
 341     addDocumentWithFields(writer);
 342     addDocumentWithFields(writer);
 343     writer.close();
 344     IndexReader reader = IndexReader.open(dir, false);
 345     reader.deleteDocument(0);
 346     reader.close();
 347     reader = IndexReader.open(dir, false);
 348     reader.undeleteAll();
 349     reader.close();
 350     reader = IndexReader.open(dir, false);
 351     assertEquals(2, reader.numDocs());  // nothing has really been deleted thanks to undeleteAll()
 352     reader.close();
 353     dir.close();
 354   }
 355
 356   // LUCENE-1647
 357   public void testIndexReaderUnDeleteAll() throws Exception {
 358     MockDirectoryWrapper dir = newDirectory();
 359     dir.setPreventDoubleWrite(false);
 360     IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
 361         TEST_VERSION_CURRENT, new MockAnalyzer(random)));
 362     writer.addDocument(createDocument("a"));
 363     writer.addDocument(createDocument("b"));
 364     writer.addDocument(createDocument("c"));
 365     writer.close();
 366     IndexReader reader = IndexReader.open(dir, false);
 367     reader.deleteDocuments(new Term("id", "a"));
 368     reader.flush();
 369     reader.deleteDocuments(new Term("id", "b"));
 370     reader.undeleteAll();
 371     reader.deleteDocuments(new Term("id", "b"));
 372     reader.close();
 373     IndexReader.open(dir,true).close();
 374     dir.close();
 375   }
 376 }