lucene-java-3.4.0/lucene/src/test/org/apache/lucene/index/TestIndexReaderDelete.java

   1 package org.apache.lucene.index;
   2
   3 /**
   4  * Licensed to the Apache Software Foundation (ASF) under one or more
   5  * contributor license agreements.  See the NOTICE file distributed with
   6  * this work for additional information regarding copyright ownership.
   7  * The ASF licenses this file to You under the Apache License, Version 2.0
   8  * (the "License"); you may not use this file except in compliance with
   9  * the License.  You may obtain a copy of the License at
  10  *
  11  *     http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  */
  19
  20 import java.io.IOException;
  21
  22 import org.apache.lucene.analysis.MockAnalyzer;
  23 import org.apache.lucene.document.Document;
  24 import org.apache.lucene.document.Field;
  25 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
  26 import org.apache.lucene.store.Directory;
  27 import org.apache.lucene.store.MockDirectoryWrapper;
  28 import org.apache.lucene.util.LuceneTestCase;
  29
  30 import static org.apache.lucene.index.TestIndexReader.addDoc;
  31 import static org.apache.lucene.index.TestIndexReader.addDocumentWithFields;
  32 import static org.apache.lucene.index.TestIndexReader.assertTermDocsCount;
  33 import static org.apache.lucene.index.TestIndexReader.createDocument;
  34
  35 public class TestIndexReaderDelete extends LuceneTestCase {
  36   private void deleteReaderReaderConflict(boolean optimize) throws IOException {
  37     Directory dir = newDirectory();
  38
  39     Term searchTerm1 = new Term("content", "aaa");
  40     Term searchTerm2 = new Term("content", "bbb");
  41     Term searchTerm3 = new Term("content", "ccc");
  42
  43     //  add 100 documents with term : aaa
  44     //  add 100 documents with term : bbb
  45     //  add 100 documents with term : ccc
  46     IndexWriter writer  = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE));
  47     for (int i = 0; i < 100; i++) {
  48         addDoc(writer, searchTerm1.text());
  49         addDoc(writer, searchTerm2.text());
  50         addDoc(writer, searchTerm3.text());
  51     }
  52     if(optimize)
  53       writer.optimize();
  54     writer.close();
  55
  56     // OPEN TWO READERS
  57     // Both readers get segment info as exists at this time
  58     IndexReader reader1 = IndexReader.open(dir, false);
  59     assertEquals("first opened", 100, reader1.docFreq(searchTerm1));
  60     assertEquals("first opened", 100, reader1.docFreq(searchTerm2));
  61     assertEquals("first opened", 100, reader1.docFreq(searchTerm3));
  62     assertTermDocsCount("first opened", reader1, searchTerm1, 100);
  63     assertTermDocsCount("first opened", reader1, searchTerm2, 100);
  64     assertTermDocsCount("first opened", reader1, searchTerm3, 100);
  65
  66     IndexReader reader2 = IndexReader.open(dir, false);
  67     assertEquals("first opened", 100, reader2.docFreq(searchTerm1));
  68     assertEquals("first opened", 100, reader2.docFreq(searchTerm2));
  69     assertEquals("first opened", 100, reader2.docFreq(searchTerm3));
  70     assertTermDocsCount("first opened", reader2, searchTerm1, 100);
  71     assertTermDocsCount("first opened", reader2, searchTerm2, 100);
  72     assertTermDocsCount("first opened", reader2, searchTerm3, 100);
  73
  74     // DELETE DOCS FROM READER 2 and CLOSE IT
  75     // delete documents containing term: aaa
  76     // when the reader is closed, the segment info is updated and
  77     // the first reader is now stale
  78     reader2.deleteDocuments(searchTerm1);
  79     assertEquals("after delete 1", 100, reader2.docFreq(searchTerm1));
  80     assertEquals("after delete 1", 100, reader2.docFreq(searchTerm2));
  81     assertEquals("after delete 1", 100, reader2.docFreq(searchTerm3));
  82     assertTermDocsCount("after delete 1", reader2, searchTerm1, 0);
  83     assertTermDocsCount("after delete 1", reader2, searchTerm2, 100);
  84     assertTermDocsCount("after delete 1", reader2, searchTerm3, 100);
  85     reader2.close();
  86
  87     // Make sure reader 1 is unchanged since it was open earlier
  88     assertEquals("after delete 1", 100, reader1.docFreq(searchTerm1));
  89     assertEquals("after delete 1", 100, reader1.docFreq(searchTerm2));
  90     assertEquals("after delete 1", 100, reader1.docFreq(searchTerm3));
  91     assertTermDocsCount("after delete 1", reader1, searchTerm1, 100);
  92     assertTermDocsCount("after delete 1", reader1, searchTerm2, 100);
  93     assertTermDocsCount("after delete 1", reader1, searchTerm3, 100);
  94
  95
  96     // ATTEMPT TO DELETE FROM STALE READER
  97     // delete documents containing term: bbb
  98     try {
  99         reader1.deleteDocuments(searchTerm2);
 100         fail("Delete allowed from a stale index reader");
 101     } catch (IOException e) {
 102         /* success */
 103     }
 104
 105     // RECREATE READER AND TRY AGAIN
 106     reader1.close();
 107     reader1 = IndexReader.open(dir, false);
 108     assertEquals("reopened", 100, reader1.docFreq(searchTerm1));
 109     assertEquals("reopened", 100, reader1.docFreq(searchTerm2));
 110     assertEquals("reopened", 100, reader1.docFreq(searchTerm3));
 111     assertTermDocsCount("reopened", reader1, searchTerm1, 0);
 112     assertTermDocsCount("reopened", reader1, searchTerm2, 100);
 113     assertTermDocsCount("reopened", reader1, searchTerm3, 100);
 114
 115     reader1.deleteDocuments(searchTerm2);
 116     assertEquals("deleted 2", 100, reader1.docFreq(searchTerm1));
 117     assertEquals("deleted 2", 100, reader1.docFreq(searchTerm2));
 118     assertEquals("deleted 2", 100, reader1.docFreq(searchTerm3));
 119     assertTermDocsCount("deleted 2", reader1, searchTerm1, 0);
 120     assertTermDocsCount("deleted 2", reader1, searchTerm2, 0);
 121     assertTermDocsCount("deleted 2", reader1, searchTerm3, 100);
 122     reader1.close();
 123
 124     // Open another reader to confirm that everything is deleted
 125     reader2 = IndexReader.open(dir, false);
 126     assertTermDocsCount("reopened 2", reader2, searchTerm1, 0);
 127     assertTermDocsCount("reopened 2", reader2, searchTerm2, 0);
 128     assertTermDocsCount("reopened 2", reader2, searchTerm3, 100);
 129     reader2.close();
 130
 131     dir.close();
 132   }
 133
 134   private void deleteReaderWriterConflict(boolean optimize) throws IOException {
 135     //Directory dir = new RAMDirectory();
 136     Directory dir = newDirectory();
 137
 138     Term searchTerm = new Term("content", "aaa");
 139     Term searchTerm2 = new Term("content", "bbb");
 140
 141     //  add 100 documents with term : aaa
 142     IndexWriter writer  = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE));
 143     for (int i = 0; i < 100; i++) {
 144         addDoc(writer, searchTerm.text());
 145     }
 146     writer.close();
 147
 148     // OPEN READER AT THIS POINT - this should fix the view of the
 149     // index at the point of having 100 "aaa" documents and 0 "bbb"
 150     IndexReader reader = IndexReader.open(dir, false);
 151     assertEquals("first docFreq", 100, reader.docFreq(searchTerm));
 152     assertEquals("first docFreq", 0, reader.docFreq(searchTerm2));
 153     assertTermDocsCount("first reader", reader, searchTerm, 100);
 154     assertTermDocsCount("first reader", reader, searchTerm2, 0);
 155
 156     // add 100 documents with term : bbb
 157     writer  = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND));
 158     for (int i = 0; i < 100; i++) {
 159         addDoc(writer, searchTerm2.text());
 160     }
 161
 162     // REQUEST OPTIMIZATION
 163     // This causes a new segment to become current for all subsequent
 164     // searchers. Because of this, deletions made via a previously open
 165     // reader, which would be applied to that reader's segment, are lost
 166     // for subsequent searchers/readers
 167     if(optimize)
 168       writer.optimize();
 169     writer.close();
 170
 171     // The reader should not see the new data
 172     assertEquals("first docFreq", 100, reader.docFreq(searchTerm));
 173     assertEquals("first docFreq", 0, reader.docFreq(searchTerm2));
 174     assertTermDocsCount("first reader", reader, searchTerm, 100);
 175     assertTermDocsCount("first reader", reader, searchTerm2, 0);
 176
 177
 178     // DELETE DOCUMENTS CONTAINING TERM: aaa
 179     // NOTE: the reader was created when only "aaa" documents were in
 180     int deleted = 0;
 181     try {
 182         deleted = reader.deleteDocuments(searchTerm);
 183         fail("Delete allowed on an index reader with stale segment information");
 184     } catch (StaleReaderException e) {
 185         /* success */
 186     }
 187
 188     // Re-open index reader and try again. This time it should see
 189     // the new data.
 190     reader.close();
 191     reader = IndexReader.open(dir, false);
 192     assertEquals("first docFreq", 100, reader.docFreq(searchTerm));
 193     assertEquals("first docFreq", 100, reader.docFreq(searchTerm2));
 194     assertTermDocsCount("first reader", reader, searchTerm, 100);
 195     assertTermDocsCount("first reader", reader, searchTerm2, 100);
 196
 197     deleted = reader.deleteDocuments(searchTerm);
 198     assertEquals("deleted count", 100, deleted);
 199     assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm));
 200     assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm2));
 201     assertTermDocsCount("deleted termDocs", reader, searchTerm, 0);
 202     assertTermDocsCount("deleted termDocs", reader, searchTerm2, 100);
 203     reader.close();
 204
 205     // CREATE A NEW READER and re-test
 206     reader = IndexReader.open(dir, false);
 207     assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm2));
 208     assertTermDocsCount("deleted termDocs", reader, searchTerm, 0);
 209     assertTermDocsCount("deleted termDocs", reader, searchTerm2, 100);
 210     reader.close();
 211     dir.close();
 212   }
 213
 214   public void testBasicDelete() throws IOException {
 215     Directory dir = newDirectory();
 216
 217     IndexWriter writer = null;
 218     IndexReader reader = null;
 219     Term searchTerm = new Term("content", "aaa");
 220
 221     //  add 100 documents with term : aaa
 222     writer  = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
 223     writer.setInfoStream(VERBOSE ? System.out : null);
 224     for (int i = 0; i < 100; i++) {
 225         addDoc(writer, searchTerm.text());
 226     }
 227     writer.close();
 228
 229     // OPEN READER AT THIS POINT - this should fix the view of the
 230     // index at the point of having 100 "aaa" documents and 0 "bbb"
 231     reader = IndexReader.open(dir, false);
 232     assertEquals("first docFreq", 100, reader.docFreq(searchTerm));
 233     assertTermDocsCount("first reader", reader, searchTerm, 100);
 234     reader.close();
 235
 236     // DELETE DOCUMENTS CONTAINING TERM: aaa
 237     int deleted = 0;
 238     reader = IndexReader.open(dir, false);
 239     deleted = reader.deleteDocuments(searchTerm);
 240     assertEquals("deleted count", 100, deleted);
 241     assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm));
 242     assertTermDocsCount("deleted termDocs", reader, searchTerm, 0);
 243
 244     // open a 2nd reader to make sure first reader can
 245     // commit its changes (.del) while second reader
 246     // is open:
 247     IndexReader reader2 = IndexReader.open(dir, false);
 248     reader.close();
 249
 250     // CREATE A NEW READER and re-test
 251     reader = IndexReader.open(dir, false);
 252     assertEquals("deleted docFreq", 0, reader.docFreq(searchTerm));
 253     assertTermDocsCount("deleted termDocs", reader, searchTerm, 0);
 254     reader.close();
 255     reader2.close();
 256     dir.close();
 257   }
 258
 259   public void testDeleteReaderReaderConflictUnoptimized() throws IOException {
 260     deleteReaderReaderConflict(false);
 261   }
 262
 263   public void testDeleteReaderReaderConflictOptimized() throws IOException {
 264     deleteReaderReaderConflict(true);
 265   }
 266
 267   public void testDeleteReaderWriterConflictUnoptimized() throws IOException {
 268     deleteReaderWriterConflict(false);
 269   }
 270
 271   public void testDeleteReaderWriterConflictOptimized() throws IOException {
 272     deleteReaderWriterConflict(true);
 273   }
 274
 275   public void testMultiReaderDeletes() throws Exception {
 276     Directory dir = newDirectory();
 277     RandomIndexWriter w= new RandomIndexWriter(random, dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy()));
 278     Document doc = new Document();
 279     doc.add(newField("f", "doctor", Field.Store.NO, Field.Index.NOT_ANALYZED));
 280     w.addDocument(doc);
 281     doc = new Document();
 282     w.commit();
 283     doc.add(newField("f", "who", Field.Store.NO, Field.Index.NOT_ANALYZED));
 284     w.addDocument(doc);
 285     IndexReader r = new SlowMultiReaderWrapper(w.getReader());
 286     w.close();
 287
 288     assertFalse(r.hasDeletions());
 289     r.close();
 290
 291     r = new SlowMultiReaderWrapper(IndexReader.open(dir, false));
 292
 293     assertFalse(r.hasDeletions());
 294     assertEquals(1, r.deleteDocuments(new Term("f", "doctor")));
 295     assertTrue(r.hasDeletions());
 296     assertTrue(r.isDeleted(0));
 297     assertEquals(1, r.deleteDocuments(new Term("f", "who")));
 298     assertTrue(r.isDeleted(1));
 299     r.close();
 300     dir.close();
 301   }
 302
 303   public void testUndeleteAll() throws IOException {
 304     Directory dir = newDirectory();
 305     IndexWriter writer  = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
 306     addDocumentWithFields(writer);
 307     addDocumentWithFields(writer);
 308     writer.close();
 309     IndexReader reader = IndexReader.open(dir, false);
 310     reader.deleteDocument(0);
 311     reader.deleteDocument(1);
 312     reader.undeleteAll();
 313     reader.close();
 314     reader = IndexReader.open(dir, false);
 315     assertEquals(2, reader.numDocs());  // nothing has really been deleted thanks to undeleteAll()
 316     reader.close();
 317     dir.close();
 318   }
 319
 320   public void testUndeleteAllAfterClose() throws IOException {
 321     Directory dir = newDirectory();
 322     IndexWriter writer  = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
 323     addDocumentWithFields(writer);
 324     addDocumentWithFields(writer);
 325     writer.close();
 326     IndexReader reader = IndexReader.open(dir, false);
 327     reader.deleteDocument(0);
 328     reader.close();
 329     reader = IndexReader.open(dir, false);
 330     reader.undeleteAll();
 331     assertEquals(2, reader.numDocs());  // nothing has really been deleted thanks to undeleteAll()
 332     reader.close();
 333     dir.close();
 334   }
 335
 336   public void testUndeleteAllAfterCloseThenReopen() throws IOException {
 337     Directory dir = newDirectory();
 338     IndexWriter writer  = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
 339     addDocumentWithFields(writer);
 340     addDocumentWithFields(writer);
 341     writer.close();
 342     IndexReader reader = IndexReader.open(dir, false);
 343     reader.deleteDocument(0);
 344     reader.close();
 345     reader = IndexReader.open(dir, false);
 346     reader.undeleteAll();
 347     reader.close();
 348     reader = IndexReader.open(dir, false);
 349     assertEquals(2, reader.numDocs());  // nothing has really been deleted thanks to undeleteAll()
 350     reader.close();
 351     dir.close();
 352   }
 353
 354   // LUCENE-1647
 355   public void testIndexReaderUnDeleteAll() throws Exception {
 356     MockDirectoryWrapper dir = newDirectory();
 357     dir.setPreventDoubleWrite(false);
 358     IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
 359         TEST_VERSION_CURRENT, new MockAnalyzer(random)));
 360     writer.addDocument(createDocument("a"));
 361     writer.addDocument(createDocument("b"));
 362     writer.addDocument(createDocument("c"));
 363     writer.close();
 364     IndexReader reader = IndexReader.open(dir, false);
 365     reader.deleteDocuments(new Term("id", "a"));
 366     reader.flush();
 367     reader.deleteDocuments(new Term("id", "b"));
 368     reader.undeleteAll();
 369     reader.deleteDocuments(new Term("id", "b"));
 370     reader.close();
 371     IndexReader.open(dir,true).close();
 372     dir.close();
 373   }
 374 }