1 package org.apache.lucene.index;
4 * Licensed to the Apache Software Foundation (ASF) under one or more
5 * contributor license agreements. See the NOTICE file distributed with
6 * this work for additional information regarding copyright ownership.
7 * The ASF licenses this file to You under the Apache License, Version 2.0
8 * (the "License"); you may not use this file except in compliance with
9 * the License. You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
20 import java.io.IOException;
21 import java.io.Reader;
22 import java.util.HashMap;
24 import java.util.concurrent.atomic.AtomicBoolean;
26 import org.apache.lucene.analysis.Analyzer;
27 import org.apache.lucene.analysis.MockAnalyzer;
28 import org.apache.lucene.analysis.MockFixedLengthPayloadFilter;
29 import org.apache.lucene.analysis.MockTokenizer;
30 import org.apache.lucene.analysis.TokenStream;
31 import org.apache.lucene.document.Document;
32 import org.apache.lucene.document.Field;
33 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
34 import org.apache.lucene.search.IndexSearcher;
35 import org.apache.lucene.search.ScoreDoc;
36 import org.apache.lucene.search.TermQuery;
37 import org.apache.lucene.store.Directory;
38 import org.apache.lucene.store.MockDirectoryWrapper;
39 import org.apache.lucene.util.LuceneTestCase;
40 import org.apache.lucene.util._TestUtil;
42 public class TestIndexWriterCommit extends LuceneTestCase {
44 * Simple test for "commit on close": open writer then
45 * add a bunch of docs, making sure reader does not see
46 * these docs until writer is closed.
48 public void testCommitOnClose() throws IOException {
49 Directory dir = newDirectory();
50 IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
51 for (int i = 0; i < 14; i++) {
52 TestIndexWriter.addDoc(writer);
56 Term searchTerm = new Term("content", "aaa");
57 IndexReader reader = IndexReader.open(dir, false);
58 IndexSearcher searcher = new IndexSearcher(reader);
59 ScoreDoc[] hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
60 assertEquals("first number of hits", 14, hits.length);
64 reader = IndexReader.open(dir, true);
66 writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
67 for(int i=0;i<3;i++) {
68 for(int j=0;j<11;j++) {
69 TestIndexWriter.addDoc(writer);
71 IndexReader r = IndexReader.open(dir, false);
72 searcher = new IndexSearcher(r);
73 hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
74 assertEquals("reader incorrectly sees changes from writer", 14, hits.length);
77 assertTrue("reader should have still been current", reader.isCurrent());
80 // Now, close the writer:
82 assertFalse("reader should not be current now", reader.isCurrent());
84 IndexReader r = IndexReader.open(dir, false);
85 searcher = new IndexSearcher(r);
86 hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
87 assertEquals("reader did not see changes after writer was closed", 47, hits.length);
95 * Simple test for "commit on close": open writer, then
96 * add a bunch of docs, making sure reader does not see
97 * them until writer has closed. Then instead of
98 * closing the writer, call abort and verify reader sees
99 * nothing was added. Then verify we can open the index
100 * and add docs to it.
102 public void testCommitOnCloseAbort() throws IOException {
103 MockDirectoryWrapper dir = newDirectory();
104 IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(10));
105 for (int i = 0; i < 14; i++) {
106 TestIndexWriter.addDoc(writer);
110 Term searchTerm = new Term("content", "aaa");
111 IndexReader reader = IndexReader.open(dir, false);
112 IndexSearcher searcher = new IndexSearcher(reader);
113 ScoreDoc[] hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
114 assertEquals("first number of hits", 14, hits.length);
118 writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))
119 .setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(10));
120 for(int j=0;j<17;j++) {
121 TestIndexWriter.addDoc(writer);
124 writer.deleteDocuments(searchTerm);
126 reader = IndexReader.open(dir, false);
127 searcher = new IndexSearcher(reader);
128 hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
129 assertEquals("reader incorrectly sees changes from writer", 14, hits.length);
133 // Now, close the writer:
136 TestIndexWriter.assertNoUnreferencedFiles(dir, "unreferenced files remain after rollback()");
138 reader = IndexReader.open(dir, false);
139 searcher = new IndexSearcher(reader);
140 hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
141 assertEquals("saw changes after writer.abort", 14, hits.length);
145 // Now make sure we can re-open the index, add docs,
147 writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))
148 .setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(10));
150 // On abort, writer in fact may write to the same
152 dir.setPreventDoubleWrite(false);
154 for(int i=0;i<12;i++) {
155 for(int j=0;j<17;j++) {
156 TestIndexWriter.addDoc(writer);
158 IndexReader r = IndexReader.open(dir, false);
159 searcher = new IndexSearcher(r);
160 hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
161 assertEquals("reader incorrectly sees changes from writer", 14, hits.length);
167 IndexReader r = IndexReader.open(dir, false);
168 searcher = new IndexSearcher(r);
169 hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
170 assertEquals("didn't see changes after close", 218, hits.length);
178 * Verify that a writer with "commit on close" indeed
179 * cleans up the temp segments created after opening
180 * that are not referenced by the starting segments
181 * file. We check this by using MockDirectoryWrapper to
182 * measure max temp disk space used.
184 public void testCommitOnCloseDiskUsage() throws IOException {
185 MockDirectoryWrapper dir = newDirectory();
187 if (random.nextBoolean()) {
189 analyzer = new Analyzer() {
191 public TokenStream tokenStream(String fieldName, Reader reader) {
192 return new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
196 // fixed length payloads
197 final int length = random.nextInt(200);
198 analyzer = new Analyzer() {
200 public TokenStream tokenStream(String fieldName, Reader reader) {
201 return new MockFixedLengthPayloadFilter(random,
202 new MockTokenizer(reader, MockTokenizer.WHITESPACE, true),
208 IndexWriter writer = new IndexWriter(
210 newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer).
211 setMaxBufferedDocs(10).
212 setReaderPooling(false).
213 setMergePolicy(newLogMergePolicy(10))
215 for(int j=0;j<30;j++) {
216 TestIndexWriter.addDocWithIndex(writer, j);
219 dir.resetMaxUsedSizeInBytes();
221 dir.setTrackDiskUsage(true);
222 long startDiskUsage = dir.getMaxUsedSizeInBytes();
223 writer = new IndexWriter(
225 newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer)
226 .setOpenMode(OpenMode.APPEND).
227 setMaxBufferedDocs(10).
228 setMergeScheduler(new SerialMergeScheduler()).
229 setReaderPooling(false).
230 setMergePolicy(newLogMergePolicy(10))
233 for(int j=0;j<1470;j++) {
234 TestIndexWriter.addDocWithIndex(writer, j);
236 long midDiskUsage = dir.getMaxUsedSizeInBytes();
237 dir.resetMaxUsedSizeInBytes();
238 writer.forceMerge(1);
241 IndexReader.open(dir, true).close();
243 long endDiskUsage = dir.getMaxUsedSizeInBytes();
245 // Ending index is 50X as large as starting index; due
246 // to 3X disk usage normally we allow 150X max
247 // transient usage. If something is wrong w/ deleter
248 // and it doesn't delete intermediate segments then it
249 // will exceed this 150X:
250 // System.out.println("start " + startDiskUsage + "; mid " + midDiskUsage + ";end " + endDiskUsage);
251 assertTrue("writer used too much space while adding documents: mid=" + midDiskUsage + " start=" + startDiskUsage + " end=" + endDiskUsage + " max=" + (startDiskUsage*150),
252 midDiskUsage < 150*startDiskUsage);
253 assertTrue("writer used too much space after close: endDiskUsage=" + endDiskUsage + " startDiskUsage=" + startDiskUsage + " max=" + (startDiskUsage*150),
254 endDiskUsage < 150*startDiskUsage);
260 * Verify that calling forceMerge when writer is open for
261 * "commit on close" works correctly both for rollback()
264 public void testCommitOnCloseForceMerge() throws IOException {
265 MockDirectoryWrapper dir = newDirectory();
266 // Must disable throwing exc on double-write: this
267 // test uses IW.rollback which easily results in
268 // writing to same file more than once
269 dir.setPreventDoubleWrite(false);
270 IndexWriter writer = new IndexWriter(
272 newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
273 setMaxBufferedDocs(10).
274 setMergePolicy(newLogMergePolicy(10))
276 for(int j=0;j<17;j++) {
277 TestIndexWriter.addDocWithIndex(writer, j);
281 writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND));
282 writer.forceMerge(1);
285 writer.setInfoStream(System.out);
288 // Open a reader before closing (commiting) the writer:
289 IndexReader reader = IndexReader.open(dir, true);
291 // Reader should see index as multi-seg at this
293 assertTrue("Reader incorrectly sees one segment", reader.getSequentialSubReaders().length > 1);
298 TestIndexWriter.assertNoUnreferencedFiles(dir, "aborted writer after forceMerge");
300 // Open a reader after aborting writer:
301 reader = IndexReader.open(dir, true);
303 // Reader should still see index as multi-segment
304 assertTrue("Reader incorrectly sees one segment", reader.getSequentialSubReaders().length > 1);
308 System.out.println("TEST: do real full merge");
310 writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND));
312 writer.setInfoStream(System.out);
314 writer.forceMerge(1);
318 System.out.println("TEST: writer closed");
320 TestIndexWriter.assertNoUnreferencedFiles(dir, "aborted writer after forceMerge");
322 // Open a reader after aborting writer:
323 reader = IndexReader.open(dir, true);
325 // Reader should see index as one segment
326 assertEquals("Reader incorrectly sees more than one segment", 1, reader.getSequentialSubReaders().length);
331 // LUCENE-2095: make sure with multiple threads commit
332 // doesn't return until all changes are in fact in the
334 public void testCommitThreadSafety() throws Throwable {
335 final int NUM_THREADS = 5;
336 final double RUN_SEC = 0.5;
337 final Directory dir = newDirectory();
338 final RandomIndexWriter w = new RandomIndexWriter(random, dir, newIndexWriterConfig(
339 TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy()));
340 _TestUtil.reduceOpenFiles(w.w);
342 final AtomicBoolean failed = new AtomicBoolean();
343 Thread[] threads = new Thread[NUM_THREADS];
344 final long endTime = System.currentTimeMillis()+((long) (RUN_SEC*1000));
345 for(int i=0;i<NUM_THREADS;i++) {
346 final int finalI = i;
347 threads[i] = new Thread() {
351 final Document doc = new Document();
352 IndexReader r = IndexReader.open(dir);
353 Field f = newField("f", "", Field.Store.NO, Field.Index.NOT_ANALYZED);
357 if (failed.get()) break;
358 for(int j=0;j<10;j++) {
359 final String s = finalI + "_" + String.valueOf(count++);
363 IndexReader r2 = IndexReader.openIfChanged(r);
368 assertEquals("term=f:" + s + "; r=" + r, 1, r.docFreq(new Term("f", s)));
370 } while(System.currentTimeMillis() < endTime);
372 } catch (Throwable t) {
374 throw new RuntimeException(t);
380 for(int i=0;i<NUM_THREADS;i++) {
383 assertFalse(failed.get());
388 // LUCENE-1044: test writer.commit() when ac=false
389 public void testForceCommit() throws IOException {
390 Directory dir = newDirectory();
392 IndexWriter writer = new IndexWriter(
394 newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
395 setMaxBufferedDocs(2).
396 setMergePolicy(newLogMergePolicy(5))
400 for (int i = 0; i < 23; i++)
401 TestIndexWriter.addDoc(writer);
403 IndexReader reader = IndexReader.open(dir, true);
404 assertEquals(0, reader.numDocs());
406 IndexReader reader2 = IndexReader.openIfChanged(reader);
407 assertNotNull(reader2);
408 assertEquals(0, reader.numDocs());
409 assertEquals(23, reader2.numDocs());
412 for (int i = 0; i < 17; i++)
413 TestIndexWriter.addDoc(writer);
414 assertEquals(23, reader2.numDocs());
416 reader = IndexReader.open(dir, true);
417 assertEquals(23, reader.numDocs());
421 reader = IndexReader.open(dir, true);
422 assertEquals(40, reader.numDocs());
428 public void testFutureCommit() throws Exception {
429 Directory dir = newDirectory();
431 IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE));
432 Document doc = new Document();
436 Map<String,String> commitData = new HashMap<String,String>();
437 commitData.put("tag", "first");
438 w.commit(commitData);
440 // commit to "second"
442 commitData.put("tag", "second");
443 w.commit(commitData);
446 // open "first" with IndexWriter
447 IndexCommit commit = null;
448 for(IndexCommit c : IndexReader.listCommits(dir)) {
449 if (c.getUserData().get("tag").equals("first")) {
455 assertNotNull(commit);
457 w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE).setIndexCommit(commit));
459 assertEquals(1, w.numDocs());
461 // commit IndexWriter to "third"
463 commitData.put("tag", "third");
464 w.commit(commitData);
467 // make sure "second" commit is still there
469 for(IndexCommit c : IndexReader.listCommits(dir)) {
470 if (c.getUserData().get("tag").equals("second")) {
476 assertNotNull(commit);
478 IndexReader r = IndexReader.open(commit, true);
479 assertEquals(2, r.numDocs());
482 // open "second", w/ writeable IndexReader & commit
483 r = IndexReader.open(commit, NoDeletionPolicy.INSTANCE, false);
484 assertEquals(2, r.numDocs());
487 commitData.put("tag", "fourth");
488 r.commit(commitData);
491 // make sure "third" commit is still there
493 for(IndexCommit c : IndexReader.listCommits(dir)) {
494 if (c.getUserData().get("tag").equals("third")) {
499 assertNotNull(commit);
504 public void testNoCommits() throws Exception {
505 // Tests that if we don't call commit(), the directory has 0 commits. This has
506 // changed since LUCENE-2386, where before IW would always commit on a fresh
508 Directory dir = newDirectory();
509 IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
511 IndexReader.listCommits(dir);
512 fail("listCommits should have thrown an exception over empty index");
513 } catch (IndexNotFoundException e) {
516 // No changes still should generate a commit, because it's a new index.
518 assertEquals("expected 1 commits!", 1, IndexReader.listCommits(dir).size());
522 // LUCENE-1274: test writer.prepareCommit()
523 public void testPrepareCommit() throws IOException {
524 Directory dir = newDirectory();
526 IndexWriter writer = new IndexWriter(
528 newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
529 setMaxBufferedDocs(2).
530 setMergePolicy(newLogMergePolicy(5))
534 for (int i = 0; i < 23; i++)
535 TestIndexWriter.addDoc(writer);
537 IndexReader reader = IndexReader.open(dir, true);
538 assertEquals(0, reader.numDocs());
540 writer.prepareCommit();
542 IndexReader reader2 = IndexReader.open(dir, true);
543 assertEquals(0, reader2.numDocs());
547 IndexReader reader3 = IndexReader.openIfChanged(reader);
548 assertNotNull(reader3);
549 assertEquals(0, reader.numDocs());
550 assertEquals(0, reader2.numDocs());
551 assertEquals(23, reader3.numDocs());
555 for (int i = 0; i < 17; i++)
556 TestIndexWriter.addDoc(writer);
558 assertEquals(23, reader3.numDocs());
560 reader = IndexReader.open(dir, true);
561 assertEquals(23, reader.numDocs());
564 writer.prepareCommit();
566 reader = IndexReader.open(dir, true);
567 assertEquals(23, reader.numDocs());
571 reader = IndexReader.open(dir, true);
572 assertEquals(40, reader.numDocs());
578 // LUCENE-1274: test writer.prepareCommit()
579 public void testPrepareCommitRollback() throws IOException {
580 MockDirectoryWrapper dir = newDirectory();
581 dir.setPreventDoubleWrite(false);
583 IndexWriter writer = new IndexWriter(
585 newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
586 setMaxBufferedDocs(2).
587 setMergePolicy(newLogMergePolicy(5))
591 for (int i = 0; i < 23; i++)
592 TestIndexWriter.addDoc(writer);
594 IndexReader reader = IndexReader.open(dir, true);
595 assertEquals(0, reader.numDocs());
597 writer.prepareCommit();
599 IndexReader reader2 = IndexReader.open(dir, true);
600 assertEquals(0, reader2.numDocs());
604 IndexReader reader3 = IndexReader.openIfChanged(reader);
606 assertEquals(0, reader.numDocs());
607 assertEquals(0, reader2.numDocs());
611 writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
612 for (int i = 0; i < 17; i++)
613 TestIndexWriter.addDoc(writer);
615 reader = IndexReader.open(dir, true);
616 assertEquals(0, reader.numDocs());
619 writer.prepareCommit();
621 reader = IndexReader.open(dir, true);
622 assertEquals(0, reader.numDocs());
626 reader = IndexReader.open(dir, true);
627 assertEquals(17, reader.numDocs());
634 public void testPrepareCommitNoChanges() throws IOException {
635 Directory dir = newDirectory();
637 IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
638 writer.prepareCommit();
642 IndexReader reader = IndexReader.open(dir, true);
643 assertEquals(0, reader.numDocs());
649 public void testCommitUserData() throws IOException {
650 Directory dir = newDirectory();
651 IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(2));
652 for(int j=0;j<17;j++)
653 TestIndexWriter.addDoc(w);
656 assertEquals(0, IndexReader.getCommitUserData(dir).size());
658 IndexReader r = IndexReader.open(dir, true);
659 // commit(Map) never called for this index
660 assertEquals(0, r.getCommitUserData().size());
663 w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(2));
664 for(int j=0;j<17;j++)
665 TestIndexWriter.addDoc(w);
666 Map<String,String> data = new HashMap<String,String>();
667 data.put("label", "test1");
671 assertEquals("test1", IndexReader.getCommitUserData(dir).get("label"));
673 r = IndexReader.open(dir, true);
674 assertEquals("test1", r.getCommitUserData().get("label"));
677 w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
681 assertEquals("test1", IndexReader.getCommitUserData(dir).get("label"));