1 package org.apache.lucene.index;
4 * Licensed to the Apache Software Foundation (ASF) under one or more
5 * contributor license agreements. See the NOTICE file distributed with
6 * this work for additional information regarding copyright ownership.
7 * The ASF licenses this file to You under the Apache License, Version 2.0
8 * (the "License"); you may not use this file except in compliance with
9 * the License. You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
20 import java.io.IOException;
21 import java.io.Reader;
22 import java.util.HashMap;
24 import java.util.concurrent.atomic.AtomicBoolean;
26 import org.apache.lucene.analysis.Analyzer;
27 import org.apache.lucene.analysis.MockAnalyzer;
28 import org.apache.lucene.analysis.MockFixedLengthPayloadFilter;
29 import org.apache.lucene.analysis.MockTokenizer;
30 import org.apache.lucene.analysis.TokenStream;
31 import org.apache.lucene.document.Document;
32 import org.apache.lucene.document.Field;
33 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
34 import org.apache.lucene.search.IndexSearcher;
35 import org.apache.lucene.search.ScoreDoc;
36 import org.apache.lucene.search.TermQuery;
37 import org.apache.lucene.store.Directory;
38 import org.apache.lucene.store.MockDirectoryWrapper;
39 import org.apache.lucene.util.LuceneTestCase;
40 import org.apache.lucene.util._TestUtil;
42 public class TestIndexWriterCommit extends LuceneTestCase {
44 * Simple test for "commit on close": open writer then
45 * add a bunch of docs, making sure reader does not see
46 * these docs until writer is closed.
48 public void testCommitOnClose() throws IOException {
49 Directory dir = newDirectory();
50 IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
51 for (int i = 0; i < 14; i++) {
52 TestIndexWriter.addDoc(writer);
56 Term searchTerm = new Term("content", "aaa");
57 IndexSearcher searcher = new IndexSearcher(dir, false);
58 ScoreDoc[] hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
59 assertEquals("first number of hits", 14, hits.length);
62 IndexReader reader = IndexReader.open(dir, true);
64 writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
65 for(int i=0;i<3;i++) {
66 for(int j=0;j<11;j++) {
67 TestIndexWriter.addDoc(writer);
69 searcher = new IndexSearcher(dir, false);
70 hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
71 assertEquals("reader incorrectly sees changes from writer", 14, hits.length);
73 assertTrue("reader should have still been current", reader.isCurrent());
76 // Now, close the writer:
78 assertFalse("reader should not be current now", reader.isCurrent());
80 searcher = new IndexSearcher(dir, false);
81 hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
82 assertEquals("reader did not see changes after writer was closed", 47, hits.length);
89 * Simple test for "commit on close": open writer, then
90 * add a bunch of docs, making sure reader does not see
91 * them until writer has closed. Then instead of
92 * closing the writer, call abort and verify reader sees
93 * nothing was added. Then verify we can open the index
96 public void testCommitOnCloseAbort() throws IOException {
97 MockDirectoryWrapper dir = newDirectory();
98 IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(10));
99 for (int i = 0; i < 14; i++) {
100 TestIndexWriter.addDoc(writer);
104 Term searchTerm = new Term("content", "aaa");
105 IndexSearcher searcher = new IndexSearcher(dir, false);
106 ScoreDoc[] hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
107 assertEquals("first number of hits", 14, hits.length);
110 writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))
111 .setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(10));
112 for(int j=0;j<17;j++) {
113 TestIndexWriter.addDoc(writer);
116 writer.deleteDocuments(searchTerm);
118 searcher = new IndexSearcher(dir, false);
119 hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
120 assertEquals("reader incorrectly sees changes from writer", 14, hits.length);
123 // Now, close the writer:
126 TestIndexWriter.assertNoUnreferencedFiles(dir, "unreferenced files remain after rollback()");
128 searcher = new IndexSearcher(dir, false);
129 hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
130 assertEquals("saw changes after writer.abort", 14, hits.length);
133 // Now make sure we can re-open the index, add docs,
135 writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))
136 .setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(10));
138 // On abort, writer in fact may write to the same
140 dir.setPreventDoubleWrite(false);
142 for(int i=0;i<12;i++) {
143 for(int j=0;j<17;j++) {
144 TestIndexWriter.addDoc(writer);
146 searcher = new IndexSearcher(dir, false);
147 hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
148 assertEquals("reader incorrectly sees changes from writer", 14, hits.length);
153 searcher = new IndexSearcher(dir, false);
154 hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
155 assertEquals("didn't see changes after close", 218, hits.length);
162 * Verify that a writer with "commit on close" indeed
163 * cleans up the temp segments created after opening
164 * that are not referenced by the starting segments
165 * file. We check this by using MockDirectoryWrapper to
166 * measure max temp disk space used.
168 public void testCommitOnCloseDiskUsage() throws IOException {
169 MockDirectoryWrapper dir = newDirectory();
171 if (random.nextBoolean()) {
173 analyzer = new Analyzer() {
175 public TokenStream tokenStream(String fieldName, Reader reader) {
176 return new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
180 // fixed length payloads
181 final int length = random.nextInt(200);
182 analyzer = new Analyzer() {
184 public TokenStream tokenStream(String fieldName, Reader reader) {
185 return new MockFixedLengthPayloadFilter(random,
186 new MockTokenizer(reader, MockTokenizer.WHITESPACE, true),
192 IndexWriter writer = new IndexWriter(
194 newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer).
195 setMaxBufferedDocs(10).
196 setReaderPooling(false).
197 setMergePolicy(newLogMergePolicy(10))
199 for(int j=0;j<30;j++) {
200 TestIndexWriter.addDocWithIndex(writer, j);
203 dir.resetMaxUsedSizeInBytes();
205 dir.setTrackDiskUsage(true);
206 long startDiskUsage = dir.getMaxUsedSizeInBytes();
207 writer = new IndexWriter(
209 newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer)
210 .setOpenMode(OpenMode.APPEND).
211 setMaxBufferedDocs(10).
212 setMergeScheduler(new SerialMergeScheduler()).
213 setReaderPooling(false).
214 setMergePolicy(newLogMergePolicy(10))
217 for(int j=0;j<1470;j++) {
218 TestIndexWriter.addDocWithIndex(writer, j);
220 long midDiskUsage = dir.getMaxUsedSizeInBytes();
221 dir.resetMaxUsedSizeInBytes();
225 IndexReader.open(dir, true).close();
227 long endDiskUsage = dir.getMaxUsedSizeInBytes();
229 // Ending index is 50X as large as starting index; due
230 // to 3X disk usage normally we allow 150X max
231 // transient usage. If something is wrong w/ deleter
232 // and it doesn't delete intermediate segments then it
233 // will exceed this 150X:
234 // System.out.println("start " + startDiskUsage + "; mid " + midDiskUsage + ";end " + endDiskUsage);
235 assertTrue("writer used too much space while adding documents: mid=" + midDiskUsage + " start=" + startDiskUsage + " end=" + endDiskUsage + " max=" + (startDiskUsage*150),
236 midDiskUsage < 150*startDiskUsage);
237 assertTrue("writer used too much space after close: endDiskUsage=" + endDiskUsage + " startDiskUsage=" + startDiskUsage + " max=" + (startDiskUsage*150),
238 endDiskUsage < 150*startDiskUsage);
244 * Verify that calling optimize when writer is open for
245 * "commit on close" works correctly both for rollback()
248 public void testCommitOnCloseOptimize() throws IOException {
249 MockDirectoryWrapper dir = newDirectory();
250 // Must disable throwing exc on double-write: this
251 // test uses IW.rollback which easily results in
252 // writing to same file more than once
253 dir.setPreventDoubleWrite(false);
254 IndexWriter writer = new IndexWriter(
256 newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
257 setMaxBufferedDocs(10).
258 setMergePolicy(newLogMergePolicy(10))
260 for(int j=0;j<17;j++) {
261 TestIndexWriter.addDocWithIndex(writer, j);
265 writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND));
269 writer.setInfoStream(System.out);
272 // Open a reader before closing (commiting) the writer:
273 IndexReader reader = IndexReader.open(dir, true);
275 // Reader should see index as unoptimized at this
277 assertFalse("Reader incorrectly sees that the index is optimized", reader.isOptimized());
282 TestIndexWriter.assertNoUnreferencedFiles(dir, "aborted writer after optimize");
284 // Open a reader after aborting writer:
285 reader = IndexReader.open(dir, true);
287 // Reader should still see index as unoptimized:
288 assertFalse("Reader incorrectly sees that the index is optimized", reader.isOptimized());
292 System.out.println("TEST: do real optimize");
294 writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND));
296 writer.setInfoStream(System.out);
302 System.out.println("TEST: writer closed");
304 TestIndexWriter.assertNoUnreferencedFiles(dir, "aborted writer after optimize");
306 // Open a reader after aborting writer:
307 reader = IndexReader.open(dir, true);
309 // Reader should still see index as unoptimized:
310 assertTrue("Reader incorrectly sees that the index is unoptimized", reader.isOptimized());
315 // LUCENE-2095: make sure with multiple threads commit
316 // doesn't return until all changes are in fact in the
318 public void testCommitThreadSafety() throws Throwable {
319 final int NUM_THREADS = 5;
320 final double RUN_SEC = 0.5;
321 final Directory dir = newDirectory();
322 final RandomIndexWriter w = new RandomIndexWriter(random, dir, newIndexWriterConfig(
323 TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy()));
324 _TestUtil.reduceOpenFiles(w.w);
326 final AtomicBoolean failed = new AtomicBoolean();
327 Thread[] threads = new Thread[NUM_THREADS];
328 final long endTime = System.currentTimeMillis()+((long) (RUN_SEC*1000));
329 for(int i=0;i<NUM_THREADS;i++) {
330 final int finalI = i;
331 threads[i] = new Thread() {
335 final Document doc = new Document();
336 IndexReader r = IndexReader.open(dir);
337 Field f = newField("f", "", Field.Store.NO, Field.Index.NOT_ANALYZED);
341 if (failed.get()) break;
342 for(int j=0;j<10;j++) {
343 final String s = finalI + "_" + String.valueOf(count++);
347 IndexReader r2 = r.reopen();
351 assertEquals("term=f:" + s + "; r=" + r, 1, r.docFreq(new Term("f", s)));
353 } while(System.currentTimeMillis() < endTime);
355 } catch (Throwable t) {
357 throw new RuntimeException(t);
363 for(int i=0;i<NUM_THREADS;i++) {
366 assertFalse(failed.get());
371 // LUCENE-1044: test writer.commit() when ac=false
372 public void testForceCommit() throws IOException {
373 Directory dir = newDirectory();
375 IndexWriter writer = new IndexWriter(
377 newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
378 setMaxBufferedDocs(2).
379 setMergePolicy(newLogMergePolicy(5))
383 for (int i = 0; i < 23; i++)
384 TestIndexWriter.addDoc(writer);
386 IndexReader reader = IndexReader.open(dir, true);
387 assertEquals(0, reader.numDocs());
389 IndexReader reader2 = reader.reopen();
390 assertEquals(0, reader.numDocs());
391 assertEquals(23, reader2.numDocs());
394 for (int i = 0; i < 17; i++)
395 TestIndexWriter.addDoc(writer);
396 assertEquals(23, reader2.numDocs());
398 reader = IndexReader.open(dir, true);
399 assertEquals(23, reader.numDocs());
403 reader = IndexReader.open(dir, true);
404 assertEquals(40, reader.numDocs());
410 public void testFutureCommit() throws Exception {
411 Directory dir = newDirectory();
413 IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE));
414 Document doc = new Document();
418 Map<String,String> commitData = new HashMap<String,String>();
419 commitData.put("tag", "first");
420 w.commit(commitData);
422 // commit to "second"
424 commitData.put("tag", "second");
425 w.commit(commitData);
428 // open "first" with IndexWriter
429 IndexCommit commit = null;
430 for(IndexCommit c : IndexReader.listCommits(dir)) {
431 if (c.getUserData().get("tag").equals("first")) {
437 assertNotNull(commit);
439 w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE).setIndexCommit(commit));
441 assertEquals(1, w.numDocs());
443 // commit IndexWriter to "third"
445 commitData.put("tag", "third");
446 w.commit(commitData);
449 // make sure "second" commit is still there
451 for(IndexCommit c : IndexReader.listCommits(dir)) {
452 if (c.getUserData().get("tag").equals("second")) {
458 assertNotNull(commit);
460 IndexReader r = IndexReader.open(commit, true);
461 assertEquals(2, r.numDocs());
464 // open "second", w/ writeable IndexReader & commit
465 r = IndexReader.open(commit, NoDeletionPolicy.INSTANCE, false);
466 assertEquals(2, r.numDocs());
469 commitData.put("tag", "fourth");
470 r.commit(commitData);
473 // make sure "third" commit is still there
475 for(IndexCommit c : IndexReader.listCommits(dir)) {
476 if (c.getUserData().get("tag").equals("third")) {
481 assertNotNull(commit);
486 public void testNoCommits() throws Exception {
487 // Tests that if we don't call commit(), the directory has 0 commits. This has
488 // changed since LUCENE-2386, where before IW would always commit on a fresh
490 Directory dir = newDirectory();
491 IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
493 IndexReader.listCommits(dir);
494 fail("listCommits should have thrown an exception over empty index");
495 } catch (IndexNotFoundException e) {
498 // No changes still should generate a commit, because it's a new index.
500 assertEquals("expected 1 commits!", 1, IndexReader.listCommits(dir).size());
504 // LUCENE-1274: test writer.prepareCommit()
505 public void testPrepareCommit() throws IOException {
506 Directory dir = newDirectory();
508 IndexWriter writer = new IndexWriter(
510 newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
511 setMaxBufferedDocs(2).
512 setMergePolicy(newLogMergePolicy(5))
516 for (int i = 0; i < 23; i++)
517 TestIndexWriter.addDoc(writer);
519 IndexReader reader = IndexReader.open(dir, true);
520 assertEquals(0, reader.numDocs());
522 writer.prepareCommit();
524 IndexReader reader2 = IndexReader.open(dir, true);
525 assertEquals(0, reader2.numDocs());
529 IndexReader reader3 = reader.reopen();
530 assertEquals(0, reader.numDocs());
531 assertEquals(0, reader2.numDocs());
532 assertEquals(23, reader3.numDocs());
536 for (int i = 0; i < 17; i++)
537 TestIndexWriter.addDoc(writer);
539 assertEquals(23, reader3.numDocs());
541 reader = IndexReader.open(dir, true);
542 assertEquals(23, reader.numDocs());
545 writer.prepareCommit();
547 reader = IndexReader.open(dir, true);
548 assertEquals(23, reader.numDocs());
552 reader = IndexReader.open(dir, true);
553 assertEquals(40, reader.numDocs());
559 // LUCENE-1274: test writer.prepareCommit()
560 public void testPrepareCommitRollback() throws IOException {
561 MockDirectoryWrapper dir = newDirectory();
562 dir.setPreventDoubleWrite(false);
564 IndexWriter writer = new IndexWriter(
566 newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
567 setMaxBufferedDocs(2).
568 setMergePolicy(newLogMergePolicy(5))
572 for (int i = 0; i < 23; i++)
573 TestIndexWriter.addDoc(writer);
575 IndexReader reader = IndexReader.open(dir, true);
576 assertEquals(0, reader.numDocs());
578 writer.prepareCommit();
580 IndexReader reader2 = IndexReader.open(dir, true);
581 assertEquals(0, reader2.numDocs());
585 IndexReader reader3 = reader.reopen();
586 assertEquals(0, reader.numDocs());
587 assertEquals(0, reader2.numDocs());
588 assertEquals(0, reader3.numDocs());
592 writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
593 for (int i = 0; i < 17; i++)
594 TestIndexWriter.addDoc(writer);
596 assertEquals(0, reader3.numDocs());
598 reader = IndexReader.open(dir, true);
599 assertEquals(0, reader.numDocs());
602 writer.prepareCommit();
604 reader = IndexReader.open(dir, true);
605 assertEquals(0, reader.numDocs());
609 reader = IndexReader.open(dir, true);
610 assertEquals(17, reader.numDocs());
617 public void testPrepareCommitNoChanges() throws IOException {
618 Directory dir = newDirectory();
620 IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
621 writer.prepareCommit();
625 IndexReader reader = IndexReader.open(dir, true);
626 assertEquals(0, reader.numDocs());
632 public void testCommitUserData() throws IOException {
633 Directory dir = newDirectory();
634 IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(2));
635 for(int j=0;j<17;j++)
636 TestIndexWriter.addDoc(w);
639 assertEquals(0, IndexReader.getCommitUserData(dir).size());
641 IndexReader r = IndexReader.open(dir, true);
642 // commit(Map) never called for this index
643 assertEquals(0, r.getCommitUserData().size());
646 w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(2));
647 for(int j=0;j<17;j++)
648 TestIndexWriter.addDoc(w);
649 Map<String,String> data = new HashMap<String,String>();
650 data.put("label", "test1");
654 assertEquals("test1", IndexReader.getCommitUserData(dir).get("label"));
656 r = IndexReader.open(dir, true);
657 assertEquals("test1", r.getCommitUserData().get("label"));
660 w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
664 assertEquals("test1", IndexReader.getCommitUserData(dir).get("label"));