1 package org.apache.lucene.index;
4 * Licensed to the Apache Software Foundation (ASF) under one or more
5 * contributor license agreements. See the NOTICE file distributed with
6 * this work for additional information regarding copyright ownership.
7 * The ASF licenses this file to You under the Apache License, Version 2.0
8 * (the "License"); you may not use this file except in compliance with
9 * the License. You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
20 import java.io.IOException;
22 import org.apache.lucene.analysis.MockAnalyzer;
23 import org.apache.lucene.document.Document;
24 import org.apache.lucene.document.Field;
25 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
26 import org.apache.lucene.search.IndexSearcher;
27 import org.apache.lucene.search.ScoreDoc;
28 import org.apache.lucene.search.TermQuery;
29 import org.apache.lucene.store.Directory;
30 import org.apache.lucene.store.MockDirectoryWrapper;
31 import org.apache.lucene.store.RAMDirectory;
32 import org.apache.lucene.util.LuceneTestCase;
33 import org.apache.lucene.util._TestUtil;
35 import static org.apache.lucene.index.TestIndexWriter.assertNoUnreferencedFiles;
38 * Tests for IndexWriter when the disk runs out of space
40 public class TestIndexWriterOnDiskFull extends LuceneTestCase {
43 * Make sure IndexWriter cleans up on hitting a disk
44 * full exception in addDocument.
45 * TODO: how to do this on windows with FSDirectory?
47 public void testAddDocumentOnDiskFull() throws IOException {
49 for(int pass=0;pass<2;pass++) {
51 System.out.println("TEST: pass=" + pass);
53 boolean doAbort = pass == 1;
54 long diskFree = _TestUtil.nextInt(random, 100, 300);
57 System.out.println("TEST: cycle: diskFree=" + diskFree);
59 MockDirectoryWrapper dir = new MockDirectoryWrapper(random, new RAMDirectory());
60 dir.setMaxSizeInBytes(diskFree);
61 IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
62 writer.setInfoStream(VERBOSE ? System.out : null);
63 MergeScheduler ms = writer.getConfig().getMergeScheduler();
64 if (ms instanceof ConcurrentMergeScheduler) {
65 // This test intentionally produces exceptions
66 // in the threads that CMS launches; we don't
67 // want to pollute test output with these.
68 ((ConcurrentMergeScheduler) ms).setSuppressExceptions();
71 boolean hitError = false;
73 for(int i=0;i<200;i++) {
77 System.out.println("TEST: done adding docs; now commit");
80 } catch (IOException e) {
82 System.out.println("TEST: exception on addDoc");
83 e.printStackTrace(System.out);
91 System.out.println("TEST: now rollback");
97 System.out.println("TEST: now close");
100 } catch (IOException e) {
102 System.out.println("TEST: exception on close; retry w/ no disk space limit");
103 e.printStackTrace(System.out);
105 dir.setMaxSizeInBytes(0);
110 //_TestUtil.syncConcurrentMerges(ms);
112 if (_TestUtil.anyFilesExceptWriteLock(dir)) {
113 assertNoUnreferencedFiles(dir, "after disk full during addDocument");
115 // Make sure reader can open the index:
116 IndexReader.open(dir, true).close();
120 // Now try again w/ more space:
122 diskFree += TEST_NIGHTLY ? _TestUtil.nextInt(random, 400, 600) : _TestUtil.nextInt(random, 3000, 5000);
124 //_TestUtil.syncConcurrentMerges(writer);
125 dir.setMaxSizeInBytes(0);
134 // TODO: make @Nightly variant that provokes more disk
137 // TODO: have test fail if on any given top
138 // iter there was not a single IOE hit
141 Test: make sure when we run out of disk space or hit
142 random IOExceptions in any of the addIndexes(*) calls
143 that 1) index is not corrupt (searcher can open/search
144 it) and 2) transactional semantics are followed:
145 either all or none of the incoming documents were in
148 public void testAddIndexOnDiskFull() throws IOException
150 int START_COUNT = 57;
152 int END_COUNT = START_COUNT + NUM_DIR*25;
154 // Build up a bunch of dirs that have indexes which we
155 // will then merge together by calling addIndexes(*):
156 Directory[] dirs = new Directory[NUM_DIR];
157 long inputDiskUsage = 0;
158 for(int i=0;i<NUM_DIR;i++) {
159 dirs[i] = newDirectory();
160 IndexWriter writer = new IndexWriter(dirs[i], newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
161 for(int j=0;j<25;j++) {
162 addDocWithIndex(writer, 25*i+j);
165 String[] files = dirs[i].listAll();
166 for(int j=0;j<files.length;j++) {
167 inputDiskUsage += dirs[i].fileLength(files[j]);
171 // Now, build a starting index that has START_COUNT docs. We
172 // will then try to addIndexesNoOptimize into a copy of this:
173 MockDirectoryWrapper startDir = newDirectory();
174 IndexWriter writer = new IndexWriter(startDir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
175 for(int j=0;j<START_COUNT;j++) {
176 addDocWithIndex(writer, j);
180 // Make sure starting index seems to be working properly:
181 Term searchTerm = new Term("content", "aaa");
182 IndexReader reader = IndexReader.open(startDir, true);
183 assertEquals("first docFreq", 57, reader.docFreq(searchTerm));
185 IndexSearcher searcher = newSearcher(reader);
186 ScoreDoc[] hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
187 assertEquals("first number of hits", 57, hits.length);
191 // Iterate with larger and larger amounts of free
192 // disk space. With little free disk space,
193 // addIndexes will certainly run out of space &
194 // fail. Verify that when this happens, index is
195 // not corrupt and index in fact has added no
196 // documents. Then, we increase disk space by 2000
197 // bytes each iteration. At some point there is
198 // enough free disk space and addIndexes should
199 // succeed and index should show all documents were
202 // String[] files = startDir.listAll();
203 long diskUsage = startDir.sizeInBytes();
205 long startDiskUsage = 0;
206 String[] files = startDir.listAll();
207 for(int i=0;i<files.length;i++) {
208 startDiskUsage += startDir.fileLength(files[i]);
211 for(int iter=0;iter<3;iter++) {
214 System.out.println("TEST: iter=" + iter);
216 // Start with 100 bytes more than we are currently using:
217 long diskFree = diskUsage+_TestUtil.nextInt(random, 50, 200);
221 boolean success = false;
222 boolean done = false;
226 methodName = "addIndexes(Directory[]) + optimize()";
227 } else if (1 == method) {
228 methodName = "addIndexes(IndexReader[])";
230 methodName = "addIndexes(Directory[])";
235 System.out.println("TEST: cycle...");
238 // Make a new dir that will enforce disk usage:
239 MockDirectoryWrapper dir = new MockDirectoryWrapper(random, new RAMDirectory(startDir));
240 writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND).setMergePolicy(newLogMergePolicy()));
241 IOException err = null;
242 writer.setInfoStream(VERBOSE ? System.out : null);
244 MergeScheduler ms = writer.getConfig().getMergeScheduler();
245 for(int x=0;x<2;x++) {
246 if (ms instanceof ConcurrentMergeScheduler)
247 // This test intentionally produces exceptions
248 // in the threads that CMS launches; we don't
249 // want to pollute test output with these.
251 ((ConcurrentMergeScheduler) ms).setSuppressExceptions();
253 ((ConcurrentMergeScheduler) ms).clearSuppressExceptions();
255 // Two loops: first time, limit disk space &
256 // throw random IOExceptions; second time, no
260 double diskRatio = ((double) diskFree)/diskUsage;
263 String testName = null;
266 thisDiskFree = diskFree;
267 if (diskRatio >= 2.0) {
270 if (diskRatio >= 4.0) {
273 if (diskRatio >= 6.0) {
277 testName = "disk full test " + methodName + " with disk full at " + diskFree + " bytes";
282 testName = "disk full test " + methodName + " with unlimited disk space";
286 System.out.println("\ncycle: " + testName);
288 dir.setTrackDiskUsage(true);
289 dir.setMaxSizeInBytes(thisDiskFree);
290 dir.setRandomIOExceptionRate(rate);
295 writer.addIndexes(dirs);
297 } else if (1 == method) {
298 IndexReader readers[] = new IndexReader[dirs.length];
299 for(int i=0;i<dirs.length;i++) {
300 readers[i] = IndexReader.open(dirs[i], true);
303 writer.addIndexes(readers);
305 for(int i=0;i<dirs.length;i++) {
310 writer.addIndexes(dirs);
315 System.out.println(" success!");
322 } catch (IOException e) {
326 System.out.println(" hit IOException: " + e);
327 e.printStackTrace(System.out);
331 e.printStackTrace(System.out);
332 fail(methodName + " hit IOException after disk space was freed up");
336 // Make sure all threads from
337 // ConcurrentMergeScheduler are done
338 _TestUtil.syncConcurrentMerges(writer);
341 System.out.println(" now test readers");
344 // Finally, verify index is not corrupt, and, if
345 // we succeeded, we see all docs added, and if we
346 // failed, we see either all docs or no docs added
347 // (transactional semantics):
349 reader = IndexReader.open(dir, true);
350 } catch (IOException e) {
351 e.printStackTrace(System.out);
352 fail(testName + ": exception when creating IndexReader: " + e);
354 int result = reader.docFreq(searchTerm);
356 if (result != START_COUNT) {
357 fail(testName + ": method did not throw exception but docFreq('aaa') is " + result + " instead of expected " + START_COUNT);
360 // On hitting exception we still may have added
362 if (result != START_COUNT && result != END_COUNT) {
363 err.printStackTrace(System.out);
364 fail(testName + ": method did throw exception but docFreq('aaa') is " + result + " instead of expected " + START_COUNT + " or " + END_COUNT);
368 searcher = newSearcher(reader);
370 hits = searcher.search(new TermQuery(searchTerm), null, END_COUNT).scoreDocs;
371 } catch (IOException e) {
372 e.printStackTrace(System.out);
373 fail(testName + ": exception when searching: " + e);
375 int result2 = hits.length;
377 if (result2 != result) {
378 fail(testName + ": method did not throw exception but hits.length for search on term 'aaa' is " + result2 + " instead of expected " + result);
381 // On hitting exception we still may have added
383 if (result2 != result) {
384 err.printStackTrace(System.out);
385 fail(testName + ": method did throw exception but hits.length for search on term 'aaa' is " + result2 + " instead of expected " + result);
392 System.out.println(" count is " + result);
395 if (done || result == END_COUNT) {
401 System.out.println(" start disk = " + startDiskUsage + "; input disk = " + inputDiskUsage + "; max used = " + dir.getMaxUsedSizeInBytes());
405 // Javadocs state that temp free Directory space
406 // required is at most 2X total input size of
407 // indices so let's make sure:
408 assertTrue("max free Directory space required exceeded 1X the total input index sizes during " + methodName +
409 ": max temp usage = " + (dir.getMaxUsedSizeInBytes()-startDiskUsage) + " bytes vs limit=" + (2*(startDiskUsage + inputDiskUsage)) +
410 "; starting disk usage = " + startDiskUsage + " bytes; " +
411 "input index disk usage = " + inputDiskUsage + " bytes",
412 (dir.getMaxUsedSizeInBytes()-startDiskUsage) < 2*(startDiskUsage + inputDiskUsage));
415 // Make sure we don't hit disk full during close below:
416 dir.setMaxSizeInBytes(0);
417 dir.setRandomIOExceptionRate(0.0);
421 // Wait for all BG threads to finish else
422 // dir.close() will throw IOException because
423 // there are still open files
424 _TestUtil.syncConcurrentMerges(ms);
428 // Try again with more free space:
429 diskFree += TEST_NIGHTLY ? _TestUtil.nextInt(random, 4000, 8000) : _TestUtil.nextInt(random, 40000, 80000);
434 for (Directory dir : dirs)
438 private static class FailTwiceDuringMerge extends MockDirectoryWrapper.Failure {
439 public boolean didFail1;
440 public boolean didFail2;
443 public void eval(MockDirectoryWrapper dir) throws IOException {
447 StackTraceElement[] trace = new Exception().getStackTrace();
448 for (int i = 0; i < trace.length; i++) {
449 if ("org.apache.lucene.index.SegmentMerger".equals(trace[i].getClassName()) && "mergeTerms".equals(trace[i].getMethodName()) && !didFail1) {
451 throw new IOException("fake disk full during mergeTerms");
453 if ("org.apache.lucene.util.BitVector".equals(trace[i].getClassName()) && "write".equals(trace[i].getMethodName()) && !didFail2) {
455 throw new IOException("fake disk full while writing BitVector");
462 public void testCorruptionAfterDiskFullDuringMerge() throws IOException {
463 MockDirectoryWrapper dir = newDirectory();
464 //IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setReaderPooling(true));
465 IndexWriter w = new IndexWriter(
467 newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
468 setMergeScheduler(new SerialMergeScheduler()).
469 setReaderPooling(true).
470 setMergePolicy(newLogMergePolicy(2))
473 _TestUtil.keepFullyDeletedSegments(w);
475 ((LogMergePolicy) w.getMergePolicy()).setMergeFactor(2);
477 Document doc = new Document();
478 doc.add(newField("f", "doctor who", Field.Store.YES, Field.Index.ANALYZED));
482 w.deleteDocuments(new Term("f", "who"));
486 FailTwiceDuringMerge ftdm = new FailTwiceDuringMerge();
492 fail("fake disk full IOExceptions not hit");
493 } catch (IOException ioe) {
495 assertTrue(ftdm.didFail1 || ftdm.didFail2);
497 _TestUtil.checkIndex(dir);
505 // LUCENE-1130: make sure immeidate disk full on creating
506 // an IndexWriter (hit during DW.ThreadState.init()) is
508 public void testImmediateDiskFull() throws IOException {
509 MockDirectoryWrapper dir = newDirectory();
510 IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))
511 .setMaxBufferedDocs(2).setMergeScheduler(new ConcurrentMergeScheduler()));
512 dir.setMaxSizeInBytes(Math.max(1, dir.getRecomputedActualSizeInBytes()));
513 final Document doc = new Document();
514 doc.add(newField("field", "aaa bbb ccc ddd eee fff ggg hhh iii jjj", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
516 writer.addDocument(doc);
517 fail("did not hit disk full");
518 } catch (IOException ioe) {
520 // Without fix for LUCENE-1130: this call will hang:
522 writer.addDocument(doc);
523 fail("did not hit disk full");
524 } catch (IOException ioe) {
528 fail("did not hit disk full");
529 } catch (IOException ioe) {
532 // Make sure once disk space is avail again, we can
534 dir.setMaxSizeInBytes(0);
539 // TODO: these are also in TestIndexWriter... add a simple doc-writing method
540 // like this to LuceneTestCase?
541 private void addDoc(IndexWriter writer) throws IOException
543 Document doc = new Document();
544 doc.add(newField("content", "aaa", Field.Store.NO, Field.Index.ANALYZED));
545 writer.addDocument(doc);
548 private void addDocWithIndex(IndexWriter writer, int index) throws IOException
550 Document doc = new Document();
551 doc.add(newField("content", "aaa " + index, Field.Store.YES, Field.Index.ANALYZED));
552 doc.add(newField("id", "" + index, Field.Store.YES, Field.Index.ANALYZED));
553 writer.addDocument(doc);