1 package org.apache.lucene.index;
4 * Licensed to the Apache Software Foundation (ASF) under one or more
5 * contributor license agreements. See the NOTICE file distributed with
6 * this work for additional information regarding copyright ownership.
7 * The ASF licenses this file to You under the Apache License, Version 2.0
8 * (the "License"); you may not use this file except in compliance with
9 * the License. You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
20 import java.io.ByteArrayOutputStream;
21 import java.io.IOException;
22 import java.io.PrintStream;
23 import java.io.Reader;
24 import java.io.StringReader;
25 import java.util.ArrayList;
26 import java.util.List;
27 import java.util.Random;
29 import org.apache.lucene.util.LuceneTestCase;
30 import org.apache.lucene.util._TestUtil;
31 import org.apache.lucene.store.Directory;
32 import org.apache.lucene.store.IndexInput;
33 import org.apache.lucene.store.IndexOutput;
34 import org.apache.lucene.store.MockDirectoryWrapper;
35 import org.apache.lucene.store.RAMDirectory;
36 import org.apache.lucene.analysis.Analyzer;
37 import org.apache.lucene.analysis.MockAnalyzer;
38 import org.apache.lucene.analysis.MockTokenizer;
39 import org.apache.lucene.analysis.TokenFilter;
40 import org.apache.lucene.analysis.TokenStream;
41 import org.apache.lucene.document.Document;
42 import org.apache.lucene.document.Field;
43 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
44 import org.apache.lucene.search.IndexSearcher;
45 import org.apache.lucene.search.PhraseQuery;
47 public class TestIndexWriterExceptions extends LuceneTestCase {
49 private class IndexerThread extends Thread {
53 final Random r = new Random(random.nextLong());
54 volatile Throwable failure;
56 public IndexerThread(int i, IndexWriter writer) {
57 setName("Indexer " + i);
64 final Document doc = new Document();
66 doc.add(newField(r, "content1", "aaa bbb ccc ddd", Field.Store.YES, Field.Index.ANALYZED));
67 doc.add(newField(r, "content6", "aaa bbb ccc ddd", Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
68 doc.add(newField(r, "content2", "aaa bbb ccc ddd", Field.Store.YES, Field.Index.NOT_ANALYZED));
69 doc.add(newField(r, "content3", "aaa bbb ccc ddd", Field.Store.YES, Field.Index.NO));
71 doc.add(newField(r, "content4", "aaa bbb ccc ddd", Field.Store.NO, Field.Index.ANALYZED));
72 doc.add(newField(r, "content5", "aaa bbb ccc ddd", Field.Store.NO, Field.Index.NOT_ANALYZED));
74 doc.add(newField(r, "content7", "aaa bbb ccc ddd", Field.Store.NO, Field.Index.NOT_ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
76 final Field idField = newField(r, "id", "", Field.Store.YES, Field.Index.NOT_ANALYZED);
79 final long stopTime = System.currentTimeMillis() + 500;
83 System.out.println(Thread.currentThread().getName() + ": TEST: IndexerThread: cycle");
86 final String id = ""+r.nextInt(50);
88 Term idTerm = new Term("id", id);
90 if (r.nextBoolean()) {
91 final List<Document> docs = new ArrayList<Document>();
92 final int count = _TestUtil.nextInt(r, 1, 20);
93 for(int c=0;c<count;c++) {
96 writer.updateDocuments(idTerm, docs);
98 writer.updateDocument(idTerm, doc);
100 } catch (RuntimeException re) {
102 System.out.println(Thread.currentThread().getName() + ": EXC: ");
103 re.printStackTrace(System.out);
106 _TestUtil.checkIndex(writer.getDirectory());
107 } catch (IOException ioe) {
108 System.out.println(Thread.currentThread().getName() + ": unexpected exception1");
109 ioe.printStackTrace(System.out);
113 } catch (Throwable t) {
114 System.out.println(Thread.currentThread().getName() + ": unexpected exception2");
115 t.printStackTrace(System.out);
122 // After a possible exception (above) I should be able
123 // to add a new document without hitting an
126 writer.updateDocument(idTerm, doc);
127 } catch (Throwable t) {
128 System.out.println(Thread.currentThread().getName() + ": unexpected exception3");
129 t.printStackTrace(System.out);
133 } while(System.currentTimeMillis() < stopTime);
137 ThreadLocal<Thread> doFail = new ThreadLocal<Thread>();
139 private class MockIndexWriter extends IndexWriter {
140 Random r = new Random(random.nextLong());
142 public MockIndexWriter(Directory dir, IndexWriterConfig conf) throws IOException {
147 boolean testPoint(String name) {
148 if (doFail.get() != null && !name.equals("startDoFlush") && r.nextInt(40) == 17) {
150 System.out.println(Thread.currentThread().getName() + ": NOW FAIL: " + name);
151 new Throwable().printStackTrace(System.out);
153 throw new RuntimeException(Thread.currentThread().getName() + ": intentionally failing at " + name);
159 public void testRandomExceptions() throws Throwable {
161 System.out.println("\nTEST: start testRandomExceptions");
163 MockDirectoryWrapper dir = newDirectory();
165 MockAnalyzer analyzer = new MockAnalyzer(random);
166 analyzer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
167 MockIndexWriter writer = new MockIndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer)
168 .setRAMBufferSizeMB(0.1).setMergeScheduler(new ConcurrentMergeScheduler()));
169 ((ConcurrentMergeScheduler) writer.getConfig().getMergeScheduler()).setSuppressExceptions();
170 //writer.setMaxBufferedDocs(10);
172 System.out.println("TEST: initial commit");
177 writer.setInfoStream(System.out);
180 IndexerThread thread = new IndexerThread(0, writer);
182 if (thread.failure != null) {
183 thread.failure.printStackTrace(System.out);
184 fail("thread " + thread.getName() + ": hit unexpected failure");
188 System.out.println("TEST: commit after thread start");
194 } catch (Throwable t) {
195 System.out.println("exception during close:");
196 t.printStackTrace(System.out);
200 // Confirm that when doc hits exception partway through tokenization, it's deleted:
201 IndexReader r2 = IndexReader.open(dir, true);
202 final int count = r2.docFreq(new Term("content4", "aaa"));
203 final int count2 = r2.docFreq(new Term("content4", "ddd"));
204 assertEquals(count, count2);
210 public void testRandomExceptionsThreads() throws Throwable {
211 MockDirectoryWrapper dir = newDirectory();
212 MockAnalyzer analyzer = new MockAnalyzer(random);
213 analyzer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
214 MockIndexWriter writer = new MockIndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer)
215 .setRAMBufferSizeMB(0.2).setMergeScheduler(new ConcurrentMergeScheduler()));
216 ((ConcurrentMergeScheduler) writer.getConfig().getMergeScheduler()).setSuppressExceptions();
217 //writer.setMaxBufferedDocs(10);
221 writer.setInfoStream(System.out);
224 final int NUM_THREADS = 4;
226 final IndexerThread[] threads = new IndexerThread[NUM_THREADS];
227 for(int i=0;i<NUM_THREADS;i++) {
228 threads[i] = new IndexerThread(i, writer);
232 for(int i=0;i<NUM_THREADS;i++)
235 for(int i=0;i<NUM_THREADS;i++)
236 if (threads[i].failure != null)
237 fail("thread " + threads[i].getName() + ": hit unexpected failure");
243 } catch (Throwable t) {
244 System.out.println("exception during close:");
245 t.printStackTrace(System.out);
249 // Confirm that when doc hits exception partway through tokenization, it's deleted:
250 IndexReader r2 = IndexReader.open(dir, true);
251 final int count = r2.docFreq(new Term("content4", "aaa"));
252 final int count2 = r2.docFreq(new Term("content4", "ddd"));
253 assertEquals(count, count2);
260 private static final class MockIndexWriter2 extends IndexWriter {
262 public MockIndexWriter2(Directory dir, IndexWriterConfig conf) throws IOException {
269 boolean testPoint(String name) {
270 if (doFail && name.equals("DocumentsWriter.ThreadState.init start"))
271 throw new RuntimeException("intentionally failing");
276 private static String CRASH_FAIL_MESSAGE = "I'm experiencing problems";
278 private class CrashingFilter extends TokenFilter {
282 public CrashingFilter(String fieldName, TokenStream input) {
284 this.fieldName = fieldName;
288 public boolean incrementToken() throws IOException {
289 if (this.fieldName.equals("crash") && count++ >= 4)
290 throw new IOException(CRASH_FAIL_MESSAGE);
291 return input.incrementToken();
295 public void reset() throws IOException {
301 public void testExceptionDocumentsWriterInit() throws IOException {
302 Directory dir = newDirectory();
303 MockIndexWriter2 w = new MockIndexWriter2(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
304 w.setInfoStream(VERBOSE ? System.out : null);
305 Document doc = new Document();
306 doc.add(newField("field", "a field", Field.Store.YES,
307 Field.Index.ANALYZED));
312 fail("did not hit exception");
313 } catch (RuntimeException re) {
321 public void testExceptionJustBeforeFlush() throws IOException {
322 Directory dir = newDirectory();
323 MockIndexWriter w = new MockIndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(2));
324 w.setInfoStream(VERBOSE ? System.out : null);
325 Document doc = new Document();
326 doc.add(newField("field", "a field", Field.Store.YES,
327 Field.Index.ANALYZED));
330 Analyzer analyzer = new Analyzer() {
332 public TokenStream tokenStream(String fieldName, Reader reader) {
333 MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
334 tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
335 return new CrashingFilter(fieldName, tokenizer);
339 Document crashDoc = new Document();
340 crashDoc.add(newField("crash", "do it on token 4", Field.Store.YES,
341 Field.Index.ANALYZED));
343 w.addDocument(crashDoc, analyzer);
344 fail("did not hit expected exception");
345 } catch (IOException ioe) {
353 private static final class MockIndexWriter3 extends IndexWriter {
355 public MockIndexWriter3(Directory dir, IndexWriterConfig conf) throws IOException {
363 boolean testPoint(String name) {
364 if (doFail && name.equals("startMergeInit")) {
366 throw new RuntimeException("intentionally failing");
374 public void testExceptionOnMergeInit() throws IOException {
375 Directory dir = newDirectory();
376 IndexWriterConfig conf = newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))
377 .setMaxBufferedDocs(2).setMergeScheduler(new ConcurrentMergeScheduler()).setMergePolicy(newLogMergePolicy());
378 ((LogMergePolicy) conf.getMergePolicy()).setMergeFactor(2);
379 MockIndexWriter3 w = new MockIndexWriter3(dir, conf);
381 Document doc = new Document();
382 doc.add(newField("field", "a field", Field.Store.YES,
383 Field.Index.ANALYZED));
384 for(int i=0;i<10;i++)
387 } catch (RuntimeException re) {
391 ((ConcurrentMergeScheduler) w.getConfig().getMergeScheduler()).sync();
392 assertTrue(w.failed);
398 public void testExceptionFromTokenStream() throws IOException {
399 Directory dir = newDirectory();
400 IndexWriterConfig conf = newIndexWriterConfig( TEST_VERSION_CURRENT, new Analyzer() {
403 public TokenStream tokenStream(String fieldName, Reader reader) {
404 MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
405 tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
406 return new TokenFilter(tokenizer) {
407 private int count = 0;
410 public boolean incrementToken() throws IOException {
412 throw new IOException();
414 return input.incrementToken();
420 conf.setMaxBufferedDocs(Math.max(3, conf.getMaxBufferedDocs()));
422 IndexWriter writer = new IndexWriter(dir, conf);
424 Document doc = new Document();
425 String contents = "aa bb cc dd ee ff gg hh ii jj kk";
426 doc.add(newField("content", contents, Field.Store.NO,
427 Field.Index.ANALYZED));
429 writer.addDocument(doc);
430 fail("did not hit expected exception");
431 } catch (Exception e) {
434 // Make sure we can add another normal document
435 doc = new Document();
436 doc.add(newField("content", "aa bb cc dd", Field.Store.NO,
437 Field.Index.ANALYZED));
438 writer.addDocument(doc);
440 // Make sure we can add another normal document
441 doc = new Document();
442 doc.add(newField("content", "aa bb cc dd", Field.Store.NO,
443 Field.Index.ANALYZED));
444 writer.addDocument(doc);
447 IndexReader reader = IndexReader.open(dir, true);
448 final Term t = new Term("content", "aa");
449 assertEquals(3, reader.docFreq(t));
451 // Make sure the doc that hit the exception was marked
453 TermDocs tdocs = reader.termDocs(t);
455 while(tdocs.next()) {
458 assertEquals(2, count);
460 assertEquals(reader.docFreq(new Term("content", "gg")), 0);
465 private static class FailOnlyOnFlush extends MockDirectoryWrapper.Failure {
466 boolean doFail = false;
470 public void setDoFail() {
474 public void clearDoFail() {
479 public void eval(MockDirectoryWrapper dir) throws IOException {
481 StackTraceElement[] trace = new Exception().getStackTrace();
482 boolean sawAppend = false;
483 boolean sawFlush = false;
484 for (int i = 0; i < trace.length; i++) {
485 if ("org.apache.lucene.index.FreqProxTermsWriter".equals(trace[i].getClassName()) && "appendPostings".equals(trace[i].getMethodName()))
487 if ("doFlush".equals(trace[i].getMethodName()))
491 if (sawAppend && sawFlush && count++ >= 30) {
493 throw new IOException("now failing during flush");
499 // LUCENE-1072: make sure an errant exception on flushing
500 // one segment only takes out those docs in that one flush
501 public void testDocumentsWriterAbort() throws IOException {
502 MockDirectoryWrapper dir = newDirectory();
503 FailOnlyOnFlush failure = new FailOnlyOnFlush();
507 IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(2));
508 Document doc = new Document();
509 String contents = "aa bb cc dd ee ff gg hh ii jj kk";
510 doc.add(newField("content", contents, Field.Store.NO,
511 Field.Index.ANALYZED));
512 boolean hitError = false;
513 for(int i=0;i<200;i++) {
515 writer.addDocument(doc);
516 } catch (IOException ioe) {
517 // only one flush should fail:
518 assertFalse(hitError);
522 assertTrue(hitError);
524 IndexReader reader = IndexReader.open(dir, true);
525 assertEquals(198, reader.docFreq(new Term("content", "aa")));
530 public void testDocumentsWriterExceptions() throws IOException {
531 Analyzer analyzer = new Analyzer() {
533 public TokenStream tokenStream(String fieldName, Reader reader) {
534 MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
535 tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
536 return new CrashingFilter(fieldName, tokenizer);
540 for(int i=0;i<2;i++) {
542 System.out.println("TEST: cycle i=" + i);
544 MockDirectoryWrapper dir = newDirectory();
545 IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer).setMergePolicy(newLogMergePolicy()));
546 writer.setInfoStream(VERBOSE ? System.out : null);
548 // don't allow a sudden merge to clean up the deleted
550 LogMergePolicy lmp = (LogMergePolicy) writer.getConfig().getMergePolicy();
551 lmp.setMergeFactor(Math.max(lmp.getMergeFactor(), 5));
553 Document doc = new Document();
554 doc.add(newField("contents", "here are some contents", Field.Store.YES,
555 Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
556 writer.addDocument(doc);
557 writer.addDocument(doc);
558 doc.add(newField("crash", "this should crash after 4 terms", Field.Store.YES,
559 Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
560 doc.add(newField("other", "this will not get indexed", Field.Store.YES,
561 Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
563 writer.addDocument(doc);
564 fail("did not hit expected exception");
565 } catch (IOException ioe) {
567 System.out.println("TEST: hit expected exception");
568 ioe.printStackTrace(System.out);
573 doc = new Document();
574 doc.add(newField("contents", "here are some contents", Field.Store.YES,
575 Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
576 writer.addDocument(doc);
577 writer.addDocument(doc);
582 System.out.println("TEST: open reader");
584 IndexReader reader = IndexReader.open(dir, true);
587 assertEquals(expected, reader.docFreq(new Term("contents", "here")));
588 assertEquals(expected, reader.maxDoc());
590 for(int j=0;j<reader.maxDoc();j++) {
591 if (reader.isDeleted(j))
595 reader.getTermFreqVectors(j);
598 assertEquals(1, numDel);
602 writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT,
603 analyzer).setMaxBufferedDocs(10));
604 doc = new Document();
605 doc.add(newField("contents", "here are some contents", Field.Store.YES,
606 Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
607 for(int j=0;j<17;j++)
608 writer.addDocument(doc);
612 reader = IndexReader.open(dir, true);
613 int expected = 19+(1-i)*2;
614 assertEquals(expected, reader.docFreq(new Term("contents", "here")));
615 assertEquals(expected, reader.maxDoc());
617 for(int j=0;j<reader.maxDoc();j++) {
618 if (reader.isDeleted(j))
622 reader.getTermFreqVectors(j);
626 assertEquals(0, numDel);
632 public void testDocumentsWriterExceptionThreads() throws Exception {
633 Analyzer analyzer = new Analyzer() {
635 public TokenStream tokenStream(String fieldName, Reader reader) {
636 MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
637 tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
638 return new CrashingFilter(fieldName, tokenizer);
642 final int NUM_THREAD = 3;
643 final int NUM_ITER = 100;
645 for(int i=0;i<2;i++) {
646 MockDirectoryWrapper dir = newDirectory();
649 final IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer).setMaxBufferedDocs(-1)
650 .setMergePolicy(newLogMergePolicy(10)));
651 final int finalI = i;
653 Thread[] threads = new Thread[NUM_THREAD];
654 for(int t=0;t<NUM_THREAD;t++) {
655 threads[t] = new Thread() {
659 for(int iter=0;iter<NUM_ITER;iter++) {
660 Document doc = new Document();
661 doc.add(newField("contents", "here are some contents", Field.Store.YES,
662 Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
663 writer.addDocument(doc);
664 writer.addDocument(doc);
665 doc.add(newField("crash", "this should crash after 4 terms", Field.Store.YES,
666 Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
667 doc.add(newField("other", "this will not get indexed", Field.Store.YES,
668 Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
670 writer.addDocument(doc);
671 fail("did not hit expected exception");
672 } catch (IOException ioe) {
676 doc = new Document();
677 doc.add(newField("contents", "here are some contents", Field.Store.YES,
678 Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
679 writer.addDocument(doc);
680 writer.addDocument(doc);
683 } catch (Throwable t) {
685 System.out.println(Thread.currentThread().getName() + ": ERROR: hit unexpected exception");
686 t.printStackTrace(System.out);
695 for(int t=0;t<NUM_THREAD;t++)
701 IndexReader reader = IndexReader.open(dir, true);
702 int expected = (3+(1-i)*2)*NUM_THREAD*NUM_ITER;
703 assertEquals("i=" + i, expected, reader.docFreq(new Term("contents", "here")));
704 assertEquals(expected, reader.maxDoc());
706 for(int j=0;j<reader.maxDoc();j++) {
707 if (reader.isDeleted(j))
711 reader.getTermFreqVectors(j);
716 assertEquals(NUM_THREAD*NUM_ITER, numDel);
718 IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
719 TEST_VERSION_CURRENT, analyzer).setMaxBufferedDocs(10));
720 Document doc = new Document();
721 doc.add(newField("contents", "here are some contents", Field.Store.YES,
722 Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
723 for(int j=0;j<17;j++)
724 writer.addDocument(doc);
728 reader = IndexReader.open(dir, true);
729 expected += 17-NUM_THREAD*NUM_ITER;
730 assertEquals(expected, reader.docFreq(new Term("contents", "here")));
731 assertEquals(expected, reader.maxDoc());
733 for(int j=0;j<reader.maxDoc();j++) {
734 if (reader.isDeleted(j))
738 reader.getTermFreqVectors(j);
747 // Throws IOException during MockDirectoryWrapper.sync
748 private static class FailOnlyInSync extends MockDirectoryWrapper.Failure {
751 public void eval(MockDirectoryWrapper dir) throws IOException {
753 StackTraceElement[] trace = new Exception().getStackTrace();
754 for (int i = 0; i < trace.length; i++) {
755 if (doFail && "org.apache.lucene.store.MockDirectoryWrapper".equals(trace[i].getClassName()) && "sync".equals(trace[i].getMethodName())) {
757 throw new IOException("now failing on purpose during sync");
764 // TODO: these are also in TestIndexWriter... add a simple doc-writing method
765 // like this to LuceneTestCase?
766 private void addDoc(IndexWriter writer) throws IOException
768 Document doc = new Document();
769 doc.add(newField("content", "aaa", Field.Store.NO, Field.Index.ANALYZED));
770 writer.addDocument(doc);
773 // LUCENE-1044: test exception during sync
774 public void testExceptionDuringSync() throws IOException {
775 MockDirectoryWrapper dir = newDirectory();
776 FailOnlyInSync failure = new FailOnlyInSync();
779 IndexWriter writer = new IndexWriter(
781 newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
782 setMaxBufferedDocs(2).
783 setMergeScheduler(new ConcurrentMergeScheduler()).
784 setMergePolicy(newLogMergePolicy(5))
787 ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(5);
789 for (int i = 0; i < 23; i++) {
794 } catch (IOException ioe) {
800 ((ConcurrentMergeScheduler) writer.getConfig().getMergeScheduler()).sync();
801 assertTrue(failure.didFail);
802 failure.clearDoFail();
805 IndexReader reader = IndexReader.open(dir, true);
806 assertEquals(23, reader.numDocs());
811 private static class FailOnlyInCommit extends MockDirectoryWrapper.Failure {
813 boolean fail1, fail2;
816 public void eval(MockDirectoryWrapper dir) throws IOException {
817 StackTraceElement[] trace = new Exception().getStackTrace();
818 boolean isCommit = false;
819 boolean isDelete = false;
820 for (int i = 0; i < trace.length; i++) {
821 if ("org.apache.lucene.index.SegmentInfos".equals(trace[i].getClassName()) && "prepareCommit".equals(trace[i].getMethodName()))
823 if ("org.apache.lucene.store.MockDirectoryWrapper".equals(trace[i].getClassName()) && "deleteFile".equals(trace[i].getMethodName()))
830 throw new RuntimeException("now fail first");
833 throw new IOException("now fail during delete");
840 public void testExceptionsDuringCommit() throws Throwable {
841 MockDirectoryWrapper dir = newDirectory();
842 FailOnlyInCommit failure = new FailOnlyInCommit();
843 IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
844 Document doc = new Document();
845 doc.add(newField("field", "a field", Field.Store.YES,
846 Field.Index.ANALYZED));
852 } catch (IOException ioe) {
853 fail("expected only RuntimeException");
854 } catch (RuntimeException re) {
857 assertTrue(failure.fail1 && failure.fail2);
862 public void testOptimizeExceptions() throws IOException {
863 Directory startDir = newDirectory();
864 IndexWriterConfig conf = newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(2).setMergePolicy(newLogMergePolicy());
865 ((LogMergePolicy) conf.getMergePolicy()).setMergeFactor(100);
866 IndexWriter w = new IndexWriter(startDir, conf);
867 for(int i=0;i<27;i++)
871 int iter = TEST_NIGHTLY ? 200 : 20;
872 for(int i=0;i<iter;i++) {
874 System.out.println("TEST: iter " + i);
876 MockDirectoryWrapper dir = new MockDirectoryWrapper(random, new RAMDirectory(startDir));
877 conf = newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergeScheduler(new ConcurrentMergeScheduler());
878 ((ConcurrentMergeScheduler) conf.getMergeScheduler()).setSuppressExceptions();
879 w = new IndexWriter(dir, conf);
880 w.setInfoStream(VERBOSE ? System.out : null);
881 dir.setRandomIOExceptionRate(0.5);
884 } catch (IOException ioe) {
885 if (ioe.getCause() == null)
886 fail("optimize threw IOException without root cause");
888 dir.setRandomIOExceptionRate(0);
896 public void testOutOfMemoryErrorCausesCloseToFail() throws Exception {
898 final List<Throwable> thrown = new ArrayList<Throwable>();
899 final Directory dir = newDirectory();
900 final IndexWriter writer = new IndexWriter(dir,
901 newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))) {
903 public void message(final String message) {
904 if (message.startsWith("now flush at close") && 0 == thrown.size()) {
906 throw new OutOfMemoryError("fake OOME at " + message);
911 // need to set an info stream so message is called
912 writer.setInfoStream(new PrintStream(new ByteArrayOutputStream()));
915 fail("OutOfMemoryError expected");
917 catch (final OutOfMemoryError expected) {}
919 // throws IllegalStateEx w/o bug fix
925 private static final class MockIndexWriter4 extends IndexWriter {
927 public MockIndexWriter4(Directory dir, IndexWriterConfig conf) throws IOException {
934 boolean testPoint(String name) {
935 if (doFail && name.equals("rollback before checkpoint"))
936 throw new RuntimeException("intentionally failing");
942 public void testRollbackExceptionHang() throws Throwable {
943 Directory dir = newDirectory();
944 MockIndexWriter4 w = new MockIndexWriter4(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
950 fail("did not hit intentional RuntimeException");
951 } catch (RuntimeException re) {
960 // LUCENE-1044: Simulate checksum error in segments_N
961 public void testSegmentsChecksumError() throws IOException {
962 Directory dir = newDirectory();
964 IndexWriter writer = null;
966 writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
969 for (int i = 0; i < 100; i++) {
976 long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
977 assertTrue("segment generation should be > 0 but got " + gen, gen > 0);
979 final String segmentsFileName = SegmentInfos.getCurrentSegmentFileName(dir);
980 IndexInput in = dir.openInput(segmentsFileName);
981 IndexOutput out = dir.createOutput(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", 1+gen));
982 out.copyBytes(in, in.length()-1);
983 byte b = in.readByte();
984 out.writeByte((byte) (1+b));
988 IndexReader reader = null;
990 reader = IndexReader.open(dir, true);
991 } catch (IOException e) {
992 e.printStackTrace(System.out);
993 fail("segmentInfos failed to retry fallback to correct segments_N file");
999 // Simulate a corrupt index by removing last byte of
1000 // latest segments file and make sure we get an
1001 // IOException trying to open the index:
1002 public void testSimulatedCorruptIndex1() throws IOException {
1003 MockDirectoryWrapper dir = newDirectory();
1004 dir.setCheckIndexOnClose(false); // we are corrupting it!
1006 IndexWriter writer = null;
1008 writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
1010 // add 100 documents
1011 for (int i = 0; i < 100; i++) {
1018 long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
1019 assertTrue("segment generation should be > 0 but got " + gen, gen > 0);
1021 String fileNameIn = SegmentInfos.getCurrentSegmentFileName(dir);
1022 String fileNameOut = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
1025 IndexInput in = dir.openInput(fileNameIn);
1026 IndexOutput out = dir.createOutput(fileNameOut);
1027 long length = in.length();
1028 for(int i=0;i<length-1;i++) {
1029 out.writeByte(in.readByte());
1033 dir.deleteFile(fileNameIn);
1035 IndexReader reader = null;
1037 reader = IndexReader.open(dir, true);
1038 fail("reader did not hit IOException on opening a corrupt index");
1039 } catch (Exception e) {
1041 if (reader != null) {
1047 // Simulate a corrupt index by removing one of the cfs
1048 // files and make sure we get an IOException trying to
1050 public void testSimulatedCorruptIndex2() throws IOException {
1051 MockDirectoryWrapper dir = newDirectory();
1052 dir.setCheckIndexOnClose(false); // we are corrupting it!
1053 IndexWriter writer = null;
1055 writer = new IndexWriter(
1057 newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
1058 setMergePolicy(newLogMergePolicy(true))
1060 ((LogMergePolicy) writer.getConfig().getMergePolicy()).setNoCFSRatio(1.0);
1062 // add 100 documents
1063 for (int i = 0; i < 100; i++) {
1070 long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
1071 assertTrue("segment generation should be > 0 but got " + gen, gen > 0);
1073 String[] files = dir.listAll();
1074 boolean corrupted = false;
1075 for(int i=0;i<files.length;i++) {
1076 if (files[i].endsWith(".cfs")) {
1077 dir.deleteFile(files[i]);
1082 assertTrue("failed to find cfs file to remove", corrupted);
1084 IndexReader reader = null;
1086 reader = IndexReader.open(dir, true);
1087 fail("reader did not hit IOException on opening a corrupt index");
1088 } catch (Exception e) {
1090 if (reader != null) {
1096 // Simulate a writer that crashed while writing segments
1097 // file: make sure we can still open the index (ie,
1098 // gracefully fallback to the previous segments file),
1099 // and that we can add to the index:
1100 public void testSimulatedCrashedWriter() throws IOException {
1101 MockDirectoryWrapper dir = newDirectory();
1102 dir.setPreventDoubleWrite(false);
1104 IndexWriter writer = null;
1106 writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
1108 // add 100 documents
1109 for (int i = 0; i < 100; i++) {
1116 long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
1117 assertTrue("segment generation should be > 0 but got " + gen, gen > 0);
1119 // Make the next segments file, with last byte
1120 // missing, to simulate a writer that crashed while
1121 // writing segments file:
1122 String fileNameIn = SegmentInfos.getCurrentSegmentFileName(dir);
1123 String fileNameOut = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
1126 IndexInput in = dir.openInput(fileNameIn);
1127 IndexOutput out = dir.createOutput(fileNameOut);
1128 long length = in.length();
1129 for(int i=0;i<length-1;i++) {
1130 out.writeByte(in.readByte());
1135 IndexReader reader = null;
1137 reader = IndexReader.open(dir, true);
1138 } catch (Exception e) {
1139 fail("reader failed to open on a crashed index");
1144 writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE));
1145 } catch (Exception e) {
1146 e.printStackTrace(System.out);
1147 fail("writer failed to open on a crashed index");
1150 // add 100 documents
1151 for (int i = 0; i < 100; i++) {
1160 public void testAddDocsNonAbortingException() throws Exception {
1161 final Directory dir = newDirectory();
1162 final RandomIndexWriter w = new RandomIndexWriter(random, dir);
1163 final int numDocs1 = random.nextInt(25);
1164 for(int docCount=0;docCount<numDocs1;docCount++) {
1165 Document doc = new Document();
1166 doc.add(newField("content", "good content", Field.Index.ANALYZED));
1170 final List<Document> docs = new ArrayList<Document>();
1171 for(int docCount=0;docCount<7;docCount++) {
1172 Document doc = new Document();
1174 doc.add(newField("id", docCount+"", Field.Index.NOT_ANALYZED));
1175 doc.add(newField("content", "silly content " + docCount, Field.Index.ANALYZED));
1176 if (docCount == 4) {
1177 Field f = newField("crash", "", Field.Index.ANALYZED);
1179 MockTokenizer tokenizer = new MockTokenizer(new StringReader("crash me on the 4th token"), MockTokenizer.WHITESPACE, false);
1180 tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
1181 f.setTokenStream(new CrashingFilter("crash", tokenizer));
1185 w.addDocuments(docs);
1186 // BUG: CrashingFilter didn't
1187 fail("did not hit expected exception");
1188 } catch (IOException ioe) {
1190 assertEquals(CRASH_FAIL_MESSAGE, ioe.getMessage());
1193 final int numDocs2 = random.nextInt(25);
1194 for(int docCount=0;docCount<numDocs2;docCount++) {
1195 Document doc = new Document();
1196 doc.add(newField("content", "good content", Field.Index.ANALYZED));
1200 final IndexReader r = w.getReader();
1203 final IndexSearcher s = new IndexSearcher(r);
1204 PhraseQuery pq = new PhraseQuery();
1205 pq.add(new Term("content", "silly"));
1206 pq.add(new Term("content", "content"));
1207 assertEquals(0, s.search(pq, 1).totalHits);
1209 pq = new PhraseQuery();
1210 pq.add(new Term("content", "good"));
1211 pq.add(new Term("content", "content"));
1212 assertEquals(numDocs1+numDocs2, s.search(pq, 1).totalHits);
1218 public void testUpdateDocsNonAbortingException() throws Exception {
1219 final Directory dir = newDirectory();
1220 final RandomIndexWriter w = new RandomIndexWriter(random, dir);
1221 final int numDocs1 = random.nextInt(25);
1222 for(int docCount=0;docCount<numDocs1;docCount++) {
1223 Document doc = new Document();
1224 doc.add(newField("content", "good content", Field.Index.ANALYZED));
1228 // Use addDocs (no exception) to get docs in the index:
1229 final List<Document> docs = new ArrayList<Document>();
1230 final int numDocs2 = random.nextInt(25);
1231 for(int docCount=0;docCount<numDocs2;docCount++) {
1232 Document doc = new Document();
1234 doc.add(newField("subid", "subs", Field.Index.NOT_ANALYZED));
1235 doc.add(newField("id", docCount+"", Field.Index.NOT_ANALYZED));
1236 doc.add(newField("content", "silly content " + docCount, Field.Index.ANALYZED));
1238 w.addDocuments(docs);
1240 final int numDocs3 = random.nextInt(25);
1241 for(int docCount=0;docCount<numDocs3;docCount++) {
1242 Document doc = new Document();
1243 doc.add(newField("content", "good content", Field.Index.ANALYZED));
1248 final int limit = _TestUtil.nextInt(random, 2, 25);
1249 final int crashAt = random.nextInt(limit);
1250 for(int docCount=0;docCount<limit;docCount++) {
1251 Document doc = new Document();
1253 doc.add(newField("id", docCount+"", Field.Index.NOT_ANALYZED));
1254 doc.add(newField("content", "silly content " + docCount, Field.Index.ANALYZED));
1255 if (docCount == crashAt) {
1256 Field f = newField("crash", "", Field.Index.ANALYZED);
1258 MockTokenizer tokenizer = new MockTokenizer(new StringReader("crash me on the 4th token"), MockTokenizer.WHITESPACE, false);
1259 tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
1260 f.setTokenStream(new CrashingFilter("crash", tokenizer));
1265 w.updateDocuments(new Term("subid", "subs"), docs);
1266 // BUG: CrashingFilter didn't
1267 fail("did not hit expected exception");
1268 } catch (IOException ioe) {
1270 assertEquals(CRASH_FAIL_MESSAGE, ioe.getMessage());
1273 final int numDocs4 = random.nextInt(25);
1274 for(int docCount=0;docCount<numDocs4;docCount++) {
1275 Document doc = new Document();
1276 doc.add(newField("content", "good content", Field.Index.ANALYZED));
1280 final IndexReader r = w.getReader();
1283 final IndexSearcher s = new IndexSearcher(r);
1284 PhraseQuery pq = new PhraseQuery();
1285 pq.add(new Term("content", "silly"));
1286 pq.add(new Term("content", "content"));
1287 assertEquals(numDocs2, s.search(pq, 1).totalHits);
1289 pq = new PhraseQuery();
1290 pq.add(new Term("content", "good"));
1291 pq.add(new Term("content", "content"));
1292 assertEquals(numDocs1+numDocs3+numDocs4, s.search(pq, 1).totalHits);