--- /dev/null
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.io.Reader;
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.IndexWriterConfig.OpenMode;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.PhraseQuery;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.MockDirectoryWrapper;
+import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
+
+public class TestIndexWriterExceptions extends LuceneTestCase {
+
+ private class IndexerThread extends Thread {
+
+ IndexWriter writer;
+
+ final Random r = new Random(random.nextLong());
+ volatile Throwable failure;
+
+ public IndexerThread(int i, IndexWriter writer) {
+ setName("Indexer " + i);
+ this.writer = writer;
+ }
+
+ @Override
+ public void run() {
+
+ final Document doc = new Document();
+
+ doc.add(newField(r, "content1", "aaa bbb ccc ddd", Field.Store.YES, Field.Index.ANALYZED));
+ doc.add(newField(r, "content6", "aaa bbb ccc ddd", Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
+ doc.add(newField(r, "content2", "aaa bbb ccc ddd", Field.Store.YES, Field.Index.NOT_ANALYZED));
+ doc.add(newField(r, "content3", "aaa bbb ccc ddd", Field.Store.YES, Field.Index.NO));
+
+ doc.add(newField(r, "content4", "aaa bbb ccc ddd", Field.Store.NO, Field.Index.ANALYZED));
+ doc.add(newField(r, "content5", "aaa bbb ccc ddd", Field.Store.NO, Field.Index.NOT_ANALYZED));
+
+ doc.add(newField(r, "content7", "aaa bbb ccc ddd", Field.Store.NO, Field.Index.NOT_ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
+
+ final Field idField = newField(r, "id", "", Field.Store.YES, Field.Index.NOT_ANALYZED);
+ doc.add(idField);
+
+ final long stopTime = System.currentTimeMillis() + 500;
+
+ do {
+ if (VERBOSE) {
+ System.out.println(Thread.currentThread().getName() + ": TEST: IndexerThread: cycle");
+ }
+ doFail.set(this);
+ final String id = ""+r.nextInt(50);
+ idField.setValue(id);
+ Term idTerm = new Term("id", id);
+ try {
+ if (r.nextBoolean()) {
+ final List<Document> docs = new ArrayList<Document>();
+ final int count = _TestUtil.nextInt(r, 1, 20);
+ for(int c=0;c<count;c++) {
+ docs.add(doc);
+ }
+ writer.updateDocuments(idTerm, docs);
+ } else {
+ writer.updateDocument(idTerm, doc);
+ }
+ } catch (RuntimeException re) {
+ if (VERBOSE) {
+ System.out.println(Thread.currentThread().getName() + ": EXC: ");
+ re.printStackTrace(System.out);
+ }
+ try {
+ _TestUtil.checkIndex(writer.getDirectory());
+ } catch (IOException ioe) {
+ System.out.println(Thread.currentThread().getName() + ": unexpected exception1");
+ ioe.printStackTrace(System.out);
+ failure = ioe;
+ break;
+ }
+ } catch (Throwable t) {
+ System.out.println(Thread.currentThread().getName() + ": unexpected exception2");
+ t.printStackTrace(System.out);
+ failure = t;
+ break;
+ }
+
+ doFail.set(null);
+
+ // After a possible exception (above) I should be able
+ // to add a new document without hitting an
+ // exception:
+ try {
+ writer.updateDocument(idTerm, doc);
+ } catch (Throwable t) {
+ System.out.println(Thread.currentThread().getName() + ": unexpected exception3");
+ t.printStackTrace(System.out);
+ failure = t;
+ break;
+ }
+ } while(System.currentTimeMillis() < stopTime);
+ }
+ }
+
+ ThreadLocal<Thread> doFail = new ThreadLocal<Thread>();
+
+ private class MockIndexWriter extends IndexWriter {
+ Random r = new Random(random.nextLong());
+
+ public MockIndexWriter(Directory dir, IndexWriterConfig conf) throws IOException {
+ super(dir, conf);
+ }
+
+ @Override
+ boolean testPoint(String name) {
+ if (doFail.get() != null && !name.equals("startDoFlush") && r.nextInt(40) == 17) {
+ if (VERBOSE) {
+ System.out.println(Thread.currentThread().getName() + ": NOW FAIL: " + name);
+ new Throwable().printStackTrace(System.out);
+ }
+ throw new RuntimeException(Thread.currentThread().getName() + ": intentionally failing at " + name);
+ }
+ return true;
+ }
+ }
+
+ public void testRandomExceptions() throws Throwable {
+ if (VERBOSE) {
+ System.out.println("\nTEST: start testRandomExceptions");
+ }
+ MockDirectoryWrapper dir = newDirectory();
+
+ MockAnalyzer analyzer = new MockAnalyzer(random);
+ analyzer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
+ MockIndexWriter writer = new MockIndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer)
+ .setRAMBufferSizeMB(0.1).setMergeScheduler(new ConcurrentMergeScheduler()));
+ ((ConcurrentMergeScheduler) writer.getConfig().getMergeScheduler()).setSuppressExceptions();
+ //writer.setMaxBufferedDocs(10);
+ if (VERBOSE) {
+ System.out.println("TEST: initial commit");
+ }
+ writer.commit();
+
+ if (VERBOSE) {
+ writer.setInfoStream(System.out);
+ }
+
+ IndexerThread thread = new IndexerThread(0, writer);
+ thread.run();
+ if (thread.failure != null) {
+ thread.failure.printStackTrace(System.out);
+ fail("thread " + thread.getName() + ": hit unexpected failure");
+ }
+
+ if (VERBOSE) {
+ System.out.println("TEST: commit after thread start");
+ }
+ writer.commit();
+
+ try {
+ writer.close();
+ } catch (Throwable t) {
+ System.out.println("exception during close:");
+ t.printStackTrace(System.out);
+ writer.rollback();
+ }
+
+ // Confirm that when doc hits exception partway through tokenization, it's deleted:
+ IndexReader r2 = IndexReader.open(dir, true);
+ final int count = r2.docFreq(new Term("content4", "aaa"));
+ final int count2 = r2.docFreq(new Term("content4", "ddd"));
+ assertEquals(count, count2);
+ r2.close();
+
+ dir.close();
+ }
+
+ public void testRandomExceptionsThreads() throws Throwable {
+ MockDirectoryWrapper dir = newDirectory();
+ MockAnalyzer analyzer = new MockAnalyzer(random);
+ analyzer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
+ MockIndexWriter writer = new MockIndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer)
+ .setRAMBufferSizeMB(0.2).setMergeScheduler(new ConcurrentMergeScheduler()));
+ ((ConcurrentMergeScheduler) writer.getConfig().getMergeScheduler()).setSuppressExceptions();
+ //writer.setMaxBufferedDocs(10);
+ writer.commit();
+
+ if (VERBOSE) {
+ writer.setInfoStream(System.out);
+ }
+
+ final int NUM_THREADS = 4;
+
+ final IndexerThread[] threads = new IndexerThread[NUM_THREADS];
+ for(int i=0;i<NUM_THREADS;i++) {
+ threads[i] = new IndexerThread(i, writer);
+ threads[i].start();
+ }
+
+ for(int i=0;i<NUM_THREADS;i++)
+ threads[i].join();
+
+ for(int i=0;i<NUM_THREADS;i++)
+ if (threads[i].failure != null)
+ fail("thread " + threads[i].getName() + ": hit unexpected failure");
+
+ writer.commit();
+
+ try {
+ writer.close();
+ } catch (Throwable t) {
+ System.out.println("exception during close:");
+ t.printStackTrace(System.out);
+ writer.rollback();
+ }
+
+ // Confirm that when doc hits exception partway through tokenization, it's deleted:
+ IndexReader r2 = IndexReader.open(dir, true);
+ final int count = r2.docFreq(new Term("content4", "aaa"));
+ final int count2 = r2.docFreq(new Term("content4", "ddd"));
+ assertEquals(count, count2);
+ r2.close();
+
+ dir.close();
+ }
+
+ // LUCENE-1198
+ private static final class MockIndexWriter2 extends IndexWriter {
+
+ public MockIndexWriter2(Directory dir, IndexWriterConfig conf) throws IOException {
+ super(dir, conf);
+ }
+
+ boolean doFail;
+
+ @Override
+ boolean testPoint(String name) {
+ if (doFail && name.equals("DocumentsWriter.ThreadState.init start"))
+ throw new RuntimeException("intentionally failing");
+ return true;
+ }
+ }
+
+ private static String CRASH_FAIL_MESSAGE = "I'm experiencing problems";
+
+ private class CrashingFilter extends TokenFilter {
+ String fieldName;
+ int count;
+
+ public CrashingFilter(String fieldName, TokenStream input) {
+ super(input);
+ this.fieldName = fieldName;
+ }
+
+ @Override
+ public boolean incrementToken() throws IOException {
+ if (this.fieldName.equals("crash") && count++ >= 4)
+ throw new IOException(CRASH_FAIL_MESSAGE);
+ return input.incrementToken();
+ }
+
+ @Override
+ public void reset() throws IOException {
+ super.reset();
+ count = 0;
+ }
+ }
+
+ public void testExceptionDocumentsWriterInit() throws IOException {
+ Directory dir = newDirectory();
+ MockIndexWriter2 w = new MockIndexWriter2(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
+ w.setInfoStream(VERBOSE ? System.out : null);
+ Document doc = new Document();
+ doc.add(newField("field", "a field", Field.Store.YES,
+ Field.Index.ANALYZED));
+ w.addDocument(doc);
+ w.doFail = true;
+ try {
+ w.addDocument(doc);
+ fail("did not hit exception");
+ } catch (RuntimeException re) {
+ // expected
+ }
+ w.close();
+ dir.close();
+ }
+
+ // LUCENE-1208
+ public void testExceptionJustBeforeFlush() throws IOException {
+ Directory dir = newDirectory();
+ MockIndexWriter w = new MockIndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(2));
+ w.setInfoStream(VERBOSE ? System.out : null);
+ Document doc = new Document();
+ doc.add(newField("field", "a field", Field.Store.YES,
+ Field.Index.ANALYZED));
+ w.addDocument(doc);
+
+ Analyzer analyzer = new Analyzer() {
+ @Override
+ public TokenStream tokenStream(String fieldName, Reader reader) {
+ MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
+ return new CrashingFilter(fieldName, tokenizer);
+ }
+ };
+
+ Document crashDoc = new Document();
+ crashDoc.add(newField("crash", "do it on token 4", Field.Store.YES,
+ Field.Index.ANALYZED));
+ try {
+ w.addDocument(crashDoc, analyzer);
+ fail("did not hit expected exception");
+ } catch (IOException ioe) {
+ // expected
+ }
+ w.addDocument(doc);
+ w.close();
+ dir.close();
+ }
+
+ private static final class MockIndexWriter3 extends IndexWriter {
+
+ public MockIndexWriter3(Directory dir, IndexWriterConfig conf) throws IOException {
+ super(dir, conf);
+ }
+
+ boolean doFail;
+ boolean failed;
+
+ @Override
+ boolean testPoint(String name) {
+ if (doFail && name.equals("startMergeInit")) {
+ failed = true;
+ throw new RuntimeException("intentionally failing");
+ }
+ return true;
+ }
+ }
+
+
+ // LUCENE-1210
+ public void testExceptionOnMergeInit() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriterConfig conf = newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))
+ .setMaxBufferedDocs(2).setMergeScheduler(new ConcurrentMergeScheduler()).setMergePolicy(newLogMergePolicy());
+ ((LogMergePolicy) conf.getMergePolicy()).setMergeFactor(2);
+ MockIndexWriter3 w = new MockIndexWriter3(dir, conf);
+ w.doFail = true;
+ Document doc = new Document();
+ doc.add(newField("field", "a field", Field.Store.YES,
+ Field.Index.ANALYZED));
+ for(int i=0;i<10;i++)
+ try {
+ w.addDocument(doc);
+ } catch (RuntimeException re) {
+ break;
+ }
+
+ ((ConcurrentMergeScheduler) w.getConfig().getMergeScheduler()).sync();
+ assertTrue(w.failed);
+ w.close();
+ dir.close();
+ }
+
+ // LUCENE-1072
+ public void testExceptionFromTokenStream() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriterConfig conf = newIndexWriterConfig( TEST_VERSION_CURRENT, new Analyzer() {
+
+ @Override
+ public TokenStream tokenStream(String fieldName, Reader reader) {
+ MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
+ return new TokenFilter(tokenizer) {
+ private int count = 0;
+
+ @Override
+ public boolean incrementToken() throws IOException {
+ if (count++ == 5) {
+ throw new IOException();
+ }
+ return input.incrementToken();
+ }
+ };
+ }
+
+ });
+ conf.setMaxBufferedDocs(Math.max(3, conf.getMaxBufferedDocs()));
+
+ IndexWriter writer = new IndexWriter(dir, conf);
+
+ Document doc = new Document();
+ String contents = "aa bb cc dd ee ff gg hh ii jj kk";
+ doc.add(newField("content", contents, Field.Store.NO,
+ Field.Index.ANALYZED));
+ try {
+ writer.addDocument(doc);
+ fail("did not hit expected exception");
+ } catch (Exception e) {
+ }
+
+ // Make sure we can add another normal document
+ doc = new Document();
+ doc.add(newField("content", "aa bb cc dd", Field.Store.NO,
+ Field.Index.ANALYZED));
+ writer.addDocument(doc);
+
+ // Make sure we can add another normal document
+ doc = new Document();
+ doc.add(newField("content", "aa bb cc dd", Field.Store.NO,
+ Field.Index.ANALYZED));
+ writer.addDocument(doc);
+
+ writer.close();
+ IndexReader reader = IndexReader.open(dir, true);
+ final Term t = new Term("content", "aa");
+ assertEquals(3, reader.docFreq(t));
+
+ // Make sure the doc that hit the exception was marked
+ // as deleted:
+ TermDocs tdocs = reader.termDocs(t);
+ int count = 0;
+ while(tdocs.next()) {
+ count++;
+ }
+ assertEquals(2, count);
+
+ assertEquals(reader.docFreq(new Term("content", "gg")), 0);
+ reader.close();
+ dir.close();
+ }
+
+ private static class FailOnlyOnFlush extends MockDirectoryWrapper.Failure {
+ boolean doFail = false;
+ int count;
+
+ @Override
+ public void setDoFail() {
+ this.doFail = true;
+ }
+ @Override
+ public void clearDoFail() {
+ this.doFail = false;
+ }
+
+ @Override
+ public void eval(MockDirectoryWrapper dir) throws IOException {
+ if (doFail) {
+ StackTraceElement[] trace = new Exception().getStackTrace();
+ boolean sawAppend = false;
+ boolean sawFlush = false;
+ for (int i = 0; i < trace.length; i++) {
+ if ("org.apache.lucene.index.FreqProxTermsWriter".equals(trace[i].getClassName()) && "appendPostings".equals(trace[i].getMethodName()))
+ sawAppend = true;
+ if ("doFlush".equals(trace[i].getMethodName()))
+ sawFlush = true;
+ }
+
+ if (sawAppend && sawFlush && count++ >= 30) {
+ doFail = false;
+ throw new IOException("now failing during flush");
+ }
+ }
+ }
+ }
+
+ // LUCENE-1072: make sure an errant exception on flushing
+ // one segment only takes out those docs in that one flush
+ public void testDocumentsWriterAbort() throws IOException {
+ MockDirectoryWrapper dir = newDirectory();
+ FailOnlyOnFlush failure = new FailOnlyOnFlush();
+ failure.setDoFail();
+ dir.failOn(failure);
+
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(2));
+ Document doc = new Document();
+ String contents = "aa bb cc dd ee ff gg hh ii jj kk";
+ doc.add(newField("content", contents, Field.Store.NO,
+ Field.Index.ANALYZED));
+ boolean hitError = false;
+ for(int i=0;i<200;i++) {
+ try {
+ writer.addDocument(doc);
+ } catch (IOException ioe) {
+ // only one flush should fail:
+ assertFalse(hitError);
+ hitError = true;
+ }
+ }
+ assertTrue(hitError);
+ writer.close();
+ IndexReader reader = IndexReader.open(dir, true);
+ assertEquals(198, reader.docFreq(new Term("content", "aa")));
+ reader.close();
+ dir.close();
+ }
+
+ public void testDocumentsWriterExceptions() throws IOException {
+ Analyzer analyzer = new Analyzer() {
+ @Override
+ public TokenStream tokenStream(String fieldName, Reader reader) {
+ MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
+ return new CrashingFilter(fieldName, tokenizer);
+ }
+ };
+
+ for(int i=0;i<2;i++) {
+ if (VERBOSE) {
+ System.out.println("TEST: cycle i=" + i);
+ }
+ MockDirectoryWrapper dir = newDirectory();
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer).setMergePolicy(newLogMergePolicy()));
+ writer.setInfoStream(VERBOSE ? System.out : null);
+
+ // don't allow a sudden merge to clean up the deleted
+ // doc below:
+ LogMergePolicy lmp = (LogMergePolicy) writer.getConfig().getMergePolicy();
+ lmp.setMergeFactor(Math.max(lmp.getMergeFactor(), 5));
+
+ Document doc = new Document();
+ doc.add(newField("contents", "here are some contents", Field.Store.YES,
+ Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
+ writer.addDocument(doc);
+ writer.addDocument(doc);
+ doc.add(newField("crash", "this should crash after 4 terms", Field.Store.YES,
+ Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
+ doc.add(newField("other", "this will not get indexed", Field.Store.YES,
+ Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
+ try {
+ writer.addDocument(doc);
+ fail("did not hit expected exception");
+ } catch (IOException ioe) {
+ if (VERBOSE) {
+ System.out.println("TEST: hit expected exception");
+ ioe.printStackTrace(System.out);
+ }
+ }
+
+ if (0 == i) {
+ doc = new Document();
+ doc.add(newField("contents", "here are some contents", Field.Store.YES,
+ Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
+ writer.addDocument(doc);
+ writer.addDocument(doc);
+ }
+ writer.close();
+
+ if (VERBOSE) {
+ System.out.println("TEST: open reader");
+ }
+ IndexReader reader = IndexReader.open(dir, true);
+ if (i == 0) {
+ int expected = 5;
+ assertEquals(expected, reader.docFreq(new Term("contents", "here")));
+ assertEquals(expected, reader.maxDoc());
+ int numDel = 0;
+ for(int j=0;j<reader.maxDoc();j++) {
+ if (reader.isDeleted(j))
+ numDel++;
+ else {
+ reader.document(j);
+ reader.getTermFreqVectors(j);
+ }
+ }
+ assertEquals(1, numDel);
+ }
+ reader.close();
+
+ writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT,
+ analyzer).setMaxBufferedDocs(10));
+ doc = new Document();
+ doc.add(newField("contents", "here are some contents", Field.Store.YES,
+ Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
+ for(int j=0;j<17;j++)
+ writer.addDocument(doc);
+ writer.forceMerge(1);
+ writer.close();
+
+ reader = IndexReader.open(dir, true);
+ int expected = 19+(1-i)*2;
+ assertEquals(expected, reader.docFreq(new Term("contents", "here")));
+ assertEquals(expected, reader.maxDoc());
+ int numDel = 0;
+ for(int j=0;j<reader.maxDoc();j++) {
+ if (reader.isDeleted(j))
+ numDel++;
+ else {
+ reader.document(j);
+ reader.getTermFreqVectors(j);
+ }
+ }
+ reader.close();
+ assertEquals(0, numDel);
+
+ dir.close();
+ }
+ }
+
+ public void testDocumentsWriterExceptionThreads() throws Exception {
+ Analyzer analyzer = new Analyzer() {
+ @Override
+ public TokenStream tokenStream(String fieldName, Reader reader) {
+ MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
+ return new CrashingFilter(fieldName, tokenizer);
+ }
+ };
+
+ final int NUM_THREAD = 3;
+ final int NUM_ITER = 100;
+
+ for(int i=0;i<2;i++) {
+ MockDirectoryWrapper dir = newDirectory();
+
+ {
+ final IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer).setMaxBufferedDocs(-1)
+ .setMergePolicy(newLogMergePolicy(10)));
+ final int finalI = i;
+
+ Thread[] threads = new Thread[NUM_THREAD];
+ for(int t=0;t<NUM_THREAD;t++) {
+ threads[t] = new Thread() {
+ @Override
+ public void run() {
+ try {
+ for(int iter=0;iter<NUM_ITER;iter++) {
+ Document doc = new Document();
+ doc.add(newField("contents", "here are some contents", Field.Store.YES,
+ Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
+ writer.addDocument(doc);
+ writer.addDocument(doc);
+ doc.add(newField("crash", "this should crash after 4 terms", Field.Store.YES,
+ Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
+ doc.add(newField("other", "this will not get indexed", Field.Store.YES,
+ Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
+ try {
+ writer.addDocument(doc);
+ fail("did not hit expected exception");
+ } catch (IOException ioe) {
+ }
+
+ if (0 == finalI) {
+ doc = new Document();
+ doc.add(newField("contents", "here are some contents", Field.Store.YES,
+ Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
+ writer.addDocument(doc);
+ writer.addDocument(doc);
+ }
+ }
+ } catch (Throwable t) {
+ synchronized(this) {
+ System.out.println(Thread.currentThread().getName() + ": ERROR: hit unexpected exception");
+ t.printStackTrace(System.out);
+ }
+ fail();
+ }
+ }
+ };
+ threads[t].start();
+ }
+
+ for(int t=0;t<NUM_THREAD;t++)
+ threads[t].join();
+
+ writer.close();
+ }
+
+ IndexReader reader = IndexReader.open(dir, true);
+ int expected = (3+(1-i)*2)*NUM_THREAD*NUM_ITER;
+ assertEquals("i=" + i, expected, reader.docFreq(new Term("contents", "here")));
+ assertEquals(expected, reader.maxDoc());
+ int numDel = 0;
+ for(int j=0;j<reader.maxDoc();j++) {
+ if (reader.isDeleted(j))
+ numDel++;
+ else {
+ reader.document(j);
+ reader.getTermFreqVectors(j);
+ }
+ }
+ reader.close();
+
+ assertEquals(NUM_THREAD*NUM_ITER, numDel);
+
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
+ TEST_VERSION_CURRENT, analyzer).setMaxBufferedDocs(10));
+ Document doc = new Document();
+ doc.add(newField("contents", "here are some contents", Field.Store.YES,
+ Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
+ for(int j=0;j<17;j++)
+ writer.addDocument(doc);
+ writer.forceMerge(1);
+ writer.close();
+
+ reader = IndexReader.open(dir, true);
+ expected += 17-NUM_THREAD*NUM_ITER;
+ assertEquals(expected, reader.docFreq(new Term("contents", "here")));
+ assertEquals(expected, reader.maxDoc());
+ numDel = 0;
+ for(int j=0;j<reader.maxDoc();j++) {
+ if (reader.isDeleted(j))
+ numDel++;
+ else {
+ reader.document(j);
+ reader.getTermFreqVectors(j);
+ }
+ }
+ reader.close();
+
+ dir.close();
+ }
+ }
+
+ // Throws IOException during MockDirectoryWrapper.sync
+ private static class FailOnlyInSync extends MockDirectoryWrapper.Failure {
+ boolean didFail;
+ @Override
+ public void eval(MockDirectoryWrapper dir) throws IOException {
+ if (doFail) {
+ StackTraceElement[] trace = new Exception().getStackTrace();
+ for (int i = 0; i < trace.length; i++) {
+ if (doFail && "org.apache.lucene.store.MockDirectoryWrapper".equals(trace[i].getClassName()) && "sync".equals(trace[i].getMethodName())) {
+ didFail = true;
+ throw new IOException("now failing on purpose during sync");
+ }
+ }
+ }
+ }
+ }
+
+ // TODO: these are also in TestIndexWriter... add a simple doc-writing method
+ // like this to LuceneTestCase?
+ private void addDoc(IndexWriter writer) throws IOException
+ {
+ Document doc = new Document();
+ doc.add(newField("content", "aaa", Field.Store.NO, Field.Index.ANALYZED));
+ writer.addDocument(doc);
+ }
+
+ // LUCENE-1044: test exception during sync
+ public void testExceptionDuringSync() throws IOException {
+ MockDirectoryWrapper dir = newDirectory();
+ FailOnlyInSync failure = new FailOnlyInSync();
+ dir.failOn(failure);
+
+ IndexWriter writer = new IndexWriter(
+ dir,
+ newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
+ setMaxBufferedDocs(2).
+ setMergeScheduler(new ConcurrentMergeScheduler()).
+ setMergePolicy(newLogMergePolicy(5))
+ );
+ failure.setDoFail();
+ ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(5);
+
+ for (int i = 0; i < 23; i++) {
+ addDoc(writer);
+ if ((i-1)%2 == 0) {
+ try {
+ writer.commit();
+ } catch (IOException ioe) {
+ // expected
+ }
+ }
+ }
+
+ ((ConcurrentMergeScheduler) writer.getConfig().getMergeScheduler()).sync();
+ assertTrue(failure.didFail);
+ failure.clearDoFail();
+ writer.close();
+
+ IndexReader reader = IndexReader.open(dir, true);
+ assertEquals(23, reader.numDocs());
+ reader.close();
+ dir.close();
+ }
+
+ private static class FailOnlyInCommit extends MockDirectoryWrapper.Failure {
+
+ boolean fail1, fail2;
+
+ @Override
+ public void eval(MockDirectoryWrapper dir) throws IOException {
+ StackTraceElement[] trace = new Exception().getStackTrace();
+ boolean isCommit = false;
+ boolean isDelete = false;
+ for (int i = 0; i < trace.length; i++) {
+ if ("org.apache.lucene.index.SegmentInfos".equals(trace[i].getClassName()) && "prepareCommit".equals(trace[i].getMethodName()))
+ isCommit = true;
+ if ("org.apache.lucene.store.MockDirectoryWrapper".equals(trace[i].getClassName()) && "deleteFile".equals(trace[i].getMethodName()))
+ isDelete = true;
+ }
+
+ if (isCommit) {
+ if (!isDelete) {
+ fail1 = true;
+ throw new RuntimeException("now fail first");
+ } else {
+ fail2 = true;
+ throw new IOException("now fail during delete");
+ }
+ }
+ }
+ }
+
+ // LUCENE-1214
+ public void testExceptionsDuringCommit() throws Throwable {
+ MockDirectoryWrapper dir = newDirectory();
+ dir.setFailOnCreateOutput(false);
+ FailOnlyInCommit failure = new FailOnlyInCommit();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
+ Document doc = new Document();
+ doc.add(newField("field", "a field", Field.Store.YES,
+ Field.Index.ANALYZED));
+ w.addDocument(doc);
+ dir.failOn(failure);
+ try {
+ w.close();
+ fail();
+ } catch (IOException ioe) {
+ fail("expected only RuntimeException");
+ } catch (RuntimeException re) {
+ // Expected
+ }
+ assertTrue(failure.fail1 && failure.fail2);
+ w.rollback();
+ dir.close();
+ }
+
+ public void testForceMergeExceptions() throws IOException {
+ Directory startDir = newDirectory();
+ IndexWriterConfig conf = newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(2).setMergePolicy(newLogMergePolicy());
+ ((LogMergePolicy) conf.getMergePolicy()).setMergeFactor(100);
+ IndexWriter w = new IndexWriter(startDir, conf);
+ for(int i=0;i<27;i++)
+ addDoc(w);
+ w.close();
+
+ int iter = TEST_NIGHTLY ? 200 : 20;
+ for(int i=0;i<iter;i++) {
+ if (VERBOSE) {
+ System.out.println("TEST: iter " + i);
+ }
+ MockDirectoryWrapper dir = new MockDirectoryWrapper(random, new RAMDirectory(startDir));
+ conf = newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergeScheduler(new ConcurrentMergeScheduler());
+ ((ConcurrentMergeScheduler) conf.getMergeScheduler()).setSuppressExceptions();
+ w = new IndexWriter(dir, conf);
+ w.setInfoStream(VERBOSE ? System.out : null);
+ dir.setRandomIOExceptionRate(0.5);
+ try {
+ w.forceMerge(1);
+ } catch (IOException ioe) {
+ if (ioe.getCause() == null)
+ fail("forceMerge threw IOException without root cause");
+ }
+ dir.setRandomIOExceptionRate(0);
+ w.close();
+ dir.close();
+ }
+ startDir.close();
+ }
+
+ // LUCENE-1429
+ public void testOutOfMemoryErrorCausesCloseToFail() throws Exception {
+
+ final List<Throwable> thrown = new ArrayList<Throwable>();
+ final Directory dir = newDirectory();
+ final IndexWriter writer = new IndexWriter(dir,
+ newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))) {
+ @Override
+ public void message(final String message) {
+ if (message.startsWith("now flush at close") && 0 == thrown.size()) {
+ thrown.add(null);
+ throw new OutOfMemoryError("fake OOME at " + message);
+ }
+ }
+ };
+
+ // need to set an info stream so message is called
+ writer.setInfoStream(new PrintStream(new ByteArrayOutputStream()));
+ try {
+ writer.close();
+ fail("OutOfMemoryError expected");
+ }
+ catch (final OutOfMemoryError expected) {}
+
+ // throws IllegalStateEx w/o bug fix
+ writer.close();
+ dir.close();
+ }
+
+ // LUCENE-1347
+ private static final class MockIndexWriter4 extends IndexWriter {
+
+ public MockIndexWriter4(Directory dir, IndexWriterConfig conf) throws IOException {
+ super(dir, conf);
+ }
+
+ boolean doFail;
+
+ @Override
+ boolean testPoint(String name) {
+ if (doFail && name.equals("rollback before checkpoint"))
+ throw new RuntimeException("intentionally failing");
+ return true;
+ }
+ }
+
+ // LUCENE-1347
+ public void testRollbackExceptionHang() throws Throwable {
+ Directory dir = newDirectory();
+ MockIndexWriter4 w = new MockIndexWriter4(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
+
+ addDoc(w);
+ w.doFail = true;
+ try {
+ w.rollback();
+ fail("did not hit intentional RuntimeException");
+ } catch (RuntimeException re) {
+ // expected
+ }
+
+ w.doFail = false;
+ w.rollback();
+ dir.close();
+ }
+
+ // LUCENE-1044: Simulate checksum error in segments_N
+ public void testSegmentsChecksumError() throws IOException {
+ Directory dir = newDirectory();
+
+ IndexWriter writer = null;
+
+ writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
+
+ // add 100 documents
+ for (int i = 0; i < 100; i++) {
+ addDoc(writer);
+ }
+
+ // close
+ writer.close();
+
+ long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
+ assertTrue("segment generation should be > 0 but got " + gen, gen > 0);
+
+ final String segmentsFileName = SegmentInfos.getCurrentSegmentFileName(dir);
+ IndexInput in = dir.openInput(segmentsFileName);
+ IndexOutput out = dir.createOutput(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", 1+gen));
+ out.copyBytes(in, in.length()-1);
+ byte b = in.readByte();
+ out.writeByte((byte) (1+b));
+ out.close();
+ in.close();
+
+ IndexReader reader = null;
+ try {
+ reader = IndexReader.open(dir, true);
+ } catch (IOException e) {
+ e.printStackTrace(System.out);
+ fail("segmentInfos failed to retry fallback to correct segments_N file");
+ }
+ reader.close();
+ dir.close();
+ }
+
+ // Simulate a corrupt index by removing last byte of
+ // latest segments file and make sure we get an
+ // IOException trying to open the index:
+ public void testSimulatedCorruptIndex1() throws IOException {
+ MockDirectoryWrapper dir = newDirectory();
+ dir.setCheckIndexOnClose(false); // we are corrupting it!
+
+ IndexWriter writer = null;
+
+ writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
+
+ // add 100 documents
+ for (int i = 0; i < 100; i++) {
+ addDoc(writer);
+ }
+
+ // close
+ writer.close();
+
+ long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
+ assertTrue("segment generation should be > 0 but got " + gen, gen > 0);
+
+ String fileNameIn = SegmentInfos.getCurrentSegmentFileName(dir);
+ String fileNameOut = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
+ "",
+ 1+gen);
+ IndexInput in = dir.openInput(fileNameIn);
+ IndexOutput out = dir.createOutput(fileNameOut);
+ long length = in.length();
+ for(int i=0;i<length-1;i++) {
+ out.writeByte(in.readByte());
+ }
+ in.close();
+ out.close();
+ dir.deleteFile(fileNameIn);
+
+ IndexReader reader = null;
+ try {
+ reader = IndexReader.open(dir, true);
+ fail("reader did not hit IOException on opening a corrupt index");
+ } catch (Exception e) {
+ }
+ if (reader != null) {
+ reader.close();
+ }
+ dir.close();
+ }
+
+ // Simulate a corrupt index by removing one of the cfs
+ // files and make sure we get an IOException trying to
+ // open the index:
+ public void testSimulatedCorruptIndex2() throws IOException {
+ MockDirectoryWrapper dir = newDirectory();
+ dir.setCheckIndexOnClose(false); // we are corrupting it!
+ IndexWriter writer = null;
+
+ writer = new IndexWriter(
+ dir,
+ newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
+ setMergePolicy(newLogMergePolicy(true))
+ );
+ ((LogMergePolicy) writer.getConfig().getMergePolicy()).setNoCFSRatio(1.0);
+
+ // add 100 documents
+ for (int i = 0; i < 100; i++) {
+ addDoc(writer);
+ }
+
+ // close
+ writer.close();
+
+ long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
+ assertTrue("segment generation should be > 0 but got " + gen, gen > 0);
+
+ String[] files = dir.listAll();
+ boolean corrupted = false;
+ for(int i=0;i<files.length;i++) {
+ if (files[i].endsWith(".cfs")) {
+ dir.deleteFile(files[i]);
+ corrupted = true;
+ break;
+ }
+ }
+ assertTrue("failed to find cfs file to remove", corrupted);
+
+ IndexReader reader = null;
+ try {
+ reader = IndexReader.open(dir, true);
+ fail("reader did not hit IOException on opening a corrupt index");
+ } catch (Exception e) {
+ }
+ if (reader != null) {
+ reader.close();
+ }
+ dir.close();
+ }
+
+ // Simulate a writer that crashed while writing segments
+ // file: make sure we can still open the index (ie,
+ // gracefully fallback to the previous segments file),
+ // and that we can add to the index:
+ public void testSimulatedCrashedWriter() throws IOException {
+ MockDirectoryWrapper dir = newDirectory();
+ dir.setPreventDoubleWrite(false);
+
+ IndexWriter writer = null;
+
+ writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
+
+ // add 100 documents
+ for (int i = 0; i < 100; i++) {
+ addDoc(writer);
+ }
+
+ // close
+ writer.close();
+
+ long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
+ assertTrue("segment generation should be > 0 but got " + gen, gen > 0);
+
+ // Make the next segments file, with last byte
+ // missing, to simulate a writer that crashed while
+ // writing segments file:
+ String fileNameIn = SegmentInfos.getCurrentSegmentFileName(dir);
+ String fileNameOut = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
+ "",
+ 1+gen);
+ IndexInput in = dir.openInput(fileNameIn);
+ IndexOutput out = dir.createOutput(fileNameOut);
+ long length = in.length();
+ for(int i=0;i<length-1;i++) {
+ out.writeByte(in.readByte());
+ }
+ in.close();
+ out.close();
+
+ IndexReader reader = null;
+ try {
+ reader = IndexReader.open(dir, true);
+ } catch (Exception e) {
+ fail("reader failed to open on a crashed index");
+ }
+ reader.close();
+
+ try {
+ writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE));
+ } catch (Exception e) {
+ e.printStackTrace(System.out);
+ fail("writer failed to open on a crashed index");
+ }
+
+ // add 100 documents
+ for (int i = 0; i < 100; i++) {
+ addDoc(writer);
+ }
+
+ // close
+ writer.close();
+ dir.close();
+ }
+
+ public void testAddDocsNonAbortingException() throws Exception {
+ final Directory dir = newDirectory();
+ final RandomIndexWriter w = new RandomIndexWriter(random, dir);
+ final int numDocs1 = random.nextInt(25);
+ for(int docCount=0;docCount<numDocs1;docCount++) {
+ Document doc = new Document();
+ doc.add(newField("content", "good content", Field.Index.ANALYZED));
+ w.addDocument(doc);
+ }
+
+ final List<Document> docs = new ArrayList<Document>();
+ for(int docCount=0;docCount<7;docCount++) {
+ Document doc = new Document();
+ docs.add(doc);
+ doc.add(newField("id", docCount+"", Field.Index.NOT_ANALYZED));
+ doc.add(newField("content", "silly content " + docCount, Field.Index.ANALYZED));
+ if (docCount == 4) {
+ Field f = newField("crash", "", Field.Index.ANALYZED);
+ doc.add(f);
+ MockTokenizer tokenizer = new MockTokenizer(new StringReader("crash me on the 4th token"), MockTokenizer.WHITESPACE, false);
+ tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
+ f.setTokenStream(new CrashingFilter("crash", tokenizer));
+ }
+ }
+ try {
+ w.addDocuments(docs);
+ // BUG: CrashingFilter didn't
+ fail("did not hit expected exception");
+ } catch (IOException ioe) {
+ // expected
+ assertEquals(CRASH_FAIL_MESSAGE, ioe.getMessage());
+ }
+
+ final int numDocs2 = random.nextInt(25);
+ for(int docCount=0;docCount<numDocs2;docCount++) {
+ Document doc = new Document();
+ doc.add(newField("content", "good content", Field.Index.ANALYZED));
+ w.addDocument(doc);
+ }
+
+ final IndexReader r = w.getReader();
+ w.close();
+
+ final IndexSearcher s = new IndexSearcher(r);
+ PhraseQuery pq = new PhraseQuery();
+ pq.add(new Term("content", "silly"));
+ pq.add(new Term("content", "content"));
+ assertEquals(0, s.search(pq, 1).totalHits);
+
+ pq = new PhraseQuery();
+ pq.add(new Term("content", "good"));
+ pq.add(new Term("content", "content"));
+ assertEquals(numDocs1+numDocs2, s.search(pq, 1).totalHits);
+ r.close();
+ dir.close();
+ }
+
+
+ public void testUpdateDocsNonAbortingException() throws Exception {
+ final Directory dir = newDirectory();
+ final RandomIndexWriter w = new RandomIndexWriter(random, dir);
+ final int numDocs1 = random.nextInt(25);
+ for(int docCount=0;docCount<numDocs1;docCount++) {
+ Document doc = new Document();
+ doc.add(newField("content", "good content", Field.Index.ANALYZED));
+ w.addDocument(doc);
+ }
+
+ // Use addDocs (no exception) to get docs in the index:
+ final List<Document> docs = new ArrayList<Document>();
+ final int numDocs2 = random.nextInt(25);
+ for(int docCount=0;docCount<numDocs2;docCount++) {
+ Document doc = new Document();
+ docs.add(doc);
+ doc.add(newField("subid", "subs", Field.Index.NOT_ANALYZED));
+ doc.add(newField("id", docCount+"", Field.Index.NOT_ANALYZED));
+ doc.add(newField("content", "silly content " + docCount, Field.Index.ANALYZED));
+ }
+ w.addDocuments(docs);
+
+ final int numDocs3 = random.nextInt(25);
+ for(int docCount=0;docCount<numDocs3;docCount++) {
+ Document doc = new Document();
+ doc.add(newField("content", "good content", Field.Index.ANALYZED));
+ w.addDocument(doc);
+ }
+
+ docs.clear();
+ final int limit = _TestUtil.nextInt(random, 2, 25);
+ final int crashAt = random.nextInt(limit);
+ for(int docCount=0;docCount<limit;docCount++) {
+ Document doc = new Document();
+ docs.add(doc);
+ doc.add(newField("id", docCount+"", Field.Index.NOT_ANALYZED));
+ doc.add(newField("content", "silly content " + docCount, Field.Index.ANALYZED));
+ if (docCount == crashAt) {
+ Field f = newField("crash", "", Field.Index.ANALYZED);
+ doc.add(f);
+ MockTokenizer tokenizer = new MockTokenizer(new StringReader("crash me on the 4th token"), MockTokenizer.WHITESPACE, false);
+ tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
+ f.setTokenStream(new CrashingFilter("crash", tokenizer));
+ }
+ }
+
+ try {
+ w.updateDocuments(new Term("subid", "subs"), docs);
+ // BUG: CrashingFilter didn't
+ fail("did not hit expected exception");
+ } catch (IOException ioe) {
+ // expected
+ assertEquals(CRASH_FAIL_MESSAGE, ioe.getMessage());
+ }
+
+ final int numDocs4 = random.nextInt(25);
+ for(int docCount=0;docCount<numDocs4;docCount++) {
+ Document doc = new Document();
+ doc.add(newField("content", "good content", Field.Index.ANALYZED));
+ w.addDocument(doc);
+ }
+
+ final IndexReader r = w.getReader();
+ w.close();
+
+ final IndexSearcher s = new IndexSearcher(r);
+ PhraseQuery pq = new PhraseQuery();
+ pq.add(new Term("content", "silly"));
+ pq.add(new Term("content", "content"));
+ assertEquals(numDocs2, s.search(pq, 1).totalHits);
+
+ pq = new PhraseQuery();
+ pq.add(new Term("content", "good"));
+ pq.add(new Term("content", "content"));
+ assertEquals(numDocs1+numDocs3+numDocs4, s.search(pq, 1).totalHits);
+ r.close();
+ dir.close();
+ }
+}
fnp / pylucene.git / blobdiff