+++ /dev/null
-package org.apache.lucene.index;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.io.Reader;
-import java.io.StringReader;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util._TestUtil;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.store.MockDirectoryWrapper;
-import org.apache.lucene.store.RAMDirectory;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.analysis.MockTokenizer;
-import org.apache.lucene.analysis.TokenFilter;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.index.IndexWriterConfig.OpenMode;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.PhraseQuery;
-
-public class TestIndexWriterExceptions extends LuceneTestCase {
-
- private class IndexerThread extends Thread {
-
- IndexWriter writer;
-
- final Random r = new Random(random.nextLong());
- volatile Throwable failure;
-
- public IndexerThread(int i, IndexWriter writer) {
- setName("Indexer " + i);
- this.writer = writer;
- }
-
- @Override
- public void run() {
-
- final Document doc = new Document();
-
- doc.add(newField(r, "content1", "aaa bbb ccc ddd", Field.Store.YES, Field.Index.ANALYZED));
- doc.add(newField(r, "content6", "aaa bbb ccc ddd", Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
- doc.add(newField(r, "content2", "aaa bbb ccc ddd", Field.Store.YES, Field.Index.NOT_ANALYZED));
- doc.add(newField(r, "content3", "aaa bbb ccc ddd", Field.Store.YES, Field.Index.NO));
-
- doc.add(newField(r, "content4", "aaa bbb ccc ddd", Field.Store.NO, Field.Index.ANALYZED));
- doc.add(newField(r, "content5", "aaa bbb ccc ddd", Field.Store.NO, Field.Index.NOT_ANALYZED));
-
- doc.add(newField(r, "content7", "aaa bbb ccc ddd", Field.Store.NO, Field.Index.NOT_ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
-
- final Field idField = newField(r, "id", "", Field.Store.YES, Field.Index.NOT_ANALYZED);
- doc.add(idField);
-
- final long stopTime = System.currentTimeMillis() + 500;
-
- do {
- if (VERBOSE) {
- System.out.println(Thread.currentThread().getName() + ": TEST: IndexerThread: cycle");
- }
- doFail.set(this);
- final String id = ""+r.nextInt(50);
- idField.setValue(id);
- Term idTerm = new Term("id", id);
- try {
- if (r.nextBoolean()) {
- final List<Document> docs = new ArrayList<Document>();
- final int count = _TestUtil.nextInt(r, 1, 20);
- for(int c=0;c<count;c++) {
- docs.add(doc);
- }
- writer.updateDocuments(idTerm, docs);
- } else {
- writer.updateDocument(idTerm, doc);
- }
- } catch (RuntimeException re) {
- if (VERBOSE) {
- System.out.println(Thread.currentThread().getName() + ": EXC: ");
- re.printStackTrace(System.out);
- }
- try {
- _TestUtil.checkIndex(writer.getDirectory());
- } catch (IOException ioe) {
- System.out.println(Thread.currentThread().getName() + ": unexpected exception1");
- ioe.printStackTrace(System.out);
- failure = ioe;
- break;
- }
- } catch (Throwable t) {
- System.out.println(Thread.currentThread().getName() + ": unexpected exception2");
- t.printStackTrace(System.out);
- failure = t;
- break;
- }
-
- doFail.set(null);
-
- // After a possible exception (above) I should be able
- // to add a new document without hitting an
- // exception:
- try {
- writer.updateDocument(idTerm, doc);
- } catch (Throwable t) {
- System.out.println(Thread.currentThread().getName() + ": unexpected exception3");
- t.printStackTrace(System.out);
- failure = t;
- break;
- }
- } while(System.currentTimeMillis() < stopTime);
- }
- }
-
- ThreadLocal<Thread> doFail = new ThreadLocal<Thread>();
-
- private class MockIndexWriter extends IndexWriter {
- Random r = new Random(random.nextLong());
-
- public MockIndexWriter(Directory dir, IndexWriterConfig conf) throws IOException {
- super(dir, conf);
- }
-
- @Override
- boolean testPoint(String name) {
- if (doFail.get() != null && !name.equals("startDoFlush") && r.nextInt(40) == 17) {
- if (VERBOSE) {
- System.out.println(Thread.currentThread().getName() + ": NOW FAIL: " + name);
- new Throwable().printStackTrace(System.out);
- }
- throw new RuntimeException(Thread.currentThread().getName() + ": intentionally failing at " + name);
- }
- return true;
- }
- }
-
- public void testRandomExceptions() throws Throwable {
- if (VERBOSE) {
- System.out.println("\nTEST: start testRandomExceptions");
- }
- MockDirectoryWrapper dir = newDirectory();
-
- MockAnalyzer analyzer = new MockAnalyzer(random);
- analyzer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
- MockIndexWriter writer = new MockIndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer)
- .setRAMBufferSizeMB(0.1).setMergeScheduler(new ConcurrentMergeScheduler()));
- ((ConcurrentMergeScheduler) writer.getConfig().getMergeScheduler()).setSuppressExceptions();
- //writer.setMaxBufferedDocs(10);
- if (VERBOSE) {
- System.out.println("TEST: initial commit");
- }
- writer.commit();
-
- if (VERBOSE) {
- writer.setInfoStream(System.out);
- }
-
- IndexerThread thread = new IndexerThread(0, writer);
- thread.run();
- if (thread.failure != null) {
- thread.failure.printStackTrace(System.out);
- fail("thread " + thread.getName() + ": hit unexpected failure");
- }
-
- if (VERBOSE) {
- System.out.println("TEST: commit after thread start");
- }
- writer.commit();
-
- try {
- writer.close();
- } catch (Throwable t) {
- System.out.println("exception during close:");
- t.printStackTrace(System.out);
- writer.rollback();
- }
-
- // Confirm that when doc hits exception partway through tokenization, it's deleted:
- IndexReader r2 = IndexReader.open(dir, true);
- final int count = r2.docFreq(new Term("content4", "aaa"));
- final int count2 = r2.docFreq(new Term("content4", "ddd"));
- assertEquals(count, count2);
- r2.close();
-
- dir.close();
- }
-
- public void testRandomExceptionsThreads() throws Throwable {
- MockDirectoryWrapper dir = newDirectory();
- MockAnalyzer analyzer = new MockAnalyzer(random);
- analyzer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
- MockIndexWriter writer = new MockIndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer)
- .setRAMBufferSizeMB(0.2).setMergeScheduler(new ConcurrentMergeScheduler()));
- ((ConcurrentMergeScheduler) writer.getConfig().getMergeScheduler()).setSuppressExceptions();
- //writer.setMaxBufferedDocs(10);
- writer.commit();
-
- if (VERBOSE) {
- writer.setInfoStream(System.out);
- }
-
- final int NUM_THREADS = 4;
-
- final IndexerThread[] threads = new IndexerThread[NUM_THREADS];
- for(int i=0;i<NUM_THREADS;i++) {
- threads[i] = new IndexerThread(i, writer);
- threads[i].start();
- }
-
- for(int i=0;i<NUM_THREADS;i++)
- threads[i].join();
-
- for(int i=0;i<NUM_THREADS;i++)
- if (threads[i].failure != null)
- fail("thread " + threads[i].getName() + ": hit unexpected failure");
-
- writer.commit();
-
- try {
- writer.close();
- } catch (Throwable t) {
- System.out.println("exception during close:");
- t.printStackTrace(System.out);
- writer.rollback();
- }
-
- // Confirm that when doc hits exception partway through tokenization, it's deleted:
- IndexReader r2 = IndexReader.open(dir, true);
- final int count = r2.docFreq(new Term("content4", "aaa"));
- final int count2 = r2.docFreq(new Term("content4", "ddd"));
- assertEquals(count, count2);
- r2.close();
-
- dir.close();
- }
-
- // LUCENE-1198
- private static final class MockIndexWriter2 extends IndexWriter {
-
- public MockIndexWriter2(Directory dir, IndexWriterConfig conf) throws IOException {
- super(dir, conf);
- }
-
- boolean doFail;
-
- @Override
- boolean testPoint(String name) {
- if (doFail && name.equals("DocumentsWriter.ThreadState.init start"))
- throw new RuntimeException("intentionally failing");
- return true;
- }
- }
-
- private static String CRASH_FAIL_MESSAGE = "I'm experiencing problems";
-
- private class CrashingFilter extends TokenFilter {
- String fieldName;
- int count;
-
- public CrashingFilter(String fieldName, TokenStream input) {
- super(input);
- this.fieldName = fieldName;
- }
-
- @Override
- public boolean incrementToken() throws IOException {
- if (this.fieldName.equals("crash") && count++ >= 4)
- throw new IOException(CRASH_FAIL_MESSAGE);
- return input.incrementToken();
- }
-
- @Override
- public void reset() throws IOException {
- super.reset();
- count = 0;
- }
- }
-
- public void testExceptionDocumentsWriterInit() throws IOException {
- Directory dir = newDirectory();
- MockIndexWriter2 w = new MockIndexWriter2(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
- w.setInfoStream(VERBOSE ? System.out : null);
- Document doc = new Document();
- doc.add(newField("field", "a field", Field.Store.YES,
- Field.Index.ANALYZED));
- w.addDocument(doc);
- w.doFail = true;
- try {
- w.addDocument(doc);
- fail("did not hit exception");
- } catch (RuntimeException re) {
- // expected
- }
- w.close();
- dir.close();
- }
-
- // LUCENE-1208
- public void testExceptionJustBeforeFlush() throws IOException {
- Directory dir = newDirectory();
- MockIndexWriter w = new MockIndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(2));
- w.setInfoStream(VERBOSE ? System.out : null);
- Document doc = new Document();
- doc.add(newField("field", "a field", Field.Store.YES,
- Field.Index.ANALYZED));
- w.addDocument(doc);
-
- Analyzer analyzer = new Analyzer() {
- @Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
- MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
- tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
- return new CrashingFilter(fieldName, tokenizer);
- }
- };
-
- Document crashDoc = new Document();
- crashDoc.add(newField("crash", "do it on token 4", Field.Store.YES,
- Field.Index.ANALYZED));
- try {
- w.addDocument(crashDoc, analyzer);
- fail("did not hit expected exception");
- } catch (IOException ioe) {
- // expected
- }
- w.addDocument(doc);
- w.close();
- dir.close();
- }
-
- private static final class MockIndexWriter3 extends IndexWriter {
-
- public MockIndexWriter3(Directory dir, IndexWriterConfig conf) throws IOException {
- super(dir, conf);
- }
-
- boolean doFail;
- boolean failed;
-
- @Override
- boolean testPoint(String name) {
- if (doFail && name.equals("startMergeInit")) {
- failed = true;
- throw new RuntimeException("intentionally failing");
- }
- return true;
- }
- }
-
-
- // LUCENE-1210
- public void testExceptionOnMergeInit() throws IOException {
- Directory dir = newDirectory();
- IndexWriterConfig conf = newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))
- .setMaxBufferedDocs(2).setMergeScheduler(new ConcurrentMergeScheduler()).setMergePolicy(newLogMergePolicy());
- ((LogMergePolicy) conf.getMergePolicy()).setMergeFactor(2);
- MockIndexWriter3 w = new MockIndexWriter3(dir, conf);
- w.doFail = true;
- Document doc = new Document();
- doc.add(newField("field", "a field", Field.Store.YES,
- Field.Index.ANALYZED));
- for(int i=0;i<10;i++)
- try {
- w.addDocument(doc);
- } catch (RuntimeException re) {
- break;
- }
-
- ((ConcurrentMergeScheduler) w.getConfig().getMergeScheduler()).sync();
- assertTrue(w.failed);
- w.close();
- dir.close();
- }
-
- // LUCENE-1072
- public void testExceptionFromTokenStream() throws IOException {
- Directory dir = newDirectory();
- IndexWriterConfig conf = newIndexWriterConfig( TEST_VERSION_CURRENT, new Analyzer() {
-
- @Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
- MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
- tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
- return new TokenFilter(tokenizer) {
- private int count = 0;
-
- @Override
- public boolean incrementToken() throws IOException {
- if (count++ == 5) {
- throw new IOException();
- }
- return input.incrementToken();
- }
- };
- }
-
- });
- conf.setMaxBufferedDocs(Math.max(3, conf.getMaxBufferedDocs()));
-
- IndexWriter writer = new IndexWriter(dir, conf);
-
- Document doc = new Document();
- String contents = "aa bb cc dd ee ff gg hh ii jj kk";
- doc.add(newField("content", contents, Field.Store.NO,
- Field.Index.ANALYZED));
- try {
- writer.addDocument(doc);
- fail("did not hit expected exception");
- } catch (Exception e) {
- }
-
- // Make sure we can add another normal document
- doc = new Document();
- doc.add(newField("content", "aa bb cc dd", Field.Store.NO,
- Field.Index.ANALYZED));
- writer.addDocument(doc);
-
- // Make sure we can add another normal document
- doc = new Document();
- doc.add(newField("content", "aa bb cc dd", Field.Store.NO,
- Field.Index.ANALYZED));
- writer.addDocument(doc);
-
- writer.close();
- IndexReader reader = IndexReader.open(dir, true);
- final Term t = new Term("content", "aa");
- assertEquals(3, reader.docFreq(t));
-
- // Make sure the doc that hit the exception was marked
- // as deleted:
- TermDocs tdocs = reader.termDocs(t);
- int count = 0;
- while(tdocs.next()) {
- count++;
- }
- assertEquals(2, count);
-
- assertEquals(reader.docFreq(new Term("content", "gg")), 0);
- reader.close();
- dir.close();
- }
-
- private static class FailOnlyOnFlush extends MockDirectoryWrapper.Failure {
- boolean doFail = false;
- int count;
-
- @Override
- public void setDoFail() {
- this.doFail = true;
- }
- @Override
- public void clearDoFail() {
- this.doFail = false;
- }
-
- @Override
- public void eval(MockDirectoryWrapper dir) throws IOException {
- if (doFail) {
- StackTraceElement[] trace = new Exception().getStackTrace();
- boolean sawAppend = false;
- boolean sawFlush = false;
- for (int i = 0; i < trace.length; i++) {
- if ("org.apache.lucene.index.FreqProxTermsWriter".equals(trace[i].getClassName()) && "appendPostings".equals(trace[i].getMethodName()))
- sawAppend = true;
- if ("doFlush".equals(trace[i].getMethodName()))
- sawFlush = true;
- }
-
- if (sawAppend && sawFlush && count++ >= 30) {
- doFail = false;
- throw new IOException("now failing during flush");
- }
- }
- }
- }
-
- // LUCENE-1072: make sure an errant exception on flushing
- // one segment only takes out those docs in that one flush
- public void testDocumentsWriterAbort() throws IOException {
- MockDirectoryWrapper dir = newDirectory();
- FailOnlyOnFlush failure = new FailOnlyOnFlush();
- failure.setDoFail();
- dir.failOn(failure);
-
- IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(2));
- Document doc = new Document();
- String contents = "aa bb cc dd ee ff gg hh ii jj kk";
- doc.add(newField("content", contents, Field.Store.NO,
- Field.Index.ANALYZED));
- boolean hitError = false;
- for(int i=0;i<200;i++) {
- try {
- writer.addDocument(doc);
- } catch (IOException ioe) {
- // only one flush should fail:
- assertFalse(hitError);
- hitError = true;
- }
- }
- assertTrue(hitError);
- writer.close();
- IndexReader reader = IndexReader.open(dir, true);
- assertEquals(198, reader.docFreq(new Term("content", "aa")));
- reader.close();
- dir.close();
- }
-
- public void testDocumentsWriterExceptions() throws IOException {
- Analyzer analyzer = new Analyzer() {
- @Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
- MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
- tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
- return new CrashingFilter(fieldName, tokenizer);
- }
- };
-
- for(int i=0;i<2;i++) {
- if (VERBOSE) {
- System.out.println("TEST: cycle i=" + i);
- }
- MockDirectoryWrapper dir = newDirectory();
- IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer).setMergePolicy(newLogMergePolicy()));
- writer.setInfoStream(VERBOSE ? System.out : null);
-
- // don't allow a sudden merge to clean up the deleted
- // doc below:
- LogMergePolicy lmp = (LogMergePolicy) writer.getConfig().getMergePolicy();
- lmp.setMergeFactor(Math.max(lmp.getMergeFactor(), 5));
-
- Document doc = new Document();
- doc.add(newField("contents", "here are some contents", Field.Store.YES,
- Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
- writer.addDocument(doc);
- writer.addDocument(doc);
- doc.add(newField("crash", "this should crash after 4 terms", Field.Store.YES,
- Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
- doc.add(newField("other", "this will not get indexed", Field.Store.YES,
- Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
- try {
- writer.addDocument(doc);
- fail("did not hit expected exception");
- } catch (IOException ioe) {
- if (VERBOSE) {
- System.out.println("TEST: hit expected exception");
- ioe.printStackTrace(System.out);
- }
- }
-
- if (0 == i) {
- doc = new Document();
- doc.add(newField("contents", "here are some contents", Field.Store.YES,
- Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
- writer.addDocument(doc);
- writer.addDocument(doc);
- }
- writer.close();
-
- if (VERBOSE) {
- System.out.println("TEST: open reader");
- }
- IndexReader reader = IndexReader.open(dir, true);
- if (i == 0) {
- int expected = 5;
- assertEquals(expected, reader.docFreq(new Term("contents", "here")));
- assertEquals(expected, reader.maxDoc());
- int numDel = 0;
- for(int j=0;j<reader.maxDoc();j++) {
- if (reader.isDeleted(j))
- numDel++;
- else {
- reader.document(j);
- reader.getTermFreqVectors(j);
- }
- }
- assertEquals(1, numDel);
- }
- reader.close();
-
- writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT,
- analyzer).setMaxBufferedDocs(10));
- doc = new Document();
- doc.add(newField("contents", "here are some contents", Field.Store.YES,
- Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
- for(int j=0;j<17;j++)
- writer.addDocument(doc);
- writer.optimize();
- writer.close();
-
- reader = IndexReader.open(dir, true);
- int expected = 19+(1-i)*2;
- assertEquals(expected, reader.docFreq(new Term("contents", "here")));
- assertEquals(expected, reader.maxDoc());
- int numDel = 0;
- for(int j=0;j<reader.maxDoc();j++) {
- if (reader.isDeleted(j))
- numDel++;
- else {
- reader.document(j);
- reader.getTermFreqVectors(j);
- }
- }
- reader.close();
- assertEquals(0, numDel);
-
- dir.close();
- }
- }
-
- public void testDocumentsWriterExceptionThreads() throws Exception {
- Analyzer analyzer = new Analyzer() {
- @Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
- MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
- tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
- return new CrashingFilter(fieldName, tokenizer);
- }
- };
-
- final int NUM_THREAD = 3;
- final int NUM_ITER = 100;
-
- for(int i=0;i<2;i++) {
- MockDirectoryWrapper dir = newDirectory();
-
- {
- final IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer).setMaxBufferedDocs(-1)
- .setMergePolicy(newLogMergePolicy(10)));
- final int finalI = i;
-
- Thread[] threads = new Thread[NUM_THREAD];
- for(int t=0;t<NUM_THREAD;t++) {
- threads[t] = new Thread() {
- @Override
- public void run() {
- try {
- for(int iter=0;iter<NUM_ITER;iter++) {
- Document doc = new Document();
- doc.add(newField("contents", "here are some contents", Field.Store.YES,
- Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
- writer.addDocument(doc);
- writer.addDocument(doc);
- doc.add(newField("crash", "this should crash after 4 terms", Field.Store.YES,
- Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
- doc.add(newField("other", "this will not get indexed", Field.Store.YES,
- Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
- try {
- writer.addDocument(doc);
- fail("did not hit expected exception");
- } catch (IOException ioe) {
- }
-
- if (0 == finalI) {
- doc = new Document();
- doc.add(newField("contents", "here are some contents", Field.Store.YES,
- Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
- writer.addDocument(doc);
- writer.addDocument(doc);
- }
- }
- } catch (Throwable t) {
- synchronized(this) {
- System.out.println(Thread.currentThread().getName() + ": ERROR: hit unexpected exception");
- t.printStackTrace(System.out);
- }
- fail();
- }
- }
- };
- threads[t].start();
- }
-
- for(int t=0;t<NUM_THREAD;t++)
- threads[t].join();
-
- writer.close();
- }
-
- IndexReader reader = IndexReader.open(dir, true);
- int expected = (3+(1-i)*2)*NUM_THREAD*NUM_ITER;
- assertEquals("i=" + i, expected, reader.docFreq(new Term("contents", "here")));
- assertEquals(expected, reader.maxDoc());
- int numDel = 0;
- for(int j=0;j<reader.maxDoc();j++) {
- if (reader.isDeleted(j))
- numDel++;
- else {
- reader.document(j);
- reader.getTermFreqVectors(j);
- }
- }
- reader.close();
-
- assertEquals(NUM_THREAD*NUM_ITER, numDel);
-
- IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
- TEST_VERSION_CURRENT, analyzer).setMaxBufferedDocs(10));
- Document doc = new Document();
- doc.add(newField("contents", "here are some contents", Field.Store.YES,
- Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
- for(int j=0;j<17;j++)
- writer.addDocument(doc);
- writer.optimize();
- writer.close();
-
- reader = IndexReader.open(dir, true);
- expected += 17-NUM_THREAD*NUM_ITER;
- assertEquals(expected, reader.docFreq(new Term("contents", "here")));
- assertEquals(expected, reader.maxDoc());
- numDel = 0;
- for(int j=0;j<reader.maxDoc();j++) {
- if (reader.isDeleted(j))
- numDel++;
- else {
- reader.document(j);
- reader.getTermFreqVectors(j);
- }
- }
- reader.close();
-
- dir.close();
- }
- }
-
- // Throws IOException during MockDirectoryWrapper.sync
- private static class FailOnlyInSync extends MockDirectoryWrapper.Failure {
- boolean didFail;
- @Override
- public void eval(MockDirectoryWrapper dir) throws IOException {
- if (doFail) {
- StackTraceElement[] trace = new Exception().getStackTrace();
- for (int i = 0; i < trace.length; i++) {
- if (doFail && "org.apache.lucene.store.MockDirectoryWrapper".equals(trace[i].getClassName()) && "sync".equals(trace[i].getMethodName())) {
- didFail = true;
- throw new IOException("now failing on purpose during sync");
- }
- }
- }
- }
- }
-
- // TODO: these are also in TestIndexWriter... add a simple doc-writing method
- // like this to LuceneTestCase?
- private void addDoc(IndexWriter writer) throws IOException
- {
- Document doc = new Document();
- doc.add(newField("content", "aaa", Field.Store.NO, Field.Index.ANALYZED));
- writer.addDocument(doc);
- }
-
- // LUCENE-1044: test exception during sync
- public void testExceptionDuringSync() throws IOException {
- MockDirectoryWrapper dir = newDirectory();
- FailOnlyInSync failure = new FailOnlyInSync();
- dir.failOn(failure);
-
- IndexWriter writer = new IndexWriter(
- dir,
- newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
- setMaxBufferedDocs(2).
- setMergeScheduler(new ConcurrentMergeScheduler()).
- setMergePolicy(newLogMergePolicy(5))
- );
- failure.setDoFail();
- ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(5);
-
- for (int i = 0; i < 23; i++) {
- addDoc(writer);
- if ((i-1)%2 == 0) {
- try {
- writer.commit();
- } catch (IOException ioe) {
- // expected
- }
- }
- }
-
- ((ConcurrentMergeScheduler) writer.getConfig().getMergeScheduler()).sync();
- assertTrue(failure.didFail);
- failure.clearDoFail();
- writer.close();
-
- IndexReader reader = IndexReader.open(dir, true);
- assertEquals(23, reader.numDocs());
- reader.close();
- dir.close();
- }
-
- private static class FailOnlyInCommit extends MockDirectoryWrapper.Failure {
-
- boolean fail1, fail2;
-
- @Override
- public void eval(MockDirectoryWrapper dir) throws IOException {
- StackTraceElement[] trace = new Exception().getStackTrace();
- boolean isCommit = false;
- boolean isDelete = false;
- for (int i = 0; i < trace.length; i++) {
- if ("org.apache.lucene.index.SegmentInfos".equals(trace[i].getClassName()) && "prepareCommit".equals(trace[i].getMethodName()))
- isCommit = true;
- if ("org.apache.lucene.store.MockDirectoryWrapper".equals(trace[i].getClassName()) && "deleteFile".equals(trace[i].getMethodName()))
- isDelete = true;
- }
-
- if (isCommit) {
- if (!isDelete) {
- fail1 = true;
- throw new RuntimeException("now fail first");
- } else {
- fail2 = true;
- throw new IOException("now fail during delete");
- }
- }
- }
- }
-
- // LUCENE-1214
- public void testExceptionsDuringCommit() throws Throwable {
- MockDirectoryWrapper dir = newDirectory();
- FailOnlyInCommit failure = new FailOnlyInCommit();
- IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
- Document doc = new Document();
- doc.add(newField("field", "a field", Field.Store.YES,
- Field.Index.ANALYZED));
- w.addDocument(doc);
- dir.failOn(failure);
- try {
- w.close();
- fail();
- } catch (IOException ioe) {
- fail("expected only RuntimeException");
- } catch (RuntimeException re) {
- // Expected
- }
- assertTrue(failure.fail1 && failure.fail2);
- w.rollback();
- dir.close();
- }
-
- public void testOptimizeExceptions() throws IOException {
- Directory startDir = newDirectory();
- IndexWriterConfig conf = newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(2).setMergePolicy(newLogMergePolicy());
- ((LogMergePolicy) conf.getMergePolicy()).setMergeFactor(100);
- IndexWriter w = new IndexWriter(startDir, conf);
- for(int i=0;i<27;i++)
- addDoc(w);
- w.close();
-
- int iter = TEST_NIGHTLY ? 200 : 20;
- for(int i=0;i<iter;i++) {
- if (VERBOSE) {
- System.out.println("TEST: iter " + i);
- }
- MockDirectoryWrapper dir = new MockDirectoryWrapper(random, new RAMDirectory(startDir));
- conf = newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergeScheduler(new ConcurrentMergeScheduler());
- ((ConcurrentMergeScheduler) conf.getMergeScheduler()).setSuppressExceptions();
- w = new IndexWriter(dir, conf);
- w.setInfoStream(VERBOSE ? System.out : null);
- dir.setRandomIOExceptionRate(0.5);
- try {
- w.optimize();
- } catch (IOException ioe) {
- if (ioe.getCause() == null)
- fail("optimize threw IOException without root cause");
- }
- dir.setRandomIOExceptionRate(0);
- w.close();
- dir.close();
- }
- startDir.close();
- }
-
- // LUCENE-1429
- public void testOutOfMemoryErrorCausesCloseToFail() throws Exception {
-
- final List<Throwable> thrown = new ArrayList<Throwable>();
- final Directory dir = newDirectory();
- final IndexWriter writer = new IndexWriter(dir,
- newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))) {
- @Override
- public void message(final String message) {
- if (message.startsWith("now flush at close") && 0 == thrown.size()) {
- thrown.add(null);
- throw new OutOfMemoryError("fake OOME at " + message);
- }
- }
- };
-
- // need to set an info stream so message is called
- writer.setInfoStream(new PrintStream(new ByteArrayOutputStream()));
- try {
- writer.close();
- fail("OutOfMemoryError expected");
- }
- catch (final OutOfMemoryError expected) {}
-
- // throws IllegalStateEx w/o bug fix
- writer.close();
- dir.close();
- }
-
- // LUCENE-1347
- private static final class MockIndexWriter4 extends IndexWriter {
-
- public MockIndexWriter4(Directory dir, IndexWriterConfig conf) throws IOException {
- super(dir, conf);
- }
-
- boolean doFail;
-
- @Override
- boolean testPoint(String name) {
- if (doFail && name.equals("rollback before checkpoint"))
- throw new RuntimeException("intentionally failing");
- return true;
- }
- }
-
- // LUCENE-1347
- public void testRollbackExceptionHang() throws Throwable {
- Directory dir = newDirectory();
- MockIndexWriter4 w = new MockIndexWriter4(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
-
- addDoc(w);
- w.doFail = true;
- try {
- w.rollback();
- fail("did not hit intentional RuntimeException");
- } catch (RuntimeException re) {
- // expected
- }
-
- w.doFail = false;
- w.rollback();
- dir.close();
- }
-
- // LUCENE-1044: Simulate checksum error in segments_N
- public void testSegmentsChecksumError() throws IOException {
- Directory dir = newDirectory();
-
- IndexWriter writer = null;
-
- writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
-
- // add 100 documents
- for (int i = 0; i < 100; i++) {
- addDoc(writer);
- }
-
- // close
- writer.close();
-
- long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
- assertTrue("segment generation should be > 0 but got " + gen, gen > 0);
-
- final String segmentsFileName = SegmentInfos.getCurrentSegmentFileName(dir);
- IndexInput in = dir.openInput(segmentsFileName);
- IndexOutput out = dir.createOutput(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", 1+gen));
- out.copyBytes(in, in.length()-1);
- byte b = in.readByte();
- out.writeByte((byte) (1+b));
- out.close();
- in.close();
-
- IndexReader reader = null;
- try {
- reader = IndexReader.open(dir, true);
- } catch (IOException e) {
- e.printStackTrace(System.out);
- fail("segmentInfos failed to retry fallback to correct segments_N file");
- }
- reader.close();
- dir.close();
- }
-
- // Simulate a corrupt index by removing last byte of
- // latest segments file and make sure we get an
- // IOException trying to open the index:
- public void testSimulatedCorruptIndex1() throws IOException {
- MockDirectoryWrapper dir = newDirectory();
- dir.setCheckIndexOnClose(false); // we are corrupting it!
-
- IndexWriter writer = null;
-
- writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
-
- // add 100 documents
- for (int i = 0; i < 100; i++) {
- addDoc(writer);
- }
-
- // close
- writer.close();
-
- long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
- assertTrue("segment generation should be > 0 but got " + gen, gen > 0);
-
- String fileNameIn = SegmentInfos.getCurrentSegmentFileName(dir);
- String fileNameOut = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
- "",
- 1+gen);
- IndexInput in = dir.openInput(fileNameIn);
- IndexOutput out = dir.createOutput(fileNameOut);
- long length = in.length();
- for(int i=0;i<length-1;i++) {
- out.writeByte(in.readByte());
- }
- in.close();
- out.close();
- dir.deleteFile(fileNameIn);
-
- IndexReader reader = null;
- try {
- reader = IndexReader.open(dir, true);
- fail("reader did not hit IOException on opening a corrupt index");
- } catch (Exception e) {
- }
- if (reader != null) {
- reader.close();
- }
- dir.close();
- }
-
- // Simulate a corrupt index by removing one of the cfs
- // files and make sure we get an IOException trying to
- // open the index:
- public void testSimulatedCorruptIndex2() throws IOException {
- MockDirectoryWrapper dir = newDirectory();
- dir.setCheckIndexOnClose(false); // we are corrupting it!
- IndexWriter writer = null;
-
- writer = new IndexWriter(
- dir,
- newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
- setMergePolicy(newLogMergePolicy(true))
- );
- ((LogMergePolicy) writer.getConfig().getMergePolicy()).setNoCFSRatio(1.0);
-
- // add 100 documents
- for (int i = 0; i < 100; i++) {
- addDoc(writer);
- }
-
- // close
- writer.close();
-
- long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
- assertTrue("segment generation should be > 0 but got " + gen, gen > 0);
-
- String[] files = dir.listAll();
- boolean corrupted = false;
- for(int i=0;i<files.length;i++) {
- if (files[i].endsWith(".cfs")) {
- dir.deleteFile(files[i]);
- corrupted = true;
- break;
- }
- }
- assertTrue("failed to find cfs file to remove", corrupted);
-
- IndexReader reader = null;
- try {
- reader = IndexReader.open(dir, true);
- fail("reader did not hit IOException on opening a corrupt index");
- } catch (Exception e) {
- }
- if (reader != null) {
- reader.close();
- }
- dir.close();
- }
-
- // Simulate a writer that crashed while writing segments
- // file: make sure we can still open the index (ie,
- // gracefully fallback to the previous segments file),
- // and that we can add to the index:
- public void testSimulatedCrashedWriter() throws IOException {
- MockDirectoryWrapper dir = newDirectory();
- dir.setPreventDoubleWrite(false);
-
- IndexWriter writer = null;
-
- writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
-
- // add 100 documents
- for (int i = 0; i < 100; i++) {
- addDoc(writer);
- }
-
- // close
- writer.close();
-
- long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
- assertTrue("segment generation should be > 0 but got " + gen, gen > 0);
-
- // Make the next segments file, with last byte
- // missing, to simulate a writer that crashed while
- // writing segments file:
- String fileNameIn = SegmentInfos.getCurrentSegmentFileName(dir);
- String fileNameOut = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
- "",
- 1+gen);
- IndexInput in = dir.openInput(fileNameIn);
- IndexOutput out = dir.createOutput(fileNameOut);
- long length = in.length();
- for(int i=0;i<length-1;i++) {
- out.writeByte(in.readByte());
- }
- in.close();
- out.close();
-
- IndexReader reader = null;
- try {
- reader = IndexReader.open(dir, true);
- } catch (Exception e) {
- fail("reader failed to open on a crashed index");
- }
- reader.close();
-
- try {
- writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE));
- } catch (Exception e) {
- e.printStackTrace(System.out);
- fail("writer failed to open on a crashed index");
- }
-
- // add 100 documents
- for (int i = 0; i < 100; i++) {
- addDoc(writer);
- }
-
- // close
- writer.close();
- dir.close();
- }
-
- public void testAddDocsNonAbortingException() throws Exception {
- final Directory dir = newDirectory();
- final RandomIndexWriter w = new RandomIndexWriter(random, dir);
- final int numDocs1 = random.nextInt(25);
- for(int docCount=0;docCount<numDocs1;docCount++) {
- Document doc = new Document();
- doc.add(newField("content", "good content", Field.Index.ANALYZED));
- w.addDocument(doc);
- }
-
- final List<Document> docs = new ArrayList<Document>();
- for(int docCount=0;docCount<7;docCount++) {
- Document doc = new Document();
- docs.add(doc);
- doc.add(newField("id", docCount+"", Field.Index.NOT_ANALYZED));
- doc.add(newField("content", "silly content " + docCount, Field.Index.ANALYZED));
- if (docCount == 4) {
- Field f = newField("crash", "", Field.Index.ANALYZED);
- doc.add(f);
- MockTokenizer tokenizer = new MockTokenizer(new StringReader("crash me on the 4th token"), MockTokenizer.WHITESPACE, false);
- tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
- f.setTokenStream(new CrashingFilter("crash", tokenizer));
- }
- }
- try {
- w.addDocuments(docs);
- // BUG: CrashingFilter didn't
- fail("did not hit expected exception");
- } catch (IOException ioe) {
- // expected
- assertEquals(CRASH_FAIL_MESSAGE, ioe.getMessage());
- }
-
- final int numDocs2 = random.nextInt(25);
- for(int docCount=0;docCount<numDocs2;docCount++) {
- Document doc = new Document();
- doc.add(newField("content", "good content", Field.Index.ANALYZED));
- w.addDocument(doc);
- }
-
- final IndexReader r = w.getReader();
- w.close();
-
- final IndexSearcher s = new IndexSearcher(r);
- PhraseQuery pq = new PhraseQuery();
- pq.add(new Term("content", "silly"));
- pq.add(new Term("content", "content"));
- assertEquals(0, s.search(pq, 1).totalHits);
-
- pq = new PhraseQuery();
- pq.add(new Term("content", "good"));
- pq.add(new Term("content", "content"));
- assertEquals(numDocs1+numDocs2, s.search(pq, 1).totalHits);
- r.close();
- dir.close();
- }
-
-
- public void testUpdateDocsNonAbortingException() throws Exception {
- final Directory dir = newDirectory();
- final RandomIndexWriter w = new RandomIndexWriter(random, dir);
- final int numDocs1 = random.nextInt(25);
- for(int docCount=0;docCount<numDocs1;docCount++) {
- Document doc = new Document();
- doc.add(newField("content", "good content", Field.Index.ANALYZED));
- w.addDocument(doc);
- }
-
- // Use addDocs (no exception) to get docs in the index:
- final List<Document> docs = new ArrayList<Document>();
- final int numDocs2 = random.nextInt(25);
- for(int docCount=0;docCount<numDocs2;docCount++) {
- Document doc = new Document();
- docs.add(doc);
- doc.add(newField("subid", "subs", Field.Index.NOT_ANALYZED));
- doc.add(newField("id", docCount+"", Field.Index.NOT_ANALYZED));
- doc.add(newField("content", "silly content " + docCount, Field.Index.ANALYZED));
- }
- w.addDocuments(docs);
-
- final int numDocs3 = random.nextInt(25);
- for(int docCount=0;docCount<numDocs3;docCount++) {
- Document doc = new Document();
- doc.add(newField("content", "good content", Field.Index.ANALYZED));
- w.addDocument(doc);
- }
-
- docs.clear();
- final int limit = _TestUtil.nextInt(random, 2, 25);
- final int crashAt = random.nextInt(limit);
- for(int docCount=0;docCount<limit;docCount++) {
- Document doc = new Document();
- docs.add(doc);
- doc.add(newField("id", docCount+"", Field.Index.NOT_ANALYZED));
- doc.add(newField("content", "silly content " + docCount, Field.Index.ANALYZED));
- if (docCount == crashAt) {
- Field f = newField("crash", "", Field.Index.ANALYZED);
- doc.add(f);
- MockTokenizer tokenizer = new MockTokenizer(new StringReader("crash me on the 4th token"), MockTokenizer.WHITESPACE, false);
- tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
- f.setTokenStream(new CrashingFilter("crash", tokenizer));
- }
- }
-
- try {
- w.updateDocuments(new Term("subid", "subs"), docs);
- // BUG: CrashingFilter didn't
- fail("did not hit expected exception");
- } catch (IOException ioe) {
- // expected
- assertEquals(CRASH_FAIL_MESSAGE, ioe.getMessage());
- }
-
- final int numDocs4 = random.nextInt(25);
- for(int docCount=0;docCount<numDocs4;docCount++) {
- Document doc = new Document();
- doc.add(newField("content", "good content", Field.Index.ANALYZED));
- w.addDocument(doc);
- }
-
- final IndexReader r = w.getReader();
- w.close();
-
- final IndexSearcher s = new IndexSearcher(r);
- PhraseQuery pq = new PhraseQuery();
- pq.add(new Term("content", "silly"));
- pq.add(new Term("content", "content"));
- assertEquals(numDocs2, s.search(pq, 1).totalHits);
-
- pq = new PhraseQuery();
- pq.add(new Term("content", "good"));
- pq.add(new Term("content", "content"));
- assertEquals(numDocs1+numDocs3+numDocs4, s.search(pq, 1).totalHits);
- r.close();
- dir.close();
- }
-}
fnp / pylucene.git / blobdiff