1 package org.apache.lucene.index;
4 * Licensed to the Apache Software Foundation (ASF) under one or more
5 * contributor license agreements. See the NOTICE file distributed with
6 * this work for additional information regarding copyright ownership.
7 * The ASF licenses this file to You under the Apache License, Version 2.0
8 * (the "License"); you may not use this file except in compliance with
9 * the License. You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
20 import java.io.IOException;
22 import org.apache.lucene.analysis.MockAnalyzer;
23 import org.apache.lucene.document.Document;
24 import org.apache.lucene.document.Field;
25 import org.apache.lucene.store.AlreadyClosedException;
26 import org.apache.lucene.store.Directory;
27 import org.apache.lucene.store.MockDirectoryWrapper;
28 import org.apache.lucene.util.LuceneTestCase;
29 import org.apache.lucene.util.ThreadInterruptedException;
32 * MultiThreaded IndexWriter tests
34 public class TestIndexWriterWithThreads extends LuceneTestCase {
36 // Used by test cases below
37 private class IndexerThread extends Thread {
41 AlreadyClosedException ace;
44 volatile int addCount;
46 public IndexerThread(IndexWriter writer, boolean noErrors) {
48 this.noErrors = noErrors;
54 final Document doc = new Document();
55 doc.add(newField("field", "aaa bbb ccc ddd eee fff ggg hhh iii jjj", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
59 final long stopTime = System.currentTimeMillis() + 200;
63 writer.updateDocument(new Term("id", ""+(idUpto++)), doc);
65 } catch (IOException ioe) {
67 System.out.println("TEST: expected exc:");
68 ioe.printStackTrace(System.out);
70 //System.out.println(Thread.currentThread().getName() + ": hit exc");
71 //ioe.printStackTrace(System.out);
72 if (ioe.getMessage().startsWith("fake disk full at") ||
73 ioe.getMessage().equals("now failing on purpose")) {
77 } catch (InterruptedException ie) {
78 throw new ThreadInterruptedException(ie);
84 System.out.println(Thread.currentThread().getName() + ": ERROR: unexpected IOException:");
85 ioe.printStackTrace(System.out);
90 } catch (Throwable t) {
91 //t.printStackTrace(System.out);
93 System.out.println(Thread.currentThread().getName() + ": ERROR: unexpected Throwable:");
94 t.printStackTrace(System.out);
99 } while(System.currentTimeMillis() < stopTime);
103 // LUCENE-1130: make sure immediate disk full on creating
104 // an IndexWriter (hit during DW.ThreadState.init()), with
105 // multiple threads, is OK:
106 public void testImmediateDiskFullWithThreads() throws Exception {
110 for(int iter=0;iter<10;iter++) {
112 System.out.println("\nTEST: iter=" + iter);
114 MockDirectoryWrapper dir = newDirectory();
115 IndexWriter writer = new IndexWriter(
117 newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
118 setMaxBufferedDocs(2).
119 setMergeScheduler(new ConcurrentMergeScheduler()).
120 setMergePolicy(newLogMergePolicy(4))
122 ((ConcurrentMergeScheduler) writer.getConfig().getMergeScheduler()).setSuppressExceptions();
123 dir.setMaxSizeInBytes(4*1024+20*iter);
124 writer.setInfoStream(VERBOSE ? System.out : null);
126 IndexerThread[] threads = new IndexerThread[NUM_THREADS];
128 for(int i=0;i<NUM_THREADS;i++)
129 threads[i] = new IndexerThread(writer, true);
131 for(int i=0;i<NUM_THREADS;i++)
134 for(int i=0;i<NUM_THREADS;i++) {
135 // Without fix for LUCENE-1130: one of the
138 assertTrue("hit unexpected Throwable", threads[i].error == null);
141 // Make sure once disk space is avail again, we can
143 dir.setMaxSizeInBytes(0);
150 // LUCENE-1130: make sure we can close() even while
151 // threads are trying to add documents. Strictly
152 // speaking, this isn't valid us of Lucene's APIs, but we
153 // still want to be robust to this case:
154 public void testCloseWithThreads() throws Exception {
157 for(int iter=0;iter<7;iter++) {
158 Directory dir = newDirectory();
159 IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))
160 .setMaxBufferedDocs(10).setMergeScheduler(new ConcurrentMergeScheduler()).setMergePolicy(newLogMergePolicy(4));
161 // We expect AlreadyClosedException
162 ((ConcurrentMergeScheduler) conf.getMergeScheduler()).setSuppressExceptions();
163 IndexWriter writer = new IndexWriter(dir, conf);
165 IndexerThread[] threads = new IndexerThread[NUM_THREADS];
167 for(int i=0;i<NUM_THREADS;i++)
168 threads[i] = new IndexerThread(writer, false);
170 for(int i=0;i<NUM_THREADS;i++)
173 boolean done = false;
176 for(int i=0;i<NUM_THREADS;i++)
177 // only stop when at least one thread has added a doc
178 if (threads[i].addCount > 0) {
186 // Make sure threads that are adding docs are not hung:
187 for(int i=0;i<NUM_THREADS;i++) {
188 // Without fix for LUCENE-1130: one of the
191 if (threads[i].isAlive())
192 fail("thread seems to be hung");
195 // Quick test to make sure index is not corrupt:
196 IndexReader reader = IndexReader.open(dir, true);
197 TermDocs tdocs = reader.termDocs(new Term("field", "aaa"));
199 while(tdocs.next()) {
202 assertTrue(count > 0);
209 // Runs test, with multiple threads, using the specific
210 // failure to trigger an IOException
211 public void _testMultipleThreadsFailure(MockDirectoryWrapper.Failure failure) throws Exception {
215 for(int iter=0;iter<2;iter++) {
217 System.out.println("TEST: iter=" + iter);
219 MockDirectoryWrapper dir = newDirectory();
220 IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT,
221 new MockAnalyzer(random)).setMaxBufferedDocs(2)
222 .setMergeScheduler(new ConcurrentMergeScheduler())
223 .setMergePolicy(newLogMergePolicy(4));
224 // We expect disk full exceptions in the merge threads
225 ((ConcurrentMergeScheduler) conf.getMergeScheduler()).setSuppressExceptions();
226 IndexWriter writer = new IndexWriter(dir, conf);
227 writer.setInfoStream(VERBOSE ? System.out : null);
229 IndexerThread[] threads = new IndexerThread[NUM_THREADS];
231 for(int i=0;i<NUM_THREADS;i++)
232 threads[i] = new IndexerThread(writer, true);
234 for(int i=0;i<NUM_THREADS;i++)
242 for(int i=0;i<NUM_THREADS;i++) {
244 assertTrue("hit unexpected Throwable", threads[i].error == null);
247 boolean success = false;
251 } catch (IOException ioe) {
252 failure.clearDoFail();
257 IndexReader reader = IndexReader.open(dir, true);
258 for(int j=0;j<reader.maxDoc();j++) {
259 if (!reader.isDeleted(j)) {
261 reader.getTermFreqVectors(j);
271 // Runs test, with one thread, using the specific failure
272 // to trigger an IOException
273 public void _testSingleThreadFailure(MockDirectoryWrapper.Failure failure) throws IOException {
274 MockDirectoryWrapper dir = newDirectory();
276 IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))
277 .setMaxBufferedDocs(2).setMergeScheduler(new ConcurrentMergeScheduler()));
278 final Document doc = new Document();
279 doc.add(newField("field", "aaa bbb ccc ddd eee fff ggg hhh iii jjj", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
282 writer.addDocument(doc);
287 writer.addDocument(doc);
288 writer.addDocument(doc);
290 fail("did not hit exception");
291 } catch (IOException ioe) {
293 failure.clearDoFail();
294 writer.addDocument(doc);
299 // Throws IOException during FieldsWriter.flushDocument and during DocumentsWriter.abort
300 private static class FailOnlyOnAbortOrFlush extends MockDirectoryWrapper.Failure {
301 private boolean onlyOnce;
302 public FailOnlyOnAbortOrFlush(boolean onlyOnce) {
303 this.onlyOnce = onlyOnce;
306 public void eval(MockDirectoryWrapper dir) throws IOException {
308 StackTraceElement[] trace = new Exception().getStackTrace();
309 boolean sawAbortOrFlushDoc = false;
310 boolean sawClose = false;
311 for (int i = 0; i < trace.length; i++) {
312 if ("abort".equals(trace[i].getMethodName()) ||
313 "flushDocument".equals(trace[i].getMethodName())) {
314 sawAbortOrFlushDoc = true;
316 if ("close".equals(trace[i].getMethodName())) {
320 if (sawAbortOrFlushDoc && !sawClose) {
323 //System.out.println(Thread.currentThread().getName() + ": now fail");
324 //new Throwable().printStackTrace(System.out);
325 throw new IOException("now failing on purpose");
333 // LUCENE-1130: make sure initial IOException, and then 2nd
334 // IOException during rollback(), is OK:
335 public void testIOExceptionDuringAbort() throws IOException {
336 _testSingleThreadFailure(new FailOnlyOnAbortOrFlush(false));
339 // LUCENE-1130: make sure initial IOException, and then 2nd
340 // IOException during rollback(), is OK:
341 public void testIOExceptionDuringAbortOnlyOnce() throws IOException {
342 _testSingleThreadFailure(new FailOnlyOnAbortOrFlush(true));
345 // LUCENE-1130: make sure initial IOException, and then 2nd
346 // IOException during rollback(), with multiple threads, is OK:
347 public void testIOExceptionDuringAbortWithThreads() throws Exception {
348 _testMultipleThreadsFailure(new FailOnlyOnAbortOrFlush(false));
351 // LUCENE-1130: make sure initial IOException, and then 2nd
352 // IOException during rollback(), with multiple threads, is OK:
353 public void testIOExceptionDuringAbortWithThreadsOnlyOnce() throws Exception {
354 _testMultipleThreadsFailure(new FailOnlyOnAbortOrFlush(true));
357 // Throws IOException during DocumentsWriter.writeSegment
358 private static class FailOnlyInWriteSegment extends MockDirectoryWrapper.Failure {
359 private boolean onlyOnce;
360 public FailOnlyInWriteSegment(boolean onlyOnce) {
361 this.onlyOnce = onlyOnce;
364 public void eval(MockDirectoryWrapper dir) throws IOException {
366 StackTraceElement[] trace = new Exception().getStackTrace();
367 for (int i = 0; i < trace.length; i++) {
368 if ("flush".equals(trace[i].getMethodName()) && "org.apache.lucene.index.DocFieldProcessor".equals(trace[i].getClassName())) {
371 throw new IOException("now failing on purpose");
378 // LUCENE-1130: test IOException in writeSegment
379 public void testIOExceptionDuringWriteSegment() throws IOException {
380 _testSingleThreadFailure(new FailOnlyInWriteSegment(false));
383 // LUCENE-1130: test IOException in writeSegment
384 public void testIOExceptionDuringWriteSegmentOnlyOnce() throws IOException {
385 _testSingleThreadFailure(new FailOnlyInWriteSegment(true));
388 // LUCENE-1130: test IOException in writeSegment, with threads
389 public void testIOExceptionDuringWriteSegmentWithThreads() throws Exception {
390 _testMultipleThreadsFailure(new FailOnlyInWriteSegment(false));
393 // LUCENE-1130: test IOException in writeSegment, with threads
394 public void testIOExceptionDuringWriteSegmentWithThreadsOnlyOnce() throws Exception {
395 _testMultipleThreadsFailure(new FailOnlyInWriteSegment(true));