X-Git-Url: https://git.mdrn.pl/pylucene.git/blobdiff_plain/a2e61f0c04805cfcb8706176758d1283c7e3a55c..aaeed5504b982cf3545252ab528713250aa33eed:/lucene-java-3.4.0/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java

diff --git a/lucene-java-3.4.0/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java b/lucene-java-3.4.0/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
deleted file mode 100644
index 2c2b0d4..0000000
--- a/lucene-java-3.4.0/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
+++ /dev/null
@@ -1,1296 +0,0 @@
-package org.apache.lucene.index;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.io.Reader;
-import java.io.StringReader;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util._TestUtil;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.store.MockDirectoryWrapper;
-import org.apache.lucene.store.RAMDirectory;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.analysis.MockTokenizer;
-import org.apache.lucene.analysis.TokenFilter;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.index.IndexWriterConfig.OpenMode;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.PhraseQuery;
-
-public class TestIndexWriterExceptions extends LuceneTestCase {
-
-  private class IndexerThread extends Thread {
-
-    IndexWriter writer;
-
-    final Random r = new Random(random.nextLong());
-    volatile Throwable failure;
-
-    public IndexerThread(int i, IndexWriter writer) {
-      setName("Indexer " + i);
-      this.writer = writer;
-    }
-
-    @Override
-    public void run() {
-
-      final Document doc = new Document();
-
-      doc.add(newField("content1", "aaa bbb ccc ddd", Field.Store.YES, Field.Index.ANALYZED));
-      doc.add(newField("content6", "aaa bbb ccc ddd", Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
-      doc.add(newField("content2", "aaa bbb ccc ddd", Field.Store.YES, Field.Index.NOT_ANALYZED));
-      doc.add(newField("content3", "aaa bbb ccc ddd", Field.Store.YES, Field.Index.NO));
-
-      doc.add(newField("content4", "aaa bbb ccc ddd", Field.Store.NO, Field.Index.ANALYZED));
-      doc.add(newField("content5", "aaa bbb ccc ddd", Field.Store.NO, Field.Index.NOT_ANALYZED));
-
-      doc.add(newField("content7", "aaa bbb ccc ddd", Field.Store.NO, Field.Index.NOT_ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
-
-      final Field idField = newField("id", "", Field.Store.YES, Field.Index.NOT_ANALYZED);
-      doc.add(idField);
-
-      final long stopTime = System.currentTimeMillis() + 500;
-
-      do {
-        if (VERBOSE) {
-          System.out.println(Thread.currentThread().getName() + ": TEST: IndexerThread: cycle");
-        }
-        doFail.set(this);
-        final String id = ""+r.nextInt(50);
-        idField.setValue(id);
-        Term idTerm = new Term("id", id);
-        try {
-          if (r.nextBoolean()) {
-            final List<Document> docs = new ArrayList<Document>();
-            final int count =  _TestUtil.nextInt(r, 1, 20);
-            for(int c=0;c<count;c++) {
-              docs.add(doc);
-            }
-            writer.updateDocuments(idTerm, docs);
-          } else {
-            writer.updateDocument(idTerm, doc);
-          }
-        } catch (RuntimeException re) {
-          if (VERBOSE) {
-            System.out.println(Thread.currentThread().getName() + ": EXC: ");
-            re.printStackTrace(System.out);
-          }
-          try {
-            _TestUtil.checkIndex(writer.getDirectory());
-          } catch (IOException ioe) {
-            System.out.println(Thread.currentThread().getName() + ": unexpected exception1");
-            ioe.printStackTrace(System.out);
-            failure = ioe;
-            break;
-          }
-        } catch (Throwable t) {
-          System.out.println(Thread.currentThread().getName() + ": unexpected exception2");
-          t.printStackTrace(System.out);
-          failure = t;
-          break;
-        }
-
-        doFail.set(null);
-
-        // After a possible exception (above) I should be able
-        // to add a new document without hitting an
-        // exception:
-        try {
-          writer.updateDocument(idTerm, doc);
-        } catch (Throwable t) {
-          System.out.println(Thread.currentThread().getName() + ": unexpected exception3");
-          t.printStackTrace(System.out);
-          failure = t;
-          break;
-        }
-      } while(System.currentTimeMillis() < stopTime);
-    }
-  }
-
-  ThreadLocal<Thread> doFail = new ThreadLocal<Thread>();
-
-  private class MockIndexWriter extends IndexWriter {
-    Random r = new Random(random.nextLong());
-
-    public MockIndexWriter(Directory dir, IndexWriterConfig conf) throws IOException {
-      super(dir, conf);
-    }
-
-    @Override
-    boolean testPoint(String name) {
-      if (doFail.get() != null && !name.equals("startDoFlush") && r.nextInt(40) == 17) {
-        if (VERBOSE) {
-          System.out.println(Thread.currentThread().getName() + ": NOW FAIL: " + name);
-          new Throwable().printStackTrace(System.out);
-        }
-        throw new RuntimeException(Thread.currentThread().getName() + ": intentionally failing at " + name);
-      }
-      return true;
-    }
-  }
-
-  public void testRandomExceptions() throws Throwable {
-    if (VERBOSE) {
-      System.out.println("\nTEST: start testRandomExceptions");
-    }
-    MockDirectoryWrapper dir = newDirectory();
-
-    MockAnalyzer analyzer = new MockAnalyzer(random);
-    analyzer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
-    MockIndexWriter writer  = new MockIndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer)
-        .setRAMBufferSizeMB(0.1).setMergeScheduler(new ConcurrentMergeScheduler()));
-    ((ConcurrentMergeScheduler) writer.getConfig().getMergeScheduler()).setSuppressExceptions();
-    //writer.setMaxBufferedDocs(10);
-    if (VERBOSE) {
-      System.out.println("TEST: initial commit");
-    }
-    writer.commit();
-
-    if (VERBOSE) {
-      writer.setInfoStream(System.out);
-    }
-
-    IndexerThread thread = new IndexerThread(0, writer);
-    thread.run();
-    if (thread.failure != null) {
-      thread.failure.printStackTrace(System.out);
-      fail("thread " + thread.getName() + ": hit unexpected failure");
-    }
-
-    if (VERBOSE) {
-      System.out.println("TEST: commit after thread start");
-    }
-    writer.commit();
-
-    try {
-      writer.close();
-    } catch (Throwable t) {
-      System.out.println("exception during close:");
-      t.printStackTrace(System.out);
-      writer.rollback();
-    }
-
-    // Confirm that when doc hits exception partway through tokenization, it's deleted:
-    IndexReader r2 = IndexReader.open(dir, true);
-    final int count = r2.docFreq(new Term("content4", "aaa"));
-    final int count2 = r2.docFreq(new Term("content4", "ddd"));
-    assertEquals(count, count2);
-    r2.close();
-
-    dir.close();
-  }
-
-  public void testRandomExceptionsThreads() throws Throwable {
-    MockDirectoryWrapper dir = newDirectory();
-    MockAnalyzer analyzer = new MockAnalyzer(random);
-    analyzer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
-    MockIndexWriter writer  = new MockIndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer)
-        .setRAMBufferSizeMB(0.2).setMergeScheduler(new ConcurrentMergeScheduler()));
-    ((ConcurrentMergeScheduler) writer.getConfig().getMergeScheduler()).setSuppressExceptions();
-    //writer.setMaxBufferedDocs(10);
-    writer.commit();
-
-    if (VERBOSE) {
-      writer.setInfoStream(System.out);
-    }
-
-    final int NUM_THREADS = 4;
-
-    final IndexerThread[] threads = new IndexerThread[NUM_THREADS];
-    for(int i=0;i<NUM_THREADS;i++) {
-      threads[i] = new IndexerThread(i, writer);
-      threads[i].start();
-    }
-
-    for(int i=0;i<NUM_THREADS;i++)
-      threads[i].join();
-
-    for(int i=0;i<NUM_THREADS;i++)
-      if (threads[i].failure != null)
-        fail("thread " + threads[i].getName() + ": hit unexpected failure");
-
-    writer.commit();
-
-    try {
-      writer.close();
-    } catch (Throwable t) {
-      System.out.println("exception during close:");
-      t.printStackTrace(System.out);
-      writer.rollback();
-    }
-
-    // Confirm that when doc hits exception partway through tokenization, it's deleted:
-    IndexReader r2 = IndexReader.open(dir, true);
-    final int count = r2.docFreq(new Term("content4", "aaa"));
-    final int count2 = r2.docFreq(new Term("content4", "ddd"));
-    assertEquals(count, count2);
-    r2.close();
-
-    dir.close();
-  }
-  
-  // LUCENE-1198
-  private static final class MockIndexWriter2 extends IndexWriter {
-
-    public MockIndexWriter2(Directory dir, IndexWriterConfig conf) throws IOException {
-      super(dir, conf);
-    }
-
-    boolean doFail;
-
-    @Override
-    boolean testPoint(String name) {
-      if (doFail && name.equals("DocumentsWriter.ThreadState.init start"))
-        throw new RuntimeException("intentionally failing");
-      return true;
-    }
-  }
-  
-  private static String CRASH_FAIL_MESSAGE = "I'm experiencing problems";
-
-  private class CrashingFilter extends TokenFilter {
-    String fieldName;
-    int count;
-
-    public CrashingFilter(String fieldName, TokenStream input) {
-      super(input);
-      this.fieldName = fieldName;
-    }
-
-    @Override
-    public boolean incrementToken() throws IOException {
-      if (this.fieldName.equals("crash") && count++ >= 4)
-        throw new IOException(CRASH_FAIL_MESSAGE);
-      return input.incrementToken();
-    }
-
-    @Override
-    public void reset() throws IOException {
-      super.reset();
-      count = 0;
-    }
-  }
-
-  public void testExceptionDocumentsWriterInit() throws IOException {
-    Directory dir = newDirectory();
-    MockIndexWriter2 w = new MockIndexWriter2(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
-    w.setInfoStream(VERBOSE ? System.out : null);
-    Document doc = new Document();
-    doc.add(newField("field", "a field", Field.Store.YES,
-                      Field.Index.ANALYZED));
-    w.addDocument(doc);
-    w.doFail = true;
-    try {
-      w.addDocument(doc);
-      fail("did not hit exception");
-    } catch (RuntimeException re) {
-      // expected
-    }
-    w.close();
-    dir.close();
-  }
-
-  // LUCENE-1208
-  public void testExceptionJustBeforeFlush() throws IOException {
-    Directory dir = newDirectory();
-    MockIndexWriter w = new MockIndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(2));
-    w.setInfoStream(VERBOSE ? System.out : null);
-    Document doc = new Document();
-    doc.add(newField("field", "a field", Field.Store.YES,
-                      Field.Index.ANALYZED));
-    w.addDocument(doc);
-
-    Analyzer analyzer = new Analyzer() {
-      @Override
-      public TokenStream tokenStream(String fieldName, Reader reader) {
-        MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
-        tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
-        return new CrashingFilter(fieldName, tokenizer);
-      }
-    };
-
-    Document crashDoc = new Document();
-    crashDoc.add(newField("crash", "do it on token 4", Field.Store.YES,
-                           Field.Index.ANALYZED));
-    try {
-      w.addDocument(crashDoc, analyzer);
-      fail("did not hit expected exception");
-    } catch (IOException ioe) {
-      // expected
-    }
-    w.addDocument(doc);
-    w.close();
-    dir.close();
-  }    
-
-  private static final class MockIndexWriter3 extends IndexWriter {
-
-    public MockIndexWriter3(Directory dir, IndexWriterConfig conf) throws IOException {
-      super(dir, conf);
-    }
-
-    boolean doFail;
-    boolean failed;
-
-    @Override
-    boolean testPoint(String name) {
-      if (doFail && name.equals("startMergeInit")) {
-        failed = true;
-        throw new RuntimeException("intentionally failing");
-      }
-      return true;
-    }
-  }
-  
-
-  // LUCENE-1210
-  public void testExceptionOnMergeInit() throws IOException {
-    Directory dir = newDirectory();
-    IndexWriterConfig conf = newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))
-      .setMaxBufferedDocs(2).setMergeScheduler(new ConcurrentMergeScheduler()).setMergePolicy(newLogMergePolicy());
-    ((LogMergePolicy) conf.getMergePolicy()).setMergeFactor(2);
-    MockIndexWriter3 w = new MockIndexWriter3(dir, conf);
-    w.doFail = true;
-    Document doc = new Document();
-    doc.add(newField("field", "a field", Field.Store.YES,
-                      Field.Index.ANALYZED));
-    for(int i=0;i<10;i++)
-      try {
-        w.addDocument(doc);
-      } catch (RuntimeException re) {
-        break;
-      }
-
-    ((ConcurrentMergeScheduler) w.getConfig().getMergeScheduler()).sync();
-    assertTrue(w.failed);
-    w.close();
-    dir.close();
-  }
-  
-  // LUCENE-1072
-  public void testExceptionFromTokenStream() throws IOException {
-    Directory dir = newDirectory();
-    IndexWriterConfig conf = newIndexWriterConfig( TEST_VERSION_CURRENT, new Analyzer() {
-
-      @Override
-      public TokenStream tokenStream(String fieldName, Reader reader) {
-        MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
-        tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
-        return new TokenFilter(tokenizer) {
-          private int count = 0;
-
-          @Override
-          public boolean incrementToken() throws IOException {
-            if (count++ == 5) {
-              throw new IOException();
-            }
-            return input.incrementToken();
-          }
-        };
-      }
-
-    });
-    conf.setMaxBufferedDocs(Math.max(3, conf.getMaxBufferedDocs()));
-
-    IndexWriter writer = new IndexWriter(dir, conf);
-
-    Document doc = new Document();
-    String contents = "aa bb cc dd ee ff gg hh ii jj kk";
-    doc.add(newField("content", contents, Field.Store.NO,
-        Field.Index.ANALYZED));
-    try {
-      writer.addDocument(doc);
-      fail("did not hit expected exception");
-    } catch (Exception e) {
-    }
-
-    // Make sure we can add another normal document
-    doc = new Document();
-    doc.add(newField("content", "aa bb cc dd", Field.Store.NO,
-        Field.Index.ANALYZED));
-    writer.addDocument(doc);
-
-    // Make sure we can add another normal document
-    doc = new Document();
-    doc.add(newField("content", "aa bb cc dd", Field.Store.NO,
-        Field.Index.ANALYZED));
-    writer.addDocument(doc);
-
-    writer.close();
-    IndexReader reader = IndexReader.open(dir, true);
-    final Term t = new Term("content", "aa");
-    assertEquals(3, reader.docFreq(t));
-
-    // Make sure the doc that hit the exception was marked
-    // as deleted:
-    TermDocs tdocs = reader.termDocs(t);
-    int count = 0;
-    while(tdocs.next()) {
-      count++;
-    }
-    assertEquals(2, count);
-
-    assertEquals(reader.docFreq(new Term("content", "gg")), 0);
-    reader.close();
-    dir.close();
-  }
-
-  private static class FailOnlyOnFlush extends MockDirectoryWrapper.Failure {
-    boolean doFail = false;
-    int count;
-
-    @Override
-    public void setDoFail() {
-      this.doFail = true;
-    }
-    @Override
-    public void clearDoFail() {
-      this.doFail = false;
-    }
-
-    @Override
-    public void eval(MockDirectoryWrapper dir)  throws IOException {
-      if (doFail) {
-        StackTraceElement[] trace = new Exception().getStackTrace();
-        boolean sawAppend = false;
-        boolean sawFlush = false;
-        for (int i = 0; i < trace.length; i++) {
-          if ("org.apache.lucene.index.FreqProxTermsWriter".equals(trace[i].getClassName()) && "appendPostings".equals(trace[i].getMethodName()))
-            sawAppend = true;
-          if ("doFlush".equals(trace[i].getMethodName()))
-            sawFlush = true;
-        }
-
-        if (sawAppend && sawFlush && count++ >= 30) {
-          doFail = false;
-          throw new IOException("now failing during flush");
-        }
-      }
-    }
-  }
-
-  // LUCENE-1072: make sure an errant exception on flushing
-  // one segment only takes out those docs in that one flush
-  public void testDocumentsWriterAbort() throws IOException {
-    MockDirectoryWrapper dir = newDirectory();
-    FailOnlyOnFlush failure = new FailOnlyOnFlush();
-    failure.setDoFail();
-    dir.failOn(failure);
-
-    IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(2));
-    Document doc = new Document();
-    String contents = "aa bb cc dd ee ff gg hh ii jj kk";
-    doc.add(newField("content", contents, Field.Store.NO,
-        Field.Index.ANALYZED));
-    boolean hitError = false;
-    for(int i=0;i<200;i++) {
-      try {
-        writer.addDocument(doc);
-      } catch (IOException ioe) {
-        // only one flush should fail:
-        assertFalse(hitError);
-        hitError = true;
-      }
-    }
-    assertTrue(hitError);
-    writer.close();
-    IndexReader reader = IndexReader.open(dir, true);
-    assertEquals(198, reader.docFreq(new Term("content", "aa")));
-    reader.close();
-    dir.close();
-  }
-
-  public void testDocumentsWriterExceptions() throws IOException {
-    Analyzer analyzer = new Analyzer() {
-      @Override
-      public TokenStream tokenStream(String fieldName, Reader reader) {
-        MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
-        tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
-        return new CrashingFilter(fieldName, tokenizer);
-      }
-    };
-
-    for(int i=0;i<2;i++) {
-      if (VERBOSE) {
-        System.out.println("TEST: cycle i=" + i);
-      }
-      MockDirectoryWrapper dir = newDirectory();
-      IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer).setMergePolicy(newLogMergePolicy()));
-      writer.setInfoStream(VERBOSE ? System.out : null);
-
-      // don't allow a sudden merge to clean up the deleted
-      // doc below:
-      LogMergePolicy lmp = (LogMergePolicy) writer.getConfig().getMergePolicy();
-      lmp.setMergeFactor(Math.max(lmp.getMergeFactor(), 5));
-
-      Document doc = new Document();
-      doc.add(newField("contents", "here are some contents", Field.Store.YES,
-                        Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
-      writer.addDocument(doc);
-      writer.addDocument(doc);
-      doc.add(newField("crash", "this should crash after 4 terms", Field.Store.YES,
-                        Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
-      doc.add(newField("other", "this will not get indexed", Field.Store.YES,
-                        Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
-      try {
-        writer.addDocument(doc);
-        fail("did not hit expected exception");
-      } catch (IOException ioe) {
-        if (VERBOSE) {
-          System.out.println("TEST: hit expected exception");
-          ioe.printStackTrace(System.out);
-        }
-      }
-
-      if (0 == i) {
-        doc = new Document();
-        doc.add(newField("contents", "here are some contents", Field.Store.YES,
-                          Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
-        writer.addDocument(doc);
-        writer.addDocument(doc);
-      }
-      writer.close();
-
-      if (VERBOSE) {
-        System.out.println("TEST: open reader");
-      }
-      IndexReader reader = IndexReader.open(dir, true);
-      if (i == 0) { 
-        int expected = 5;
-        assertEquals(expected, reader.docFreq(new Term("contents", "here")));
-        assertEquals(expected, reader.maxDoc());
-        int numDel = 0;
-        for(int j=0;j<reader.maxDoc();j++) {
-          if (reader.isDeleted(j))
-            numDel++;
-          else {
-            reader.document(j);
-            reader.getTermFreqVectors(j);
-          }
-        }
-        assertEquals(1, numDel);
-      }
-      reader.close();
-
-      writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT,
-          analyzer).setMaxBufferedDocs(10));
-      doc = new Document();
-      doc.add(newField("contents", "here are some contents", Field.Store.YES,
-                        Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
-      for(int j=0;j<17;j++)
-        writer.addDocument(doc);
-      writer.optimize();
-      writer.close();
-
-      reader = IndexReader.open(dir, true);
-      int expected = 19+(1-i)*2;
-      assertEquals(expected, reader.docFreq(new Term("contents", "here")));
-      assertEquals(expected, reader.maxDoc());
-      int numDel = 0;
-      for(int j=0;j<reader.maxDoc();j++) {
-        if (reader.isDeleted(j))
-          numDel++;
-        else {
-          reader.document(j);
-          reader.getTermFreqVectors(j);
-        }
-      }
-      reader.close();
-      assertEquals(0, numDel);
-
-      dir.close();
-    }
-  }
-
-  public void testDocumentsWriterExceptionThreads() throws Exception {
-    Analyzer analyzer = new Analyzer() {
-      @Override
-      public TokenStream tokenStream(String fieldName, Reader reader) {
-        MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
-        tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
-        return new CrashingFilter(fieldName, tokenizer);
-      }
-    };
-
-    final int NUM_THREAD = 3;
-    final int NUM_ITER = 100;
-
-    for(int i=0;i<2;i++) {
-      MockDirectoryWrapper dir = newDirectory();
-
-      {
-        final IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer).setMaxBufferedDocs(-1)
-                                                   .setMergePolicy(newLogMergePolicy(10)));
-        final int finalI = i;
-
-        Thread[] threads = new Thread[NUM_THREAD];
-        for(int t=0;t<NUM_THREAD;t++) {
-          threads[t] = new Thread() {
-              @Override
-              public void run() {
-                try {
-                  for(int iter=0;iter<NUM_ITER;iter++) {
-                    Document doc = new Document();
-                    doc.add(newField("contents", "here are some contents", Field.Store.YES,
-                                      Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
-                    writer.addDocument(doc);
-                    writer.addDocument(doc);
-                    doc.add(newField("crash", "this should crash after 4 terms", Field.Store.YES,
-                                      Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
-                    doc.add(newField("other", "this will not get indexed", Field.Store.YES,
-                                      Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
-                    try {
-                      writer.addDocument(doc);
-                      fail("did not hit expected exception");
-                    } catch (IOException ioe) {
-                    }
-
-                    if (0 == finalI) {
-                      doc = new Document();
-                      doc.add(newField("contents", "here are some contents", Field.Store.YES,
-                                        Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
-                      writer.addDocument(doc);
-                      writer.addDocument(doc);
-                    }
-                  }
-                } catch (Throwable t) {
-                  synchronized(this) {
-                    System.out.println(Thread.currentThread().getName() + ": ERROR: hit unexpected exception");
-                    t.printStackTrace(System.out);
-                  }
-                  fail();
-                }
-              }
-            };
-          threads[t].start();
-        }
-
-        for(int t=0;t<NUM_THREAD;t++)
-          threads[t].join();
-            
-        writer.close();
-      }
-
-      IndexReader reader = IndexReader.open(dir, true);
-      int expected = (3+(1-i)*2)*NUM_THREAD*NUM_ITER;
-      assertEquals("i=" + i, expected, reader.docFreq(new Term("contents", "here")));
-      assertEquals(expected, reader.maxDoc());
-      int numDel = 0;
-      for(int j=0;j<reader.maxDoc();j++) {
-        if (reader.isDeleted(j))
-          numDel++;
-        else {
-          reader.document(j);
-          reader.getTermFreqVectors(j);
-        }
-      }
-      reader.close();
-
-      assertEquals(NUM_THREAD*NUM_ITER, numDel);
-
-      IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
-          TEST_VERSION_CURRENT, analyzer).setMaxBufferedDocs(10));
-      Document doc = new Document();
-      doc.add(newField("contents", "here are some contents", Field.Store.YES,
-                        Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
-      for(int j=0;j<17;j++)
-        writer.addDocument(doc);
-      writer.optimize();
-      writer.close();
-
-      reader = IndexReader.open(dir, true);
-      expected += 17-NUM_THREAD*NUM_ITER;
-      assertEquals(expected, reader.docFreq(new Term("contents", "here")));
-      assertEquals(expected, reader.maxDoc());
-      numDel = 0;
-      for(int j=0;j<reader.maxDoc();j++) {
-        if (reader.isDeleted(j))
-          numDel++;
-        else {
-          reader.document(j);
-          reader.getTermFreqVectors(j);
-        }
-      }
-      reader.close();
-
-      dir.close();
-    }
-  }
-  
-  // Throws IOException during MockDirectoryWrapper.sync
-  private static class FailOnlyInSync extends MockDirectoryWrapper.Failure {
-    boolean didFail;
-    @Override
-    public void eval(MockDirectoryWrapper dir)  throws IOException {
-      if (doFail) {
-        StackTraceElement[] trace = new Exception().getStackTrace();
-        for (int i = 0; i < trace.length; i++) {
-          if (doFail && "org.apache.lucene.store.MockDirectoryWrapper".equals(trace[i].getClassName()) && "sync".equals(trace[i].getMethodName())) {
-            didFail = true;
-            throw new IOException("now failing on purpose during sync");
-          }
-        }
-      }
-    }
-  }
-  
-  // TODO: these are also in TestIndexWriter... add a simple doc-writing method
-  // like this to LuceneTestCase?
-  private void addDoc(IndexWriter writer) throws IOException
-  {
-      Document doc = new Document();
-      doc.add(newField("content", "aaa", Field.Store.NO, Field.Index.ANALYZED));
-      writer.addDocument(doc);
-  }
-  
-  // LUCENE-1044: test exception during sync
-  public void testExceptionDuringSync() throws IOException {
-    MockDirectoryWrapper dir = newDirectory();
-    FailOnlyInSync failure = new FailOnlyInSync();
-    dir.failOn(failure);
-
-    IndexWriter writer = new IndexWriter(
-        dir,
-        newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
-            setMaxBufferedDocs(2).
-            setMergeScheduler(new ConcurrentMergeScheduler()).
-            setMergePolicy(newLogMergePolicy(5))
-    );
-    failure.setDoFail();
-    ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(5);
-
-    for (int i = 0; i < 23; i++) {
-      addDoc(writer);
-      if ((i-1)%2 == 0) {
-        try {
-          writer.commit();
-        } catch (IOException ioe) {
-          // expected
-        }
-      }
-    }
-
-    ((ConcurrentMergeScheduler) writer.getConfig().getMergeScheduler()).sync();
-    assertTrue(failure.didFail);
-    failure.clearDoFail();
-    writer.close();
-
-    IndexReader reader = IndexReader.open(dir, true);
-    assertEquals(23, reader.numDocs());
-    reader.close();
-    dir.close();
-  }
-  
-  private static class FailOnlyInCommit extends MockDirectoryWrapper.Failure {
-
-    boolean fail1, fail2;
-
-    @Override
-    public void eval(MockDirectoryWrapper dir)  throws IOException {
-      StackTraceElement[] trace = new Exception().getStackTrace();
-      boolean isCommit = false;
-      boolean isDelete = false;
-      for (int i = 0; i < trace.length; i++) {
-        if ("org.apache.lucene.index.SegmentInfos".equals(trace[i].getClassName()) && "prepareCommit".equals(trace[i].getMethodName()))
-          isCommit = true;
-        if ("org.apache.lucene.store.MockDirectoryWrapper".equals(trace[i].getClassName()) && "deleteFile".equals(trace[i].getMethodName()))
-          isDelete = true;
-      }
-
-      if (isCommit) {
-        if (!isDelete) {
-          fail1 = true;
-          throw new RuntimeException("now fail first");
-        } else {
-          fail2 = true;
-          throw new IOException("now fail during delete");
-        }
-      }
-    }
-  }
-  
-  // LUCENE-1214
-  public void testExceptionsDuringCommit() throws Throwable {
-    MockDirectoryWrapper dir = newDirectory();
-    FailOnlyInCommit failure = new FailOnlyInCommit();
-    IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
-    Document doc = new Document();
-    doc.add(newField("field", "a field", Field.Store.YES,
-                      Field.Index.ANALYZED));
-    w.addDocument(doc);
-    dir.failOn(failure);
-    try {
-      w.close();
-      fail();
-    } catch (IOException ioe) {
-      fail("expected only RuntimeException");
-    } catch (RuntimeException re) {
-      // Expected
-    }
-    assertTrue(failure.fail1 && failure.fail2);
-    w.rollback();
-    dir.close();
-  }
-  
-  public void testOptimizeExceptions() throws IOException {
-    Directory startDir = newDirectory();
-    IndexWriterConfig conf = newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(2).setMergePolicy(newLogMergePolicy());
-    ((LogMergePolicy) conf.getMergePolicy()).setMergeFactor(100);
-    IndexWriter w = new IndexWriter(startDir, conf);
-    for(int i=0;i<27;i++)
-      addDoc(w);
-    w.close();
-
-    int iter = TEST_NIGHTLY ? 200 : 20;
-    for(int i=0;i<iter;i++) {
-      if (VERBOSE) {
-        System.out.println("TEST: iter " + i);
-      }
-      MockDirectoryWrapper dir = new MockDirectoryWrapper(random, new RAMDirectory(startDir));
-      conf = newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergeScheduler(new ConcurrentMergeScheduler());
-      ((ConcurrentMergeScheduler) conf.getMergeScheduler()).setSuppressExceptions();
-      w = new IndexWriter(dir, conf);
-      w.setInfoStream(VERBOSE ? System.out : null);
-      dir.setRandomIOExceptionRate(0.5);
-      try {
-        w.optimize();
-      } catch (IOException ioe) {
-        if (ioe.getCause() == null)
-          fail("optimize threw IOException without root cause");
-      }
-      dir.setRandomIOExceptionRate(0);
-      w.close();
-      dir.close();
-    }
-    startDir.close();
-  }
-  
-  // LUCENE-1429
-  public void testOutOfMemoryErrorCausesCloseToFail() throws Exception {
-
-    final List<Throwable> thrown = new ArrayList<Throwable>();
-    final Directory dir = newDirectory();
-    final IndexWriter writer = new IndexWriter(dir,
-        newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))) {
-        @Override
-        public void message(final String message) {
-          if (message.startsWith("now flush at close") && 0 == thrown.size()) {
-            thrown.add(null);
-            throw new OutOfMemoryError("fake OOME at " + message);
-          }
-        }
-      };
-
-    // need to set an info stream so message is called
-    writer.setInfoStream(new PrintStream(new ByteArrayOutputStream()));
-    try {
-      writer.close();
-      fail("OutOfMemoryError expected");
-    }
-    catch (final OutOfMemoryError expected) {}
-
-    // throws IllegalStateEx w/o bug fix
-    writer.close();
-    dir.close();
-  }
-  
-  // LUCENE-1347
-  private static final class MockIndexWriter4 extends IndexWriter {
-
-    public MockIndexWriter4(Directory dir, IndexWriterConfig conf) throws IOException {
-      super(dir, conf);
-    }
-
-    boolean doFail;
-
-    @Override
-    boolean testPoint(String name) {
-      if (doFail && name.equals("rollback before checkpoint"))
-        throw new RuntimeException("intentionally failing");
-      return true;
-    }
-  }
-  
-  // LUCENE-1347
-  public void testRollbackExceptionHang() throws Throwable {
-    Directory dir = newDirectory();
-    MockIndexWriter4 w = new MockIndexWriter4(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
-
-    addDoc(w);
-    w.doFail = true;
-    try {
-      w.rollback();
-      fail("did not hit intentional RuntimeException");
-    } catch (RuntimeException re) {
-      // expected
-    }
-    
-    w.doFail = false;
-    w.rollback();
-    dir.close();
-  }
-  
-  // LUCENE-1044: Simulate checksum error in segments_N
-  public void testSegmentsChecksumError() throws IOException {
-    Directory dir = newDirectory();
-
-    IndexWriter writer = null;
-
-    writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
-
-    // add 100 documents
-    for (int i = 0; i < 100; i++) {
-      addDoc(writer);
-    }
-
-    // close
-    writer.close();
-
-    long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
-    assertTrue("segment generation should be > 0 but got " + gen, gen > 0);
-
-    final String segmentsFileName = SegmentInfos.getCurrentSegmentFileName(dir);
-    IndexInput in = dir.openInput(segmentsFileName);
-    IndexOutput out = dir.createOutput(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", 1+gen));
-    out.copyBytes(in, in.length()-1);
-    byte b = in.readByte();
-    out.writeByte((byte) (1+b));
-    out.close();
-    in.close();
-
-    IndexReader reader = null;
-    try {
-      reader = IndexReader.open(dir, true);
-    } catch (IOException e) {
-      e.printStackTrace(System.out);
-      fail("segmentInfos failed to retry fallback to correct segments_N file");
-    }
-    reader.close();
-    dir.close();
-  }
-  
-  // Simulate a corrupt index by removing last byte of
-  // latest segments file and make sure we get an
-  // IOException trying to open the index:
-  public void testSimulatedCorruptIndex1() throws IOException {
-      MockDirectoryWrapper dir = newDirectory();
-      dir.setCheckIndexOnClose(false); // we are corrupting it!
-
-      IndexWriter writer = null;
-
-      writer  = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
-
-      // add 100 documents
-      for (int i = 0; i < 100; i++) {
-          addDoc(writer);
-      }
-
-      // close
-      writer.close();
-
-      long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
-      assertTrue("segment generation should be > 0 but got " + gen, gen > 0);
-
-      String fileNameIn = SegmentInfos.getCurrentSegmentFileName(dir);
-      String fileNameOut = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
-                                                                 "",
-                                                                 1+gen);
-      IndexInput in = dir.openInput(fileNameIn);
-      IndexOutput out = dir.createOutput(fileNameOut);
-      long length = in.length();
-      for(int i=0;i<length-1;i++) {
-        out.writeByte(in.readByte());
-      }
-      in.close();
-      out.close();
-      dir.deleteFile(fileNameIn);
-
-      IndexReader reader = null;
-      try {
-        reader = IndexReader.open(dir, true);
-        fail("reader did not hit IOException on opening a corrupt index");
-      } catch (Exception e) {
-      }
-      if (reader != null) {
-        reader.close();
-      }
-      dir.close();
-  }
-  
-  // Simulate a corrupt index by removing one of the cfs
-  // files and make sure we get an IOException trying to
-  // open the index:
-  public void testSimulatedCorruptIndex2() throws IOException {
-      MockDirectoryWrapper dir = newDirectory();
-      dir.setCheckIndexOnClose(false); // we are corrupting it!
-      IndexWriter writer = null;
-
-      writer  = new IndexWriter(
-          dir,
-          newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
-              setMergePolicy(newLogMergePolicy(true))
-      );
-      ((LogMergePolicy) writer.getConfig().getMergePolicy()).setNoCFSRatio(1.0);
-
-      // add 100 documents
-      for (int i = 0; i < 100; i++) {
-          addDoc(writer);
-      }
-
-      // close
-      writer.close();
-
-      long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
-      assertTrue("segment generation should be > 0 but got " + gen, gen > 0);
-
-      String[] files = dir.listAll();
-      boolean corrupted = false;
-      for(int i=0;i<files.length;i++) {
-        if (files[i].endsWith(".cfs")) {
-          dir.deleteFile(files[i]);
-          corrupted = true;
-          break;
-        }
-      }
-      assertTrue("failed to find cfs file to remove", corrupted);
-
-      IndexReader reader = null;
-      try {
-        reader = IndexReader.open(dir, true);
-        fail("reader did not hit IOException on opening a corrupt index");
-      } catch (Exception e) {
-      }
-      if (reader != null) {
-        reader.close();
-      }
-      dir.close();
-  }
-  
-  // Simulate a writer that crashed while writing segments
-  // file: make sure we can still open the index (ie,
-  // gracefully fallback to the previous segments file),
-  // and that we can add to the index:
-  public void testSimulatedCrashedWriter() throws IOException {
-      MockDirectoryWrapper dir = newDirectory();
-      dir.setPreventDoubleWrite(false);
-
-      IndexWriter writer = null;
-
-      writer  = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
-
-      // add 100 documents
-      for (int i = 0; i < 100; i++) {
-          addDoc(writer);
-      }
-
-      // close
-      writer.close();
-
-      long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
-      assertTrue("segment generation should be > 0 but got " + gen, gen > 0);
-
-      // Make the next segments file, with last byte
-      // missing, to simulate a writer that crashed while
-      // writing segments file:
-      String fileNameIn = SegmentInfos.getCurrentSegmentFileName(dir);
-      String fileNameOut = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
-                                                                 "",
-                                                                 1+gen);
-      IndexInput in = dir.openInput(fileNameIn);
-      IndexOutput out = dir.createOutput(fileNameOut);
-      long length = in.length();
-      for(int i=0;i<length-1;i++) {
-        out.writeByte(in.readByte());
-      }
-      in.close();
-      out.close();
-
-      IndexReader reader = null;
-      try {
-        reader = IndexReader.open(dir, true);
-      } catch (Exception e) {
-        fail("reader failed to open on a crashed index");
-      }
-      reader.close();
-
-      try {
-        writer  = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE));
-      } catch (Exception e) {
-        e.printStackTrace(System.out);
-        fail("writer failed to open on a crashed index");
-      }
-
-      // add 100 documents
-      for (int i = 0; i < 100; i++) {
-          addDoc(writer);
-      }
-
-      // close
-      writer.close();
-      dir.close();
-  }
-
-  public void testAddDocsNonAbortingException() throws Exception {
-    final Directory dir = newDirectory();
-    final RandomIndexWriter w = new RandomIndexWriter(random, dir);
-    final int numDocs1 = random.nextInt(25);
-    for(int docCount=0;docCount<numDocs1;docCount++) {
-      Document doc = new Document();
-      doc.add(newField("content", "good content", Field.Index.ANALYZED));
-      w.addDocument(doc);
-    }
-    
-    final List<Document> docs = new ArrayList<Document>();
-    for(int docCount=0;docCount<7;docCount++) {
-      Document doc = new Document();
-      docs.add(doc);
-      doc.add(newField("id", docCount+"", Field.Index.NOT_ANALYZED));
-      doc.add(newField("content", "silly content " + docCount, Field.Index.ANALYZED));
-      if (docCount == 4) {
-        Field f = newField("crash", "", Field.Index.ANALYZED);
-        doc.add(f);
-        MockTokenizer tokenizer = new MockTokenizer(new StringReader("crash me on the 4th token"), MockTokenizer.WHITESPACE, false);
-        tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
-        f.setTokenStream(new CrashingFilter("crash", tokenizer));
-      }
-    }
-    try {
-      w.addDocuments(docs);
-      // BUG: CrashingFilter didn't
-      fail("did not hit expected exception");
-    } catch (IOException ioe) {
-      // expected
-      assertEquals(CRASH_FAIL_MESSAGE, ioe.getMessage());
-    }
-
-    final int numDocs2 = random.nextInt(25);
-    for(int docCount=0;docCount<numDocs2;docCount++) {
-      Document doc = new Document();
-      doc.add(newField("content", "good content", Field.Index.ANALYZED));
-      w.addDocument(doc);
-    }
-
-    final IndexReader r = w.getReader();
-    w.close();
-
-    final IndexSearcher s = new IndexSearcher(r);
-    PhraseQuery pq = new PhraseQuery();
-    pq.add(new Term("content", "silly"));
-    pq.add(new Term("content", "content"));
-    assertEquals(0, s.search(pq, 1).totalHits);
-
-    pq = new PhraseQuery();
-    pq.add(new Term("content", "good"));
-    pq.add(new Term("content", "content"));
-    assertEquals(numDocs1+numDocs2, s.search(pq, 1).totalHits);
-    r.close();
-    dir.close();
-  }
-
-
-  public void testUpdateDocsNonAbortingException() throws Exception {
-    final Directory dir = newDirectory();
-    final RandomIndexWriter w = new RandomIndexWriter(random, dir);
-    final int numDocs1 = random.nextInt(25);
-    for(int docCount=0;docCount<numDocs1;docCount++) {
-      Document doc = new Document();
-      doc.add(newField("content", "good content", Field.Index.ANALYZED));
-      w.addDocument(doc);
-    }
-
-    // Use addDocs (no exception) to get docs in the index:
-    final List<Document> docs = new ArrayList<Document>();
-    final int numDocs2 = random.nextInt(25);
-    for(int docCount=0;docCount<numDocs2;docCount++) {
-      Document doc = new Document();
-      docs.add(doc);
-      doc.add(newField("subid", "subs", Field.Index.NOT_ANALYZED));
-      doc.add(newField("id", docCount+"", Field.Index.NOT_ANALYZED));
-      doc.add(newField("content", "silly content " + docCount, Field.Index.ANALYZED));
-    }
-    w.addDocuments(docs);
-
-    final int numDocs3 = random.nextInt(25);
-    for(int docCount=0;docCount<numDocs3;docCount++) {
-      Document doc = new Document();
-      doc.add(newField("content", "good content", Field.Index.ANALYZED));
-      w.addDocument(doc);
-    }
-
-    docs.clear();
-    final int limit = _TestUtil.nextInt(random, 2, 25);
-    final int crashAt = random.nextInt(limit);
-    for(int docCount=0;docCount<limit;docCount++) {
-      Document doc = new Document();
-      docs.add(doc);
-      doc.add(newField("id", docCount+"", Field.Index.NOT_ANALYZED));
-      doc.add(newField("content", "silly content " + docCount, Field.Index.ANALYZED));
-      if (docCount == crashAt) {
-        Field f = newField("crash", "", Field.Index.ANALYZED);
-        doc.add(f);
-        MockTokenizer tokenizer = new MockTokenizer(new StringReader("crash me on the 4th token"), MockTokenizer.WHITESPACE, false);
-        tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
-        f.setTokenStream(new CrashingFilter("crash", tokenizer));
-      }
-    }
-
-    try {
-      w.updateDocuments(new Term("subid", "subs"), docs);
-      // BUG: CrashingFilter didn't
-      fail("did not hit expected exception");
-    } catch (IOException ioe) {
-      // expected
-      assertEquals(CRASH_FAIL_MESSAGE, ioe.getMessage());
-    }
-
-    final int numDocs4 = random.nextInt(25);
-    for(int docCount=0;docCount<numDocs4;docCount++) {
-      Document doc = new Document();
-      doc.add(newField("content", "good content", Field.Index.ANALYZED));
-      w.addDocument(doc);
-    }
-
-    final IndexReader r = w.getReader();
-    w.close();
-
-    final IndexSearcher s = new IndexSearcher(r);
-    PhraseQuery pq = new PhraseQuery();
-    pq.add(new Term("content", "silly"));
-    pq.add(new Term("content", "content"));
-    assertEquals(numDocs2, s.search(pq, 1).totalHits);
-
-    pq = new PhraseQuery();
-    pq.add(new Term("content", "good"));
-    pq.add(new Term("content", "content"));
-    assertEquals(numDocs1+numDocs3+numDocs4, s.search(pq, 1).totalHits);
-    r.close();
-    dir.close();
-  }
-}