--- /dev/null
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.store.MockDirectoryWrapper;
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.IndexWriterConfig.OpenMode;
+
+import org.apache.lucene.util.LuceneTestCase;
+import java.io.IOException;
+
+public class TestConcurrentMergeScheduler extends LuceneTestCase {
+
+ private class FailOnlyOnFlush extends MockDirectoryWrapper.Failure {
+ boolean doFail;
+ boolean hitExc;
+
+ @Override
+ public void setDoFail() {
+ this.doFail = true;
+ hitExc = false;
+ }
+ @Override
+ public void clearDoFail() {
+ this.doFail = false;
+ }
+
+ @Override
+ public void eval(MockDirectoryWrapper dir) throws IOException {
+ if (doFail && isTestThread()) {
+ boolean isDoFlush = false;
+ boolean isClose = false;
+ StackTraceElement[] trace = new Exception().getStackTrace();
+ for (int i = 0; i < trace.length; i++) {
+ if ("doFlush".equals(trace[i].getMethodName())) {
+ isDoFlush = true;
+ }
+ if ("close".equals(trace[i].getMethodName())) {
+ isClose = true;
+ }
+ }
+ if (isDoFlush && !isClose && random.nextBoolean()) {
+ hitExc = true;
+ throw new IOException(Thread.currentThread().getName() + ": now failing during flush");
+ }
+ }
+ }
+ }
+
+ // Make sure running BG merges still work fine even when
+ // we are hitting exceptions during flushing.
+ public void testFlushExceptions() throws IOException {
+ MockDirectoryWrapper directory = newDirectory();
+ FailOnlyOnFlush failure = new FailOnlyOnFlush();
+ directory.failOn(failure);
+
+ IndexWriter writer = new IndexWriter(directory, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(2));
+ writer.setInfoStream(VERBOSE ? System.out : null);
+ Document doc = new Document();
+ Field idField = newField("id", "", Field.Store.YES, Field.Index.NOT_ANALYZED);
+ doc.add(idField);
+ int extraCount = 0;
+
+ for(int i=0;i<10;i++) {
+ if (VERBOSE) {
+ System.out.println("TEST: iter=" + i);
+ }
+
+ for(int j=0;j<20;j++) {
+ idField.setValue(Integer.toString(i*20+j));
+ writer.addDocument(doc);
+ }
+
+ // must cycle here because sometimes the merge flushes
+ // the doc we just added and so there's nothing to
+ // flush, and we don't hit the exception
+ while(true) {
+ writer.addDocument(doc);
+ failure.setDoFail();
+ try {
+ writer.flush(true, true);
+ if (failure.hitExc) {
+ fail("failed to hit IOException");
+ }
+ extraCount++;
+ } catch (IOException ioe) {
+ if (VERBOSE) {
+ ioe.printStackTrace(System.out);
+ }
+ failure.clearDoFail();
+ break;
+ }
+ }
+ assertEquals(20*(i+1)+extraCount, writer.numDocs());
+ }
+
+ writer.close();
+ IndexReader reader = IndexReader.open(directory, true);
+ assertEquals(200+extraCount, reader.numDocs());
+ reader.close();
+ directory.close();
+ }
+
+ // Test that deletes committed after a merge started and
+ // before it finishes, are correctly merged back:
+ public void testDeleteMerging() throws IOException {
+ MockDirectoryWrapper directory = newDirectory();
+
+ LogDocMergePolicy mp = new LogDocMergePolicy();
+ // Force degenerate merging so we can get a mix of
+ // merging of segments with and without deletes at the
+ // start:
+ mp.setMinMergeDocs(1000);
+ IndexWriter writer = new IndexWriter(directory, newIndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer(random))
+ .setMergePolicy(mp));
+ writer.setInfoStream(VERBOSE ? System.out : null);
+
+ Document doc = new Document();
+ Field idField = newField("id", "", Field.Store.YES, Field.Index.NOT_ANALYZED);
+ doc.add(idField);
+ for(int i=0;i<10;i++) {
+ if (VERBOSE) {
+ System.out.println("\nTEST: cycle");
+ }
+ for(int j=0;j<100;j++) {
+ idField.setValue(Integer.toString(i*100+j));
+ writer.addDocument(doc);
+ }
+
+ int delID = i;
+ while(delID < 100*(1+i)) {
+ if (VERBOSE) {
+ System.out.println("TEST: del " + delID);
+ }
+ writer.deleteDocuments(new Term("id", ""+delID));
+ delID += 10;
+ }
+
+ writer.commit();
+ }
+
+ writer.close();
+ IndexReader reader = IndexReader.open(directory, true);
+ // Verify that we did not lose any deletes...
+ assertEquals(450, reader.numDocs());
+ reader.close();
+ directory.close();
+ }
+
+ public void testNoExtraFiles() throws IOException {
+ MockDirectoryWrapper directory = newDirectory();
+ IndexWriter writer = new IndexWriter(directory, newIndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer(random))
+ .setMaxBufferedDocs(2));
+ writer.setInfoStream(VERBOSE ? System.out : null);
+
+ for(int iter=0;iter<7;iter++) {
+ if (VERBOSE) {
+ System.out.println("TEST: iter=" + iter);
+ }
+
+ for(int j=0;j<21;j++) {
+ Document doc = new Document();
+ doc.add(newField("content", "a b c", Field.Store.NO, Field.Index.ANALYZED));
+ writer.addDocument(doc);
+ }
+
+ writer.close();
+ TestIndexWriter.assertNoUnreferencedFiles(directory, "testNoExtraFiles");
+
+ // Reopen
+ writer = new IndexWriter(directory, newIndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer(random))
+ .setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(2));
+ writer.setInfoStream(VERBOSE ? System.out : null);
+ }
+
+ writer.close();
+
+ directory.close();
+ }
+
+ public void testNoWaitClose() throws IOException {
+ MockDirectoryWrapper directory = newDirectory();
+ Document doc = new Document();
+ Field idField = newField("id", "", Field.Store.YES, Field.Index.NOT_ANALYZED);
+ doc.add(idField);
+
+ IndexWriter writer = new IndexWriter(
+ directory,
+ newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
+ setMaxBufferedDocs(2).
+ setMergePolicy(newLogMergePolicy(100))
+ );
+
+ for(int iter=0;iter<10;iter++) {
+
+ for(int j=0;j<201;j++) {
+ idField.setValue(Integer.toString(iter*201+j));
+ writer.addDocument(doc);
+ }
+
+ int delID = iter*201;
+ for(int j=0;j<20;j++) {
+ writer.deleteDocuments(new Term("id", Integer.toString(delID)));
+ delID += 5;
+ }
+
+ // Force a bunch of merge threads to kick off so we
+ // stress out aborting them on close:
+ ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(3);
+ writer.addDocument(doc);
+ writer.commit();
+
+ writer.close(false);
+
+ IndexReader reader = IndexReader.open(directory, true);
+ assertEquals((1+iter)*182, reader.numDocs());
+ reader.close();
+
+ // Reopen
+ writer = new IndexWriter(
+ directory,
+ newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
+ setOpenMode(OpenMode.APPEND).
+ setMergePolicy(newLogMergePolicy(100))
+ );
+ }
+ writer.close();
+
+ directory.close();
+ }
+}
fnp / pylucene.git / blobdiff