X-Git-Url: https://git.mdrn.pl/pylucene.git/blobdiff_plain/a2e61f0c04805cfcb8706176758d1283c7e3a55c..aaeed5504b982cf3545252ab528713250aa33eed:/lucene-java-3.5.0/lucene/src/test/org/apache/lucene/index/TestIndexReaderOnDiskFull.java?ds=sidebyside diff --git a/lucene-java-3.5.0/lucene/src/test/org/apache/lucene/index/TestIndexReaderOnDiskFull.java b/lucene-java-3.5.0/lucene/src/test/org/apache/lucene/index/TestIndexReaderOnDiskFull.java new file mode 100644 index 0000000..d17457f --- /dev/null +++ b/lucene-java-3.5.0/lucene/src/test/org/apache/lucene/index/TestIndexReaderOnDiskFull.java @@ -0,0 +1,229 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.search.DefaultSimilarity; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.Similarity; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.store.MockDirectoryWrapper; +import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util._TestUtil; + +public class TestIndexReaderOnDiskFull extends LuceneTestCase { + /** + * Make sure if reader tries to commit but hits disk + * full that reader remains consistent and usable. + */ + public void testDiskFull() throws IOException { + + Term searchTerm = new Term("content", "aaa"); + int START_COUNT = 157; + int END_COUNT = 144; + + // First build up a starting index: + MockDirectoryWrapper startDir = newDirectory(); + IndexWriter writer = new IndexWriter(startDir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); + if (VERBOSE) { + System.out.println("TEST: create initial index"); + writer.setInfoStream(System.out); + } + for(int i=0;i<157;i++) { + Document d = new Document(); + d.add(newField("id", Integer.toString(i), Field.Store.YES, Field.Index.NOT_ANALYZED)); + d.add(newField("content", "aaa " + i, Field.Store.NO, Field.Index.ANALYZED)); + writer.addDocument(d); + if (0==i%10) + writer.commit(); + } + writer.close(); + + { + IndexReader r = IndexReader.open(startDir); + IndexSearcher searcher = newSearcher(r); + ScoreDoc[] hits = null; + try { + hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs; + } catch (IOException e) { + e.printStackTrace(); + fail("exception when init searching: " + e); + } + searcher.close(); + r.close(); + } + + long diskUsage = startDir.getRecomputedActualSizeInBytes(); + long diskFree = diskUsage+_TestUtil.nextInt(random, 50, 200); + + IOException err = null; + + boolean done = false; + boolean gotExc = false; + + // Iterate w/ ever increasing free disk space: + while(!done) { + MockDirectoryWrapper dir = new MockDirectoryWrapper(random, new RAMDirectory(startDir)); + + // If IndexReader hits disk full, it can write to + // the same files again. + dir.setPreventDoubleWrite(false); + + IndexReader reader = IndexReader.open(dir, false); + + // For each disk size, first try to commit against + // dir that will hit random IOExceptions & disk + // full; after, give it infinite disk space & turn + // off random IOExceptions & retry w/ same reader: + boolean success = false; + + for(int x=0;x<2;x++) { + + double rate = 0.05; + double diskRatio = ((double) diskFree)/diskUsage; + long thisDiskFree; + String testName; + + if (0 == x) { + thisDiskFree = diskFree; + if (diskRatio >= 2.0) { + rate /= 2; + } + if (diskRatio >= 4.0) { + rate /= 2; + } + if (diskRatio >= 6.0) { + rate = 0.0; + } + if (VERBOSE) { + System.out.println("\ncycle: " + diskFree + " bytes"); + } + testName = "disk full during reader.close() @ " + thisDiskFree + " bytes"; + } else { + thisDiskFree = 0; + rate = 0.0; + if (VERBOSE) { + System.out.println("\ncycle: same writer: unlimited disk space"); + } + testName = "reader re-use after disk full"; + } + + dir.setMaxSizeInBytes(thisDiskFree); + dir.setRandomIOExceptionRate(rate); + Similarity sim = new DefaultSimilarity(); + try { + if (0 == x) { + int docId = 12; + for(int i=0;i<13;i++) { + reader.deleteDocument(docId); + reader.setNorm(docId, "content", sim.encodeNormValue(2.0f)); + docId += 12; + } + } + reader.close(); + success = true; + if (0 == x) { + done = true; + } + } catch (IOException e) { + if (VERBOSE) { + System.out.println(" hit IOException: " + e); + e.printStackTrace(System.out); + } + err = e; + gotExc = true; + if (1 == x) { + e.printStackTrace(); + fail(testName + " hit IOException after disk space was freed up"); + } + } + + // Finally, verify index is not corrupt, and, if + // we succeeded, we see all docs changed, and if + // we failed, we see either all docs or no docs + // changed (transactional semantics): + IndexReader newReader = null; + try { + newReader = IndexReader.open(dir, false); + } catch (IOException e) { + e.printStackTrace(); + fail(testName + ":exception when creating IndexReader after disk full during close: " + e); + } + /* + int result = newReader.docFreq(searchTerm); + if (success) { + if (result != END_COUNT) { + fail(testName + ": method did not throw exception but docFreq('aaa') is " + result + " instead of expected " + END_COUNT); + } + } else { + // On hitting exception we still may have added + // all docs: + if (result != START_COUNT && result != END_COUNT) { + err.printStackTrace(); + fail(testName + ": method did throw exception but docFreq('aaa') is " + result + " instead of expected " + START_COUNT + " or " + END_COUNT); + } + } + */ + + IndexSearcher searcher = newSearcher(newReader); + ScoreDoc[] hits = null; + try { + hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs; + } catch (IOException e) { + e.printStackTrace(); + fail(testName + ": exception when searching: " + e); + } + int result2 = hits.length; + if (success) { + if (result2 != END_COUNT) { + fail(testName + ": method did not throw exception but hits.length for search on term 'aaa' is " + result2 + " instead of expected " + END_COUNT); + } + } else { + // On hitting exception we still may have added + // all docs: + if (result2 != START_COUNT && result2 != END_COUNT) { + err.printStackTrace(); + fail(testName + ": method did throw exception but hits.length for search on term 'aaa' is " + result2 + " instead of expected " + START_COUNT); + } + } + + searcher.close(); + newReader.close(); + + if (result2 == END_COUNT) { + if (!gotExc) + fail("never hit disk full"); + break; + } + } + + dir.close(); + + // Try again with more bytes of free space: + diskFree += TEST_NIGHTLY ? _TestUtil.nextInt(random, 5, 20) : _TestUtil.nextInt(random, 50, 200); + } + + startDir.close(); + } +}