--- /dev/null
+package org.apache.lucene.store;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.IndexWriterConfig.OpenMode;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.store.NIOFSDirectory.NIOFSIndexInput;
+import org.apache.lucene.store.SimpleFSDirectory.SimpleFSIndexInput;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
+import org.apache.lucene.util.ArrayUtil;
+
+public class TestBufferedIndexInput extends LuceneTestCase {
+
+ private static void writeBytes(File aFile, long size) throws IOException{
+ OutputStream stream = null;
+ try {
+ stream = new FileOutputStream(aFile);
+ for (int i = 0; i < size; i++) {
+ stream.write(byten(i));
+ }
+ stream.flush();
+ } finally {
+ if (stream != null) {
+ stream.close();
+ }
+ }
+ }
+
+ private static final long TEST_FILE_LENGTH = 100*1024;
+
+ // Call readByte() repeatedly, past the buffer boundary, and see that it
+ // is working as expected.
+ // Our input comes from a dynamically generated/ "file" - see
+ // MyBufferedIndexInput below.
+ public void testReadByte() throws Exception {
+ MyBufferedIndexInput input = new MyBufferedIndexInput();
+ for (int i = 0; i < BufferedIndexInput.BUFFER_SIZE * 10; i++) {
+ assertEquals(input.readByte(), byten(i));
+ }
+ }
+
+ // Call readBytes() repeatedly, with various chunk sizes (from 1 byte to
+ // larger than the buffer size), and see that it returns the bytes we expect.
+ // Our input comes from a dynamically generated "file" -
+ // see MyBufferedIndexInput below.
+ public void testReadBytes() throws Exception {
+ MyBufferedIndexInput input = new MyBufferedIndexInput();
+ runReadBytes(input, BufferedIndexInput.BUFFER_SIZE, random);
+
+ // This tests the workaround code for LUCENE-1566 where readBytesInternal
+ // provides a workaround for a JVM Bug that incorrectly raises a OOM Error
+ // when a large byte buffer is passed to a file read.
+ // NOTE: this does only test the chunked reads and NOT if the Bug is triggered.
+ //final int tmpFileSize = 1024 * 1024 * 5;
+ final int inputBufferSize = 128;
+ File tmpInputFile = _TestUtil.createTempFile("IndexInput", "tmpFile", TEMP_DIR);
+ tmpInputFile.deleteOnExit();
+ writeBytes(tmpInputFile, TEST_FILE_LENGTH);
+
+ // run test with chunk size of 10 bytes
+ runReadBytesAndClose(new SimpleFSIndexInput(tmpInputFile,
+ inputBufferSize, 10), inputBufferSize, random);
+
+ // run test with chunk size of 10 bytes
+ runReadBytesAndClose(new NIOFSIndexInput(tmpInputFile,
+ inputBufferSize, 10), inputBufferSize, random);
+ }
+
+ private void runReadBytesAndClose(IndexInput input, int bufferSize, Random r)
+ throws IOException {
+ try {
+ runReadBytes(input, bufferSize, r);
+ } finally {
+ input.close();
+ }
+ }
+
+ private void runReadBytes(IndexInput input, int bufferSize, Random r)
+ throws IOException {
+
+ int pos = 0;
+ // gradually increasing size:
+ for (int size = 1; size < bufferSize * 10; size = size + size / 200 + 1) {
+ checkReadBytes(input, size, pos);
+ pos += size;
+ if (pos >= TEST_FILE_LENGTH) {
+ // wrap
+ pos = 0;
+ input.seek(0L);
+ }
+ }
+ // wildly fluctuating size:
+ for (long i = 0; i < 100; i++) {
+ final int size = r.nextInt(10000);
+ checkReadBytes(input, 1+size, pos);
+ pos += 1+size;
+ if (pos >= TEST_FILE_LENGTH) {
+ // wrap
+ pos = 0;
+ input.seek(0L);
+ }
+ }
+ // constant small size (7 bytes):
+ for (int i = 0; i < bufferSize; i++) {
+ checkReadBytes(input, 7, pos);
+ pos += 7;
+ if (pos >= TEST_FILE_LENGTH) {
+ // wrap
+ pos = 0;
+ input.seek(0L);
+ }
+ }
+ }
+
+ private byte[] buffer = new byte[10];
+
+ private void checkReadBytes(IndexInput input, int size, int pos) throws IOException{
+ // Just to see that "offset" is treated properly in readBytes(), we
+ // add an arbitrary offset at the beginning of the array
+ int offset = size % 10; // arbitrary
+ buffer = ArrayUtil.grow(buffer, offset+size);
+ assertEquals(pos, input.getFilePointer());
+ long left = TEST_FILE_LENGTH - input.getFilePointer();
+ if (left <= 0) {
+ return;
+ } else if (left < size) {
+ size = (int) left;
+ }
+ input.readBytes(buffer, offset, size);
+ assertEquals(pos+size, input.getFilePointer());
+ for(int i=0; i<size; i++) {
+ assertEquals("pos=" + i + " filepos=" + (pos+i), byten(pos+i), buffer[offset+i]);
+ }
+ }
+
+ // This tests that attempts to readBytes() past an EOF will fail, while
+ // reads up to the EOF will succeed. The EOF is determined by the
+ // BufferedIndexInput's arbitrary length() value.
+ public void testEOF() throws Exception {
+ MyBufferedIndexInput input = new MyBufferedIndexInput(1024);
+ // see that we can read all the bytes at one go:
+ checkReadBytes(input, (int)input.length(), 0);
+ // go back and see that we can't read more than that, for small and
+ // large overflows:
+ int pos = (int)input.length()-10;
+ input.seek(pos);
+ checkReadBytes(input, 10, pos);
+ input.seek(pos);
+ try {
+ checkReadBytes(input, 11, pos);
+ fail("Block read past end of file");
+ } catch (IOException e) {
+ /* success */
+ }
+ input.seek(pos);
+ try {
+ checkReadBytes(input, 50, pos);
+ fail("Block read past end of file");
+ } catch (IOException e) {
+ /* success */
+ }
+ input.seek(pos);
+ try {
+ checkReadBytes(input, 100000, pos);
+ fail("Block read past end of file");
+ } catch (IOException e) {
+ /* success */
+ }
+ }
+
+ // byten emulates a file - byten(n) returns the n'th byte in that file.
+ // MyBufferedIndexInput reads this "file".
+ private static byte byten(long n){
+ return (byte)(n*n%256);
+ }
+ private static class MyBufferedIndexInput extends BufferedIndexInput {
+ private long pos;
+ private long len;
+ public MyBufferedIndexInput(long len){
+ this.len = len;
+ this.pos = 0;
+ }
+ public MyBufferedIndexInput(){
+ // an infinite file
+ this(Long.MAX_VALUE);
+ }
+ @Override
+ protected void readInternal(byte[] b, int offset, int length) throws IOException {
+ for(int i=offset; i<offset+length; i++)
+ b[i] = byten(pos++);
+ }
+
+ @Override
+ protected void seekInternal(long pos) throws IOException {
+ this.pos = pos;
+ }
+
+ @Override
+ public void close() throws IOException {
+ }
+
+ @Override
+ public long length() {
+ return len;
+ }
+ }
+
+ public void testSetBufferSize() throws IOException {
+ File indexDir = _TestUtil.getTempDir("testSetBufferSize");
+ MockFSDirectory dir = new MockFSDirectory(indexDir, random);
+ try {
+ IndexWriter writer = new IndexWriter(
+ dir,
+ new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
+ setOpenMode(OpenMode.CREATE).
+ setMergePolicy(newLogMergePolicy(false))
+ );
+ for(int i=0;i<37;i++) {
+ Document doc = new Document();
+ doc.add(newField("content", "aaa bbb ccc ddd" + i, Field.Store.YES, Field.Index.ANALYZED));
+ doc.add(newField("id", "" + i, Field.Store.YES, Field.Index.ANALYZED));
+ writer.addDocument(doc);
+ }
+ writer.close();
+
+ dir.allIndexInputs.clear();
+
+ IndexReader reader = IndexReader.open(dir, false);
+ Term aaa = new Term("content", "aaa");
+ Term bbb = new Term("content", "bbb");
+ Term ccc = new Term("content", "ccc");
+ assertEquals(37, reader.docFreq(ccc));
+ reader.deleteDocument(0);
+ assertEquals(37, reader.docFreq(aaa));
+ dir.tweakBufferSizes();
+ reader.deleteDocument(4);
+ assertEquals(reader.docFreq(bbb), 37);
+ dir.tweakBufferSizes();
+
+ IndexSearcher searcher = newSearcher(reader);
+ ScoreDoc[] hits = searcher.search(new TermQuery(bbb), null, 1000).scoreDocs;
+ dir.tweakBufferSizes();
+ assertEquals(35, hits.length);
+ dir.tweakBufferSizes();
+ hits = searcher.search(new TermQuery(new Term("id", "33")), null, 1000).scoreDocs;
+ dir.tweakBufferSizes();
+ assertEquals(1, hits.length);
+ hits = searcher.search(new TermQuery(aaa), null, 1000).scoreDocs;
+ dir.tweakBufferSizes();
+ assertEquals(35, hits.length);
+ searcher.close();
+ reader.close();
+ } finally {
+ _TestUtil.rmDir(indexDir);
+ }
+ }
+
+ private static class MockFSDirectory extends Directory {
+
+ List<IndexInput> allIndexInputs = new ArrayList<IndexInput>();
+
+ Random rand;
+
+ private Directory dir;
+
+ public MockFSDirectory(File path, Random rand) throws IOException {
+ this.rand = rand;
+ lockFactory = NoLockFactory.getNoLockFactory();
+ dir = new SimpleFSDirectory(path, null);
+ }
+
+ @Override
+ public IndexInput openInput(String name) throws IOException {
+ return openInput(name, BufferedIndexInput.BUFFER_SIZE);
+ }
+
+ public void tweakBufferSizes() {
+ //int count = 0;
+ for (final IndexInput ip : allIndexInputs) {
+ BufferedIndexInput bii = (BufferedIndexInput) ip;
+ int bufferSize = 1024+Math.abs(rand.nextInt() % 32768);
+ bii.setBufferSize(bufferSize);
+ //count++;
+ }
+ //System.out.println("tweak'd " + count + " buffer sizes");
+ }
+
+ @Override
+ public IndexInput openInput(String name, int bufferSize) throws IOException {
+ // Make random changes to buffer size
+ bufferSize = 1+Math.abs(rand.nextInt() % 10);
+ IndexInput f = dir.openInput(name, bufferSize);
+ allIndexInputs.add(f);
+ return f;
+ }
+
+ @Override
+ public IndexOutput createOutput(String name) throws IOException {
+ return dir.createOutput(name);
+ }
+
+ @Override
+ public void close() throws IOException {
+ dir.close();
+ }
+
+ @Override
+ public void deleteFile(String name)
+ throws IOException
+ {
+ dir.deleteFile(name);
+ }
+ @Override
+ @Deprecated
+ /* @deprecated Lucene never uses this API; it will be
+ * removed in 4.0. */
+ public void touchFile(String name)
+ throws IOException
+ {
+ dir.touchFile(name);
+ }
+ @Override
+ public long fileModified(String name)
+ throws IOException
+ {
+ return dir.fileModified(name);
+ }
+ @Override
+ public boolean fileExists(String name)
+ throws IOException
+ {
+ return dir.fileExists(name);
+ }
+ @Override
+ public String[] listAll()
+ throws IOException
+ {
+ return dir.listAll();
+ }
+
+ @Override
+ public long fileLength(String name) throws IOException {
+ return dir.fileLength(name);
+ }
+
+
+ }
+}
fnp / pylucene.git / blobdiff