+++ /dev/null
-package org.apache.lucene.store;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-
-import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.index.IndexWriterConfig.OpenMode;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.store.NIOFSDirectory.NIOFSIndexInput;
-import org.apache.lucene.store.SimpleFSDirectory.SimpleFSIndexInput;
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util._TestUtil;
-import org.apache.lucene.util.ArrayUtil;
-
-public class TestBufferedIndexInput extends LuceneTestCase {
-
- private static void writeBytes(File aFile, long size) throws IOException{
- OutputStream stream = null;
- try {
- stream = new FileOutputStream(aFile);
- for (int i = 0; i < size; i++) {
- stream.write(byten(i));
- }
- stream.flush();
- } finally {
- if (stream != null) {
- stream.close();
- }
- }
- }
-
- private static final long TEST_FILE_LENGTH = 100*1024;
-
- // Call readByte() repeatedly, past the buffer boundary, and see that it
- // is working as expected.
- // Our input comes from a dynamically generated/ "file" - see
- // MyBufferedIndexInput below.
- public void testReadByte() throws Exception {
- MyBufferedIndexInput input = new MyBufferedIndexInput();
- for (int i = 0; i < BufferedIndexInput.BUFFER_SIZE * 10; i++) {
- assertEquals(input.readByte(), byten(i));
- }
- }
-
- // Call readBytes() repeatedly, with various chunk sizes (from 1 byte to
- // larger than the buffer size), and see that it returns the bytes we expect.
- // Our input comes from a dynamically generated "file" -
- // see MyBufferedIndexInput below.
- public void testReadBytes() throws Exception {
- MyBufferedIndexInput input = new MyBufferedIndexInput();
- runReadBytes(input, BufferedIndexInput.BUFFER_SIZE, random);
-
- // This tests the workaround code for LUCENE-1566 where readBytesInternal
- // provides a workaround for a JVM Bug that incorrectly raises a OOM Error
- // when a large byte buffer is passed to a file read.
- // NOTE: this does only test the chunked reads and NOT if the Bug is triggered.
- //final int tmpFileSize = 1024 * 1024 * 5;
- final int inputBufferSize = 128;
- File tmpInputFile = _TestUtil.createTempFile("IndexInput", "tmpFile", TEMP_DIR);
- tmpInputFile.deleteOnExit();
- writeBytes(tmpInputFile, TEST_FILE_LENGTH);
-
- // run test with chunk size of 10 bytes
- runReadBytesAndClose(new SimpleFSIndexInput(tmpInputFile,
- inputBufferSize, 10), inputBufferSize, random);
-
- // run test with chunk size of 10 bytes
- runReadBytesAndClose(new NIOFSIndexInput(tmpInputFile,
- inputBufferSize, 10), inputBufferSize, random);
- }
-
- private void runReadBytesAndClose(IndexInput input, int bufferSize, Random r)
- throws IOException {
- try {
- runReadBytes(input, bufferSize, r);
- } finally {
- input.close();
- }
- }
-
- private void runReadBytes(IndexInput input, int bufferSize, Random r)
- throws IOException {
-
- int pos = 0;
- // gradually increasing size:
- for (int size = 1; size < bufferSize * 10; size = size + size / 200 + 1) {
- checkReadBytes(input, size, pos);
- pos += size;
- if (pos >= TEST_FILE_LENGTH) {
- // wrap
- pos = 0;
- input.seek(0L);
- }
- }
- // wildly fluctuating size:
- for (long i = 0; i < 100; i++) {
- final int size = r.nextInt(10000);
- checkReadBytes(input, 1+size, pos);
- pos += 1+size;
- if (pos >= TEST_FILE_LENGTH) {
- // wrap
- pos = 0;
- input.seek(0L);
- }
- }
- // constant small size (7 bytes):
- for (int i = 0; i < bufferSize; i++) {
- checkReadBytes(input, 7, pos);
- pos += 7;
- if (pos >= TEST_FILE_LENGTH) {
- // wrap
- pos = 0;
- input.seek(0L);
- }
- }
- }
-
- private byte[] buffer = new byte[10];
-
- private void checkReadBytes(IndexInput input, int size, int pos) throws IOException{
- // Just to see that "offset" is treated properly in readBytes(), we
- // add an arbitrary offset at the beginning of the array
- int offset = size % 10; // arbitrary
- buffer = ArrayUtil.grow(buffer, offset+size);
- assertEquals(pos, input.getFilePointer());
- long left = TEST_FILE_LENGTH - input.getFilePointer();
- if (left <= 0) {
- return;
- } else if (left < size) {
- size = (int) left;
- }
- input.readBytes(buffer, offset, size);
- assertEquals(pos+size, input.getFilePointer());
- for(int i=0; i<size; i++) {
- assertEquals("pos=" + i + " filepos=" + (pos+i), byten(pos+i), buffer[offset+i]);
- }
- }
-
- // This tests that attempts to readBytes() past an EOF will fail, while
- // reads up to the EOF will succeed. The EOF is determined by the
- // BufferedIndexInput's arbitrary length() value.
- public void testEOF() throws Exception {
- MyBufferedIndexInput input = new MyBufferedIndexInput(1024);
- // see that we can read all the bytes at one go:
- checkReadBytes(input, (int)input.length(), 0);
- // go back and see that we can't read more than that, for small and
- // large overflows:
- int pos = (int)input.length()-10;
- input.seek(pos);
- checkReadBytes(input, 10, pos);
- input.seek(pos);
- try {
- checkReadBytes(input, 11, pos);
- fail("Block read past end of file");
- } catch (IOException e) {
- /* success */
- }
- input.seek(pos);
- try {
- checkReadBytes(input, 50, pos);
- fail("Block read past end of file");
- } catch (IOException e) {
- /* success */
- }
- input.seek(pos);
- try {
- checkReadBytes(input, 100000, pos);
- fail("Block read past end of file");
- } catch (IOException e) {
- /* success */
- }
- }
-
- // byten emulates a file - byten(n) returns the n'th byte in that file.
- // MyBufferedIndexInput reads this "file".
- private static byte byten(long n){
- return (byte)(n*n%256);
- }
- private static class MyBufferedIndexInput extends BufferedIndexInput {
- private long pos;
- private long len;
- public MyBufferedIndexInput(long len){
- this.len = len;
- this.pos = 0;
- }
- public MyBufferedIndexInput(){
- // an infinite file
- this(Long.MAX_VALUE);
- }
- @Override
- protected void readInternal(byte[] b, int offset, int length) throws IOException {
- for(int i=offset; i<offset+length; i++)
- b[i] = byten(pos++);
- }
-
- @Override
- protected void seekInternal(long pos) throws IOException {
- this.pos = pos;
- }
-
- @Override
- public void close() throws IOException {
- }
-
- @Override
- public long length() {
- return len;
- }
- }
-
- public void testSetBufferSize() throws IOException {
- File indexDir = _TestUtil.getTempDir("testSetBufferSize");
- MockFSDirectory dir = new MockFSDirectory(indexDir, random);
- try {
- IndexWriter writer = new IndexWriter(
- dir,
- new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
- setOpenMode(OpenMode.CREATE).
- setMergePolicy(newLogMergePolicy(false))
- );
- for(int i=0;i<37;i++) {
- Document doc = new Document();
- doc.add(newField("content", "aaa bbb ccc ddd" + i, Field.Store.YES, Field.Index.ANALYZED));
- doc.add(newField("id", "" + i, Field.Store.YES, Field.Index.ANALYZED));
- writer.addDocument(doc);
- }
- writer.close();
-
- dir.allIndexInputs.clear();
-
- IndexReader reader = IndexReader.open(dir, false);
- Term aaa = new Term("content", "aaa");
- Term bbb = new Term("content", "bbb");
- Term ccc = new Term("content", "ccc");
- assertEquals(37, reader.docFreq(ccc));
- reader.deleteDocument(0);
- assertEquals(37, reader.docFreq(aaa));
- dir.tweakBufferSizes();
- reader.deleteDocument(4);
- assertEquals(reader.docFreq(bbb), 37);
- dir.tweakBufferSizes();
-
- IndexSearcher searcher = newSearcher(reader);
- ScoreDoc[] hits = searcher.search(new TermQuery(bbb), null, 1000).scoreDocs;
- dir.tweakBufferSizes();
- assertEquals(35, hits.length);
- dir.tweakBufferSizes();
- hits = searcher.search(new TermQuery(new Term("id", "33")), null, 1000).scoreDocs;
- dir.tweakBufferSizes();
- assertEquals(1, hits.length);
- hits = searcher.search(new TermQuery(aaa), null, 1000).scoreDocs;
- dir.tweakBufferSizes();
- assertEquals(35, hits.length);
- searcher.close();
- reader.close();
- } finally {
- _TestUtil.rmDir(indexDir);
- }
- }
-
- private static class MockFSDirectory extends Directory {
-
- List<IndexInput> allIndexInputs = new ArrayList<IndexInput>();
-
- Random rand;
-
- private Directory dir;
-
- public MockFSDirectory(File path, Random rand) throws IOException {
- this.rand = rand;
- lockFactory = NoLockFactory.getNoLockFactory();
- dir = new SimpleFSDirectory(path, null);
- }
-
- @Override
- public IndexInput openInput(String name) throws IOException {
- return openInput(name, BufferedIndexInput.BUFFER_SIZE);
- }
-
- public void tweakBufferSizes() {
- //int count = 0;
- for (final IndexInput ip : allIndexInputs) {
- BufferedIndexInput bii = (BufferedIndexInput) ip;
- int bufferSize = 1024+Math.abs(rand.nextInt() % 32768);
- bii.setBufferSize(bufferSize);
- //count++;
- }
- //System.out.println("tweak'd " + count + " buffer sizes");
- }
-
- @Override
- public IndexInput openInput(String name, int bufferSize) throws IOException {
- // Make random changes to buffer size
- bufferSize = 1+Math.abs(rand.nextInt() % 10);
- IndexInput f = dir.openInput(name, bufferSize);
- allIndexInputs.add(f);
- return f;
- }
-
- @Override
- public IndexOutput createOutput(String name) throws IOException {
- return dir.createOutput(name);
- }
-
- @Override
- public void close() throws IOException {
- dir.close();
- }
-
- @Override
- public void deleteFile(String name)
- throws IOException
- {
- dir.deleteFile(name);
- }
- @Override
- @Deprecated
- /* @deprecated Lucene never uses this API; it will be
- * removed in 4.0. */
- public void touchFile(String name)
- throws IOException
- {
- dir.touchFile(name);
- }
- @Override
- public long fileModified(String name)
- throws IOException
- {
- return dir.fileModified(name);
- }
- @Override
- public boolean fileExists(String name)
- throws IOException
- {
- return dir.fileExists(name);
- }
- @Override
- public String[] listAll()
- throws IOException
- {
- return dir.listAll();
- }
-
- @Override
- public long fileLength(String name) throws IOException {
- return dir.fileLength(name);
- }
-
-
- }
-}
fnp / pylucene.git / blobdiff