1 package org.apache.lucene.store;
4 * Licensed to the Apache Software Foundation (ASF) under one or more
5 * contributor license agreements. See the NOTICE file distributed with
6 * this work for additional information regarding copyright ownership.
7 * The ASF licenses this file to You under the Apache License, Version 2.0
8 * (the "License"); you may not use this file except in compliance with
9 * the License. You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
21 import java.io.FileOutputStream;
22 import java.io.IOException;
23 import java.io.OutputStream;
24 import java.util.ArrayList;
25 import java.util.List;
26 import java.util.Random;
28 import org.apache.lucene.analysis.MockAnalyzer;
29 import org.apache.lucene.document.Document;
30 import org.apache.lucene.document.Field;
31 import org.apache.lucene.index.IndexReader;
32 import org.apache.lucene.index.IndexWriter;
33 import org.apache.lucene.index.IndexWriterConfig;
34 import org.apache.lucene.index.Term;
35 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
36 import org.apache.lucene.search.IndexSearcher;
37 import org.apache.lucene.search.ScoreDoc;
38 import org.apache.lucene.search.TermQuery;
39 import org.apache.lucene.store.NIOFSDirectory.NIOFSIndexInput;
40 import org.apache.lucene.store.SimpleFSDirectory.SimpleFSIndexInput;
41 import org.apache.lucene.util.LuceneTestCase;
42 import org.apache.lucene.util._TestUtil;
43 import org.apache.lucene.util.ArrayUtil;
45 public class TestBufferedIndexInput extends LuceneTestCase {
47 private static void writeBytes(File aFile, long size) throws IOException{
48 OutputStream stream = null;
50 stream = new FileOutputStream(aFile);
51 for (int i = 0; i < size; i++) {
52 stream.write(byten(i));
62 private static final long TEST_FILE_LENGTH = 100*1024;
64 // Call readByte() repeatedly, past the buffer boundary, and see that it
65 // is working as expected.
66 // Our input comes from a dynamically generated/ "file" - see
67 // MyBufferedIndexInput below.
68 public void testReadByte() throws Exception {
69 MyBufferedIndexInput input = new MyBufferedIndexInput();
70 for (int i = 0; i < BufferedIndexInput.BUFFER_SIZE * 10; i++) {
71 assertEquals(input.readByte(), byten(i));
75 // Call readBytes() repeatedly, with various chunk sizes (from 1 byte to
76 // larger than the buffer size), and see that it returns the bytes we expect.
77 // Our input comes from a dynamically generated "file" -
78 // see MyBufferedIndexInput below.
79 public void testReadBytes() throws Exception {
80 MyBufferedIndexInput input = new MyBufferedIndexInput();
81 runReadBytes(input, BufferedIndexInput.BUFFER_SIZE, random);
83 // This tests the workaround code for LUCENE-1566 where readBytesInternal
84 // provides a workaround for a JVM Bug that incorrectly raises a OOM Error
85 // when a large byte buffer is passed to a file read.
86 // NOTE: this does only test the chunked reads and NOT if the Bug is triggered.
87 //final int tmpFileSize = 1024 * 1024 * 5;
88 final int inputBufferSize = 128;
89 File tmpInputFile = _TestUtil.createTempFile("IndexInput", "tmpFile", TEMP_DIR);
90 tmpInputFile.deleteOnExit();
91 writeBytes(tmpInputFile, TEST_FILE_LENGTH);
93 // run test with chunk size of 10 bytes
94 runReadBytesAndClose(new SimpleFSIndexInput(tmpInputFile,
95 inputBufferSize, 10), inputBufferSize, random);
97 // run test with chunk size of 10 bytes
98 runReadBytesAndClose(new NIOFSIndexInput(tmpInputFile,
99 inputBufferSize, 10), inputBufferSize, random);
102 private void runReadBytesAndClose(IndexInput input, int bufferSize, Random r)
105 runReadBytes(input, bufferSize, r);
111 private void runReadBytes(IndexInput input, int bufferSize, Random r)
115 // gradually increasing size:
116 for (int size = 1; size < bufferSize * 10; size = size + size / 200 + 1) {
117 checkReadBytes(input, size, pos);
119 if (pos >= TEST_FILE_LENGTH) {
125 // wildly fluctuating size:
126 for (long i = 0; i < 100; i++) {
127 final int size = r.nextInt(10000);
128 checkReadBytes(input, 1+size, pos);
130 if (pos >= TEST_FILE_LENGTH) {
136 // constant small size (7 bytes):
137 for (int i = 0; i < bufferSize; i++) {
138 checkReadBytes(input, 7, pos);
140 if (pos >= TEST_FILE_LENGTH) {
148 private byte[] buffer = new byte[10];
150 private void checkReadBytes(IndexInput input, int size, int pos) throws IOException{
151 // Just to see that "offset" is treated properly in readBytes(), we
152 // add an arbitrary offset at the beginning of the array
153 int offset = size % 10; // arbitrary
154 buffer = ArrayUtil.grow(buffer, offset+size);
155 assertEquals(pos, input.getFilePointer());
156 long left = TEST_FILE_LENGTH - input.getFilePointer();
159 } else if (left < size) {
162 input.readBytes(buffer, offset, size);
163 assertEquals(pos+size, input.getFilePointer());
164 for(int i=0; i<size; i++) {
165 assertEquals("pos=" + i + " filepos=" + (pos+i), byten(pos+i), buffer[offset+i]);
169 // This tests that attempts to readBytes() past an EOF will fail, while
170 // reads up to the EOF will succeed. The EOF is determined by the
171 // BufferedIndexInput's arbitrary length() value.
172 public void testEOF() throws Exception {
173 MyBufferedIndexInput input = new MyBufferedIndexInput(1024);
174 // see that we can read all the bytes at one go:
175 checkReadBytes(input, (int)input.length(), 0);
176 // go back and see that we can't read more than that, for small and
178 int pos = (int)input.length()-10;
180 checkReadBytes(input, 10, pos);
183 checkReadBytes(input, 11, pos);
184 fail("Block read past end of file");
185 } catch (IOException e) {
190 checkReadBytes(input, 50, pos);
191 fail("Block read past end of file");
192 } catch (IOException e) {
197 checkReadBytes(input, 100000, pos);
198 fail("Block read past end of file");
199 } catch (IOException e) {
204 // byten emulates a file - byten(n) returns the n'th byte in that file.
205 // MyBufferedIndexInput reads this "file".
206 private static byte byten(long n){
207 return (byte)(n*n%256);
209 private static class MyBufferedIndexInput extends BufferedIndexInput {
212 public MyBufferedIndexInput(long len){
216 public MyBufferedIndexInput(){
218 this(Long.MAX_VALUE);
221 protected void readInternal(byte[] b, int offset, int length) throws IOException {
222 for(int i=offset; i<offset+length; i++)
227 protected void seekInternal(long pos) throws IOException {
232 public void close() throws IOException {
236 public long length() {
241 public void testSetBufferSize() throws IOException {
242 File indexDir = _TestUtil.getTempDir("testSetBufferSize");
243 MockFSDirectory dir = new MockFSDirectory(indexDir, random);
245 IndexWriter writer = new IndexWriter(
247 new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
248 setOpenMode(OpenMode.CREATE).
249 setMergePolicy(newLogMergePolicy(false))
251 for(int i=0;i<37;i++) {
252 Document doc = new Document();
253 doc.add(newField("content", "aaa bbb ccc ddd" + i, Field.Store.YES, Field.Index.ANALYZED));
254 doc.add(newField("id", "" + i, Field.Store.YES, Field.Index.ANALYZED));
255 writer.addDocument(doc);
259 dir.allIndexInputs.clear();
261 IndexReader reader = IndexReader.open(dir, false);
262 Term aaa = new Term("content", "aaa");
263 Term bbb = new Term("content", "bbb");
264 Term ccc = new Term("content", "ccc");
265 assertEquals(37, reader.docFreq(ccc));
266 reader.deleteDocument(0);
267 assertEquals(37, reader.docFreq(aaa));
268 dir.tweakBufferSizes();
269 reader.deleteDocument(4);
270 assertEquals(reader.docFreq(bbb), 37);
271 dir.tweakBufferSizes();
273 IndexSearcher searcher = newSearcher(reader);
274 ScoreDoc[] hits = searcher.search(new TermQuery(bbb), null, 1000).scoreDocs;
275 dir.tweakBufferSizes();
276 assertEquals(35, hits.length);
277 dir.tweakBufferSizes();
278 hits = searcher.search(new TermQuery(new Term("id", "33")), null, 1000).scoreDocs;
279 dir.tweakBufferSizes();
280 assertEquals(1, hits.length);
281 hits = searcher.search(new TermQuery(aaa), null, 1000).scoreDocs;
282 dir.tweakBufferSizes();
283 assertEquals(35, hits.length);
287 _TestUtil.rmDir(indexDir);
291 private static class MockFSDirectory extends Directory {
293 List<IndexInput> allIndexInputs = new ArrayList<IndexInput>();
297 private Directory dir;
299 public MockFSDirectory(File path, Random rand) throws IOException {
301 lockFactory = NoLockFactory.getNoLockFactory();
302 dir = new SimpleFSDirectory(path, null);
306 public IndexInput openInput(String name) throws IOException {
307 return openInput(name, BufferedIndexInput.BUFFER_SIZE);
310 public void tweakBufferSizes() {
312 for (final IndexInput ip : allIndexInputs) {
313 BufferedIndexInput bii = (BufferedIndexInput) ip;
314 int bufferSize = 1024+Math.abs(rand.nextInt() % 32768);
315 bii.setBufferSize(bufferSize);
318 //System.out.println("tweak'd " + count + " buffer sizes");
322 public IndexInput openInput(String name, int bufferSize) throws IOException {
323 // Make random changes to buffer size
324 bufferSize = 1+Math.abs(rand.nextInt() % 10);
325 IndexInput f = dir.openInput(name, bufferSize);
326 allIndexInputs.add(f);
331 public IndexOutput createOutput(String name) throws IOException {
332 return dir.createOutput(name);
336 public void close() throws IOException {
341 public void deleteFile(String name)
344 dir.deleteFile(name);
348 /* @deprecated Lucene never uses this API; it will be
350 public void touchFile(String name)
356 public long fileModified(String name)
359 return dir.fileModified(name);
362 public boolean fileExists(String name)
365 return dir.fileExists(name);
368 public String[] listAll()
371 return dir.listAll();
375 public long fileLength(String name) throws IOException {
376 return dir.fileLength(name);