X-Git-Url: https://git.mdrn.pl/pylucene.git/blobdiff_plain/a2e61f0c04805cfcb8706176758d1283c7e3a55c..aaeed5504b982cf3545252ab528713250aa33eed:/lucene-java-3.5.0/lucene/src/test/org/apache/lucene/store/TestBufferedIndexInput.java?ds=sidebyside

diff --git a/lucene-java-3.5.0/lucene/src/test/org/apache/lucene/store/TestBufferedIndexInput.java b/lucene-java-3.5.0/lucene/src/test/org/apache/lucene/store/TestBufferedIndexInput.java
new file mode 100755
index 0000000..4c2469a
--- /dev/null
+++ b/lucene-java-3.5.0/lucene/src/test/org/apache/lucene/store/TestBufferedIndexInput.java
@@ -0,0 +1,382 @@
+package org.apache.lucene.store;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.IndexWriterConfig.OpenMode;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.store.NIOFSDirectory.NIOFSIndexInput;
+import org.apache.lucene.store.SimpleFSDirectory.SimpleFSIndexInput;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
+import org.apache.lucene.util.ArrayUtil;
+
+public class TestBufferedIndexInput extends LuceneTestCase {
+  
+  private static void writeBytes(File aFile, long size) throws IOException{
+    OutputStream stream = null;
+    try {
+      stream = new FileOutputStream(aFile);
+      for (int i = 0; i < size; i++) {
+        stream.write(byten(i));  
+      }
+      stream.flush();
+    } finally {
+      if (stream != null) {
+        stream.close();
+      }
+    }
+  }
+
+  private static final long TEST_FILE_LENGTH = 100*1024;
+ 
+  // Call readByte() repeatedly, past the buffer boundary, and see that it
+  // is working as expected.
+  // Our input comes from a dynamically generated/ "file" - see
+  // MyBufferedIndexInput below.
+  public void testReadByte() throws Exception {
+    MyBufferedIndexInput input = new MyBufferedIndexInput();
+    for (int i = 0; i < BufferedIndexInput.BUFFER_SIZE * 10; i++) {
+      assertEquals(input.readByte(), byten(i));
+    }
+  }
+ 
+  // Call readBytes() repeatedly, with various chunk sizes (from 1 byte to
+  // larger than the buffer size), and see that it returns the bytes we expect.
+  // Our input comes from a dynamically generated "file" -
+  // see MyBufferedIndexInput below.
+  public void testReadBytes() throws Exception {
+    MyBufferedIndexInput input = new MyBufferedIndexInput();
+    runReadBytes(input, BufferedIndexInput.BUFFER_SIZE, random);
+
+    // This tests the workaround code for LUCENE-1566 where readBytesInternal
+    // provides a workaround for a JVM Bug that incorrectly raises a OOM Error
+    // when a large byte buffer is passed to a file read.
+    // NOTE: this does only test the chunked reads and NOT if the Bug is triggered.
+    //final int tmpFileSize = 1024 * 1024 * 5;
+    final int inputBufferSize = 128;
+    File tmpInputFile = _TestUtil.createTempFile("IndexInput", "tmpFile", TEMP_DIR);
+    tmpInputFile.deleteOnExit();
+    writeBytes(tmpInputFile, TEST_FILE_LENGTH);
+
+    // run test with chunk size of 10 bytes
+    runReadBytesAndClose(new SimpleFSIndexInput("SimpleFSIndexInput(path=\"" + tmpInputFile + "\")", tmpInputFile,
+                                                inputBufferSize, 10), inputBufferSize, random);
+
+    // run test with chunk size of 10 bytes
+    runReadBytesAndClose(new NIOFSIndexInput(tmpInputFile,
+                                             inputBufferSize, 10), inputBufferSize, random);
+  }
+
+  private void runReadBytesAndClose(IndexInput input, int bufferSize, Random r)
+      throws IOException {
+    try {
+      runReadBytes(input, bufferSize, r);
+    } finally {
+      input.close();
+    }
+  }
+  
+  private void runReadBytes(IndexInput input, int bufferSize, Random r)
+      throws IOException {
+
+    int pos = 0;
+    // gradually increasing size:
+    for (int size = 1; size < bufferSize * 10; size = size + size / 200 + 1) {
+      checkReadBytes(input, size, pos);
+      pos += size;
+      if (pos >= TEST_FILE_LENGTH) {
+        // wrap
+        pos = 0;
+        input.seek(0L);
+      }
+    }
+    // wildly fluctuating size:
+    for (long i = 0; i < 100; i++) {
+      final int size = r.nextInt(10000);
+      checkReadBytes(input, 1+size, pos);
+      pos += 1+size;
+      if (pos >= TEST_FILE_LENGTH) {
+        // wrap
+        pos = 0;
+        input.seek(0L);
+      }
+    }
+    // constant small size (7 bytes):
+    for (int i = 0; i < bufferSize; i++) {
+      checkReadBytes(input, 7, pos);
+      pos += 7;
+      if (pos >= TEST_FILE_LENGTH) {
+        // wrap
+        pos = 0;
+        input.seek(0L);
+      }
+    }
+  }
+
+  private byte[] buffer = new byte[10];
+    
+  private void checkReadBytes(IndexInput input, int size, int pos) throws IOException{
+    // Just to see that "offset" is treated properly in readBytes(), we
+    // add an arbitrary offset at the beginning of the array
+    int offset = size % 10; // arbitrary
+    buffer = ArrayUtil.grow(buffer, offset+size);
+    assertEquals(pos, input.getFilePointer());
+    long left = TEST_FILE_LENGTH - input.getFilePointer();
+    if (left <= 0) {
+      return;
+    } else if (left < size) {
+      size = (int) left;
+    }
+    input.readBytes(buffer, offset, size);
+    assertEquals(pos+size, input.getFilePointer());
+    for(int i=0; i<size; i++) {
+      assertEquals("pos=" + i + " filepos=" + (pos+i), byten(pos+i), buffer[offset+i]);
+    }
+  }
+   
+  // This tests that attempts to readBytes() past an EOF will fail, while
+  // reads up to the EOF will succeed. The EOF is determined by the
+  // BufferedIndexInput's arbitrary length() value.
+  public void testEOF() throws Exception {
+     MyBufferedIndexInput input = new MyBufferedIndexInput(1024);
+     // see that we can read all the bytes at one go:
+     checkReadBytes(input, (int)input.length(), 0);  
+     // go back and see that we can't read more than that, for small and
+     // large overflows:
+     int pos = (int)input.length()-10;
+     input.seek(pos);
+     checkReadBytes(input, 10, pos);  
+     input.seek(pos);
+     try {
+       checkReadBytes(input, 11, pos);
+           fail("Block read past end of file");
+       } catch (IOException e) {
+           /* success */
+       }
+     input.seek(pos);
+     try {
+       checkReadBytes(input, 50, pos);
+           fail("Block read past end of file");
+       } catch (IOException e) {
+           /* success */
+       }
+     input.seek(pos);
+     try {
+       checkReadBytes(input, 100000, pos);
+           fail("Block read past end of file");
+       } catch (IOException e) {
+           /* success */
+       }
+  }
+
+    // byten emulates a file - byten(n) returns the n'th byte in that file.
+    // MyBufferedIndexInput reads this "file".
+    private static byte byten(long n){
+      return (byte)(n*n%256);
+    }
+    private static class MyBufferedIndexInput extends BufferedIndexInput {
+      private long pos;
+      private long len;
+      public MyBufferedIndexInput(long len){
+        super("MyBufferedIndexInput(len=" + len + ")", BufferedIndexInput.BUFFER_SIZE);
+        this.len = len;
+        this.pos = 0;
+      }
+      public MyBufferedIndexInput(){
+        // an infinite file
+        this(Long.MAX_VALUE);
+      }
+      @Override
+      protected void readInternal(byte[] b, int offset, int length) throws IOException {
+        for(int i=offset; i<offset+length; i++)
+          b[i] = byten(pos++);
+      }
+
+      @Override
+      protected void seekInternal(long pos) throws IOException {
+        this.pos = pos;
+      }
+
+      @Override
+      public void close() throws IOException {
+      }
+
+      @Override
+      public long length() {
+        return len;
+      }
+    }
+
+    public void testSetBufferSize() throws IOException {
+      File indexDir = _TestUtil.getTempDir("testSetBufferSize");
+      MockFSDirectory dir = new MockFSDirectory(indexDir, random);
+      try {
+        IndexWriter writer = new IndexWriter(
+            dir,
+            new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
+                setOpenMode(OpenMode.CREATE).
+                setMergePolicy(newLogMergePolicy(false))
+        );
+        for(int i=0;i<37;i++) {
+          Document doc = new Document();
+          doc.add(newField("content", "aaa bbb ccc ddd" + i, Field.Store.YES, Field.Index.ANALYZED));
+          doc.add(newField("id", "" + i, Field.Store.YES, Field.Index.ANALYZED));
+          writer.addDocument(doc);
+        }
+        writer.close();
+
+        dir.allIndexInputs.clear();
+
+        IndexReader reader = IndexReader.open(dir, false);
+        Term aaa = new Term("content", "aaa");
+        Term bbb = new Term("content", "bbb");
+        Term ccc = new Term("content", "ccc");
+        assertEquals(37, reader.docFreq(ccc));
+        reader.deleteDocument(0);
+        assertEquals(37, reader.docFreq(aaa));
+        dir.tweakBufferSizes();
+        reader.deleteDocument(4);
+        assertEquals(reader.docFreq(bbb), 37);
+        dir.tweakBufferSizes();
+
+        IndexSearcher searcher = newSearcher(reader);
+        ScoreDoc[] hits = searcher.search(new TermQuery(bbb), null, 1000).scoreDocs;
+        dir.tweakBufferSizes();
+        assertEquals(35, hits.length);
+        dir.tweakBufferSizes();
+        hits = searcher.search(new TermQuery(new Term("id", "33")), null, 1000).scoreDocs;
+        dir.tweakBufferSizes();
+        assertEquals(1, hits.length);
+        hits = searcher.search(new TermQuery(aaa), null, 1000).scoreDocs;
+        dir.tweakBufferSizes();
+        assertEquals(35, hits.length);
+        searcher.close();
+        reader.close();
+      } finally {
+        _TestUtil.rmDir(indexDir);
+      }
+    }
+
+    private static class MockFSDirectory extends Directory {
+
+      List<IndexInput> allIndexInputs = new ArrayList<IndexInput>();
+
+      Random rand;
+
+      private Directory dir;
+
+      public MockFSDirectory(File path, Random rand) throws IOException {
+        this.rand = rand;
+        lockFactory = NoLockFactory.getNoLockFactory();
+        dir = new SimpleFSDirectory(path, null);
+      }
+
+      @Override
+      public IndexInput openInput(String name) throws IOException {
+        return openInput(name, BufferedIndexInput.BUFFER_SIZE);
+      }
+
+      public void tweakBufferSizes() {
+        //int count = 0;
+        for (final IndexInput ip : allIndexInputs) {
+          BufferedIndexInput bii = (BufferedIndexInput) ip;
+          int bufferSize = 1024+Math.abs(rand.nextInt() % 32768);
+          bii.setBufferSize(bufferSize);
+          //count++;
+        }
+        //System.out.println("tweak'd " + count + " buffer sizes");
+      }
+      
+      @Override
+      public IndexInput openInput(String name, int bufferSize) throws IOException {
+        // Make random changes to buffer size
+        bufferSize = 1+Math.abs(rand.nextInt() % 10);
+        IndexInput f = dir.openInput(name, bufferSize);
+        allIndexInputs.add(f);
+        return f;
+      }
+
+      @Override
+      public IndexOutput createOutput(String name) throws IOException {
+        return dir.createOutput(name);
+      }
+
+      @Override
+      public void close() throws IOException {
+        dir.close();
+      }
+
+      @Override
+      public void deleteFile(String name)
+        throws IOException
+      {
+        dir.deleteFile(name);
+      }
+      @Override
+      @Deprecated
+      /*  @deprecated Lucene never uses this API; it will be
+       *  removed in 4.0. */
+      public void touchFile(String name)
+        throws IOException
+      {
+        dir.touchFile(name);
+      }
+      @Override
+      public long fileModified(String name)
+        throws IOException
+      {
+        return dir.fileModified(name);
+      }
+      @Override
+      public boolean fileExists(String name)
+        throws IOException
+      {
+        return dir.fileExists(name);
+      }
+      @Override
+      public String[] listAll()
+        throws IOException
+      {
+        return dir.listAll();
+      }
+
+      @Override
+      public long fileLength(String name) throws IOException {
+        return dir.fileLength(name);
+      }
+
+
+    }
+}