--- /dev/null
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.io.File;
+
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.MockDirectoryWrapper;
+import org.apache.lucene.store.MockDirectoryWrapper.Failure;
+import org.apache.lucene.store.SimpleFSDirectory;
+import org.apache.lucene.store._TestHelper;
+import org.apache.lucene.util._TestUtil;
+
+
+public class TestCompoundFile extends LuceneTestCase
+{
+ private Directory dir;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ File file = _TestUtil.getTempDir("testIndex");
+ // use a simple FSDir here, to be sure to have SimpleFSInputs
+ dir = new SimpleFSDirectory(file,null);
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ dir.close();
+ super.tearDown();
+ }
+
+ /** Creates a file of the specified size with random data. */
+ private void createRandomFile(Directory dir, String name, int size)
+ throws IOException
+ {
+ IndexOutput os = dir.createOutput(name);
+ for (int i=0; i<size; i++) {
+ byte b = (byte) (Math.random() * 256);
+ os.writeByte(b);
+ }
+ os.close();
+ }
+
+ /** Creates a file of the specified size with sequential data. The first
+ * byte is written as the start byte provided. All subsequent bytes are
+ * computed as start + offset where offset is the number of the byte.
+ */
+ private void createSequenceFile(Directory dir,
+ String name,
+ byte start,
+ int size)
+ throws IOException
+ {
+ IndexOutput os = dir.createOutput(name);
+ for (int i=0; i < size; i++) {
+ os.writeByte(start);
+ start ++;
+ }
+ os.close();
+ }
+
+
+ private void assertSameStreams(String msg,
+ IndexInput expected,
+ IndexInput test)
+ throws IOException
+ {
+ assertNotNull(msg + " null expected", expected);
+ assertNotNull(msg + " null test", test);
+ assertEquals(msg + " length", expected.length(), test.length());
+ assertEquals(msg + " position", expected.getFilePointer(),
+ test.getFilePointer());
+
+ byte expectedBuffer[] = new byte[512];
+ byte testBuffer[] = new byte[expectedBuffer.length];
+
+ long remainder = expected.length() - expected.getFilePointer();
+ while(remainder > 0) {
+ int readLen = (int) Math.min(remainder, expectedBuffer.length);
+ expected.readBytes(expectedBuffer, 0, readLen);
+ test.readBytes(testBuffer, 0, readLen);
+ assertEqualArrays(msg + ", remainder " + remainder, expectedBuffer,
+ testBuffer, 0, readLen);
+ remainder -= readLen;
+ }
+ }
+
+
+ private void assertSameStreams(String msg,
+ IndexInput expected,
+ IndexInput actual,
+ long seekTo)
+ throws IOException
+ {
+ if(seekTo >= 0 && seekTo < expected.length())
+ {
+ expected.seek(seekTo);
+ actual.seek(seekTo);
+ assertSameStreams(msg + ", seek(mid)", expected, actual);
+ }
+ }
+
+
+
+ private void assertSameSeekBehavior(String msg,
+ IndexInput expected,
+ IndexInput actual)
+ throws IOException
+ {
+ // seek to 0
+ long point = 0;
+ assertSameStreams(msg + ", seek(0)", expected, actual, point);
+
+ // seek to middle
+ point = expected.length() / 2l;
+ assertSameStreams(msg + ", seek(mid)", expected, actual, point);
+
+ // seek to end - 2
+ point = expected.length() - 2;
+ assertSameStreams(msg + ", seek(end-2)", expected, actual, point);
+
+ // seek to end - 1
+ point = expected.length() - 1;
+ assertSameStreams(msg + ", seek(end-1)", expected, actual, point);
+
+ // seek to the end
+ point = expected.length();
+ assertSameStreams(msg + ", seek(end)", expected, actual, point);
+
+ // seek past end
+ point = expected.length() + 1;
+ assertSameStreams(msg + ", seek(end+1)", expected, actual, point);
+ }
+
+
+ private void assertEqualArrays(String msg,
+ byte[] expected,
+ byte[] test,
+ int start,
+ int len)
+ {
+ assertNotNull(msg + " null expected", expected);
+ assertNotNull(msg + " null test", test);
+
+ for (int i=start; i<len; i++) {
+ assertEquals(msg + " " + i, expected[i], test[i]);
+ }
+ }
+
+
+ // ===========================================================
+ // Tests of the basic CompoundFile functionality
+ // ===========================================================
+
+
+ /** This test creates compound file based on a single file.
+ * Files of different sizes are tested: 0, 1, 10, 100 bytes.
+ */
+ public void testSingleFile() throws IOException {
+ int data[] = new int[] { 0, 1, 10, 100 };
+ for (int i=0; i<data.length; i++) {
+ String name = "t" + data[i];
+ createSequenceFile(dir, name, (byte) 0, data[i]);
+ CompoundFileWriter csw = new CompoundFileWriter(dir, name + ".cfs");
+ csw.addFile(name);
+ csw.close();
+
+ CompoundFileReader csr = new CompoundFileReader(dir, name + ".cfs");
+ IndexInput expected = dir.openInput(name);
+ IndexInput actual = csr.openInput(name);
+ assertSameStreams(name, expected, actual);
+ assertSameSeekBehavior(name, expected, actual);
+ expected.close();
+ actual.close();
+ csr.close();
+ }
+ }
+
+
+ /** This test creates compound file based on two files.
+ *
+ */
+ public void testTwoFiles() throws IOException {
+ createSequenceFile(dir, "d1", (byte) 0, 15);
+ createSequenceFile(dir, "d2", (byte) 0, 114);
+
+ CompoundFileWriter csw = new CompoundFileWriter(dir, "d.csf");
+ csw.addFile("d1");
+ csw.addFile("d2");
+ csw.close();
+
+ CompoundFileReader csr = new CompoundFileReader(dir, "d.csf");
+ IndexInput expected = dir.openInput("d1");
+ IndexInput actual = csr.openInput("d1");
+ assertSameStreams("d1", expected, actual);
+ assertSameSeekBehavior("d1", expected, actual);
+ expected.close();
+ actual.close();
+
+ expected = dir.openInput("d2");
+ actual = csr.openInput("d2");
+ assertSameStreams("d2", expected, actual);
+ assertSameSeekBehavior("d2", expected, actual);
+ expected.close();
+ actual.close();
+ csr.close();
+ }
+
+ /** This test creates a compound file based on a large number of files of
+ * various length. The file content is generated randomly. The sizes range
+ * from 0 to 1Mb. Some of the sizes are selected to test the buffering
+ * logic in the file reading code. For this the chunk variable is set to
+ * the length of the buffer used internally by the compound file logic.
+ */
+ public void testRandomFiles() throws IOException {
+ // Setup the test segment
+ String segment = "test";
+ int chunk = 1024; // internal buffer size used by the stream
+ createRandomFile(dir, segment + ".zero", 0);
+ createRandomFile(dir, segment + ".one", 1);
+ createRandomFile(dir, segment + ".ten", 10);
+ createRandomFile(dir, segment + ".hundred", 100);
+ createRandomFile(dir, segment + ".big1", chunk);
+ createRandomFile(dir, segment + ".big2", chunk - 1);
+ createRandomFile(dir, segment + ".big3", chunk + 1);
+ createRandomFile(dir, segment + ".big4", 3 * chunk);
+ createRandomFile(dir, segment + ".big5", 3 * chunk - 1);
+ createRandomFile(dir, segment + ".big6", 3 * chunk + 1);
+ createRandomFile(dir, segment + ".big7", 1000 * chunk);
+
+ // Setup extraneous files
+ createRandomFile(dir, "onetwothree", 100);
+ createRandomFile(dir, segment + ".notIn", 50);
+ createRandomFile(dir, segment + ".notIn2", 51);
+
+ // Now test
+ CompoundFileWriter csw = new CompoundFileWriter(dir, "test.cfs");
+ final String data[] = new String[] {
+ ".zero", ".one", ".ten", ".hundred", ".big1", ".big2", ".big3",
+ ".big4", ".big5", ".big6", ".big7"
+ };
+ for (int i=0; i<data.length; i++) {
+ csw.addFile(segment + data[i]);
+ }
+ csw.close();
+
+ CompoundFileReader csr = new CompoundFileReader(dir, "test.cfs");
+ for (int i=0; i<data.length; i++) {
+ IndexInput check = dir.openInput(segment + data[i]);
+ IndexInput test = csr.openInput(segment + data[i]);
+ assertSameStreams(data[i], check, test);
+ assertSameSeekBehavior(data[i], check, test);
+ test.close();
+ check.close();
+ }
+ csr.close();
+ }
+
+
+ /** Setup a larger compound file with a number of components, each of
+ * which is a sequential file (so that we can easily tell that we are
+ * reading in the right byte). The methods sets up 20 files - f0 to f19,
+ * the size of each file is 1000 bytes.
+ */
+ private void setUp_2() throws IOException {
+ CompoundFileWriter cw = new CompoundFileWriter(dir, "f.comp");
+ for (int i=0; i<20; i++) {
+ createSequenceFile(dir, "f" + i, (byte) 0, 2000);
+ cw.addFile("f" + i);
+ }
+ cw.close();
+ }
+
+
+ public void testReadAfterClose() throws IOException {
+ demo_FSIndexInputBug(dir, "test");
+ }
+
+ private void demo_FSIndexInputBug(Directory fsdir, String file)
+ throws IOException
+ {
+ // Setup the test file - we need more than 1024 bytes
+ IndexOutput os = fsdir.createOutput(file);
+ for(int i=0; i<2000; i++) {
+ os.writeByte((byte) i);
+ }
+ os.close();
+
+ IndexInput in = fsdir.openInput(file);
+
+ // This read primes the buffer in IndexInput
+ in.readByte();
+
+ // Close the file
+ in.close();
+
+ // ERROR: this call should fail, but succeeds because the buffer
+ // is still filled
+ in.readByte();
+
+ // ERROR: this call should fail, but succeeds for some reason as well
+ in.seek(1099);
+
+ try {
+ // OK: this call correctly fails. We are now past the 1024 internal
+ // buffer, so an actual IO is attempted, which fails
+ in.readByte();
+ fail("expected readByte() to throw exception");
+ } catch (IOException e) {
+ // expected exception
+ }
+ }
+
+
+ static boolean isCSIndexInput(IndexInput is) {
+ return is instanceof CompoundFileReader.CSIndexInput;
+ }
+
+ static boolean isCSIndexInputOpen(IndexInput is) throws IOException {
+ if (isCSIndexInput(is)) {
+ CompoundFileReader.CSIndexInput cis =
+ (CompoundFileReader.CSIndexInput) is;
+
+ return _TestHelper.isSimpleFSIndexInputOpen(cis.base);
+ } else {
+ return false;
+ }
+ }
+
+
+ public void testClonedStreamsClosing() throws IOException {
+ setUp_2();
+ CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
+
+ // basic clone
+ IndexInput expected = dir.openInput("f11");
+
+ // this test only works for FSIndexInput
+ assertTrue(_TestHelper.isSimpleFSIndexInput(expected));
+ assertTrue(_TestHelper.isSimpleFSIndexInputOpen(expected));
+
+ IndexInput one = cr.openInput("f11");
+ assertTrue(isCSIndexInputOpen(one));
+
+ IndexInput two = (IndexInput) one.clone();
+ assertTrue(isCSIndexInputOpen(two));
+
+ assertSameStreams("basic clone one", expected, one);
+ expected.seek(0);
+ assertSameStreams("basic clone two", expected, two);
+
+ // Now close the first stream
+ one.close();
+ assertTrue("Only close when cr is closed", isCSIndexInputOpen(one));
+
+ // The following should really fail since we couldn't expect to
+ // access a file once close has been called on it (regardless of
+ // buffering and/or clone magic)
+ expected.seek(0);
+ two.seek(0);
+ assertSameStreams("basic clone two/2", expected, two);
+
+
+ // Now close the compound reader
+ cr.close();
+ assertFalse("Now closed one", isCSIndexInputOpen(one));
+ assertFalse("Now closed two", isCSIndexInputOpen(two));
+
+ // The following may also fail since the compound stream is closed
+ expected.seek(0);
+ two.seek(0);
+ //assertSameStreams("basic clone two/3", expected, two);
+
+
+ // Now close the second clone
+ two.close();
+ expected.seek(0);
+ two.seek(0);
+ //assertSameStreams("basic clone two/4", expected, two);
+
+ expected.close();
+ }
+
+
+ /** This test opens two files from a compound stream and verifies that
+ * their file positions are independent of each other.
+ */
+ public void testRandomAccess() throws IOException {
+ setUp_2();
+ CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
+
+ // Open two files
+ IndexInput e1 = dir.openInput("f11");
+ IndexInput e2 = dir.openInput("f3");
+
+ IndexInput a1 = cr.openInput("f11");
+ IndexInput a2 = dir.openInput("f3");
+
+ // Seek the first pair
+ e1.seek(100);
+ a1.seek(100);
+ assertEquals(100, e1.getFilePointer());
+ assertEquals(100, a1.getFilePointer());
+ byte be1 = e1.readByte();
+ byte ba1 = a1.readByte();
+ assertEquals(be1, ba1);
+
+ // Now seek the second pair
+ e2.seek(1027);
+ a2.seek(1027);
+ assertEquals(1027, e2.getFilePointer());
+ assertEquals(1027, a2.getFilePointer());
+ byte be2 = e2.readByte();
+ byte ba2 = a2.readByte();
+ assertEquals(be2, ba2);
+
+ // Now make sure the first one didn't move
+ assertEquals(101, e1.getFilePointer());
+ assertEquals(101, a1.getFilePointer());
+ be1 = e1.readByte();
+ ba1 = a1.readByte();
+ assertEquals(be1, ba1);
+
+ // Now more the first one again, past the buffer length
+ e1.seek(1910);
+ a1.seek(1910);
+ assertEquals(1910, e1.getFilePointer());
+ assertEquals(1910, a1.getFilePointer());
+ be1 = e1.readByte();
+ ba1 = a1.readByte();
+ assertEquals(be1, ba1);
+
+ // Now make sure the second set didn't move
+ assertEquals(1028, e2.getFilePointer());
+ assertEquals(1028, a2.getFilePointer());
+ be2 = e2.readByte();
+ ba2 = a2.readByte();
+ assertEquals(be2, ba2);
+
+ // Move the second set back, again cross the buffer size
+ e2.seek(17);
+ a2.seek(17);
+ assertEquals(17, e2.getFilePointer());
+ assertEquals(17, a2.getFilePointer());
+ be2 = e2.readByte();
+ ba2 = a2.readByte();
+ assertEquals(be2, ba2);
+
+ // Finally, make sure the first set didn't move
+ // Now make sure the first one didn't move
+ assertEquals(1911, e1.getFilePointer());
+ assertEquals(1911, a1.getFilePointer());
+ be1 = e1.readByte();
+ ba1 = a1.readByte();
+ assertEquals(be1, ba1);
+
+ e1.close();
+ e2.close();
+ a1.close();
+ a2.close();
+ cr.close();
+ }
+
+ /** This test opens two files from a compound stream and verifies that
+ * their file positions are independent of each other.
+ */
+ public void testRandomAccessClones() throws IOException {
+ setUp_2();
+ CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
+
+ // Open two files
+ IndexInput e1 = cr.openInput("f11");
+ IndexInput e2 = cr.openInput("f3");
+
+ IndexInput a1 = (IndexInput) e1.clone();
+ IndexInput a2 = (IndexInput) e2.clone();
+
+ // Seek the first pair
+ e1.seek(100);
+ a1.seek(100);
+ assertEquals(100, e1.getFilePointer());
+ assertEquals(100, a1.getFilePointer());
+ byte be1 = e1.readByte();
+ byte ba1 = a1.readByte();
+ assertEquals(be1, ba1);
+
+ // Now seek the second pair
+ e2.seek(1027);
+ a2.seek(1027);
+ assertEquals(1027, e2.getFilePointer());
+ assertEquals(1027, a2.getFilePointer());
+ byte be2 = e2.readByte();
+ byte ba2 = a2.readByte();
+ assertEquals(be2, ba2);
+
+ // Now make sure the first one didn't move
+ assertEquals(101, e1.getFilePointer());
+ assertEquals(101, a1.getFilePointer());
+ be1 = e1.readByte();
+ ba1 = a1.readByte();
+ assertEquals(be1, ba1);
+
+ // Now more the first one again, past the buffer length
+ e1.seek(1910);
+ a1.seek(1910);
+ assertEquals(1910, e1.getFilePointer());
+ assertEquals(1910, a1.getFilePointer());
+ be1 = e1.readByte();
+ ba1 = a1.readByte();
+ assertEquals(be1, ba1);
+
+ // Now make sure the second set didn't move
+ assertEquals(1028, e2.getFilePointer());
+ assertEquals(1028, a2.getFilePointer());
+ be2 = e2.readByte();
+ ba2 = a2.readByte();
+ assertEquals(be2, ba2);
+
+ // Move the second set back, again cross the buffer size
+ e2.seek(17);
+ a2.seek(17);
+ assertEquals(17, e2.getFilePointer());
+ assertEquals(17, a2.getFilePointer());
+ be2 = e2.readByte();
+ ba2 = a2.readByte();
+ assertEquals(be2, ba2);
+
+ // Finally, make sure the first set didn't move
+ // Now make sure the first one didn't move
+ assertEquals(1911, e1.getFilePointer());
+ assertEquals(1911, a1.getFilePointer());
+ be1 = e1.readByte();
+ ba1 = a1.readByte();
+ assertEquals(be1, ba1);
+
+ e1.close();
+ e2.close();
+ a1.close();
+ a2.close();
+ cr.close();
+ }
+
+
+ public void testFileNotFound() throws IOException {
+ setUp_2();
+ CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
+
+ // Open two files
+ try {
+ cr.openInput("bogus");
+ fail("File not found");
+
+ } catch (IOException e) {
+ /* success */
+ //System.out.println("SUCCESS: File Not Found: " + e);
+ }
+
+ cr.close();
+ }
+
+
+ public void testReadPastEOF() throws IOException {
+ setUp_2();
+ CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
+ IndexInput is = cr.openInput("f2");
+ is.seek(is.length() - 10);
+ byte b[] = new byte[100];
+ is.readBytes(b, 0, 10);
+
+ try {
+ is.readByte();
+ fail("Single byte read past end of file");
+ } catch (IOException e) {
+ /* success */
+ //System.out.println("SUCCESS: single byte read past end of file: " + e);
+ }
+
+ is.seek(is.length() - 10);
+ try {
+ is.readBytes(b, 0, 50);
+ fail("Block read past end of file");
+ } catch (IOException e) {
+ /* success */
+ //System.out.println("SUCCESS: block read past end of file: " + e);
+ }
+
+ is.close();
+ cr.close();
+ }
+
+ /** This test that writes larger than the size of the buffer output
+ * will correctly increment the file pointer.
+ */
+ public void testLargeWrites() throws IOException {
+ IndexOutput os = dir.createOutput("testBufferStart.txt");
+
+ byte[] largeBuf = new byte[2048];
+ for (int i=0; i<largeBuf.length; i++) {
+ largeBuf[i] = (byte) (Math.random() * 256);
+ }
+
+ long currentPos = os.getFilePointer();
+ os.writeBytes(largeBuf, largeBuf.length);
+
+ try {
+ assertEquals(currentPos + largeBuf.length, os.getFilePointer());
+ } finally {
+ os.close();
+ }
+
+ }
+
+ public void testAddExternalFile() throws IOException {
+ createSequenceFile(dir, "d1", (byte) 0, 15);
+
+ Directory newDir = newDirectory();
+ CompoundFileWriter csw = new CompoundFileWriter(newDir, "d.csf");
+ csw.addFile("d1", dir);
+ csw.close();
+
+ CompoundFileReader csr = new CompoundFileReader(newDir, "d.csf");
+ IndexInput expected = dir.openInput("d1");
+ IndexInput actual = csr.openInput("d1");
+ assertSameStreams("d1", expected, actual);
+ assertSameSeekBehavior("d1", expected, actual);
+ expected.close();
+ actual.close();
+ csr.close();
+
+ newDir.close();
+ }
+
+ // Make sure we don't somehow use more than 1 descriptor
+ // when reading a CFS with many subs:
+ public void testManySubFiles() throws IOException {
+
+ final Directory d = newFSDirectory(_TestUtil.getTempDir("CFSManySubFiles"));
+ final int FILE_COUNT = 10000;
+
+ for(int fileIdx=0;fileIdx<FILE_COUNT;fileIdx++) {
+ IndexOutput out = d.createOutput("file." + fileIdx);
+ out.writeByte((byte) fileIdx);
+ out.close();
+ }
+
+ final CompoundFileWriter cfw = new CompoundFileWriter(d, "c.cfs");
+ for(int fileIdx=0;fileIdx<FILE_COUNT;fileIdx++) {
+ cfw.addFile("file." + fileIdx);
+ }
+ cfw.close();
+
+ final IndexInput[] ins = new IndexInput[FILE_COUNT];
+ final CompoundFileReader cfr = new CompoundFileReader(d, "c.cfs");
+ for(int fileIdx=0;fileIdx<FILE_COUNT;fileIdx++) {
+ ins[fileIdx] = cfr.openInput("file." + fileIdx);
+ }
+
+ for(int fileIdx=0;fileIdx<FILE_COUNT;fileIdx++) {
+ assertEquals((byte) fileIdx, ins[fileIdx].readByte());
+ }
+
+ for(int fileIdx=0;fileIdx<FILE_COUNT;fileIdx++) {
+ ins[fileIdx].close();
+ }
+ cfr.close();
+ d.close();
+ }
+}
fnp / pylucene.git / blobdiff