1 package org.apache.lucene.index;
4 * Licensed to the Apache Software Foundation (ASF) under one or more
5 * contributor license agreements. See the NOTICE file distributed with
6 * this work for additional information regarding copyright ownership.
7 * The ASF licenses this file to You under the Apache License, Version 2.0
8 * (the "License"); you may not use this file except in compliance with
9 * the License. You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
20 import java.io.IOException;
23 import org.apache.lucene.util.LuceneTestCase;
24 import org.apache.lucene.store.IndexOutput;
25 import org.apache.lucene.store.Directory;
26 import org.apache.lucene.store.IndexInput;
27 import org.apache.lucene.store.MockDirectoryWrapper;
28 import org.apache.lucene.store.MockDirectoryWrapper.Failure;
29 import org.apache.lucene.store.SimpleFSDirectory;
30 import org.apache.lucene.store._TestHelper;
31 import org.apache.lucene.util._TestUtil;
34 public class TestCompoundFile extends LuceneTestCase
36 private Directory dir;
39 public void setUp() throws Exception {
41 File file = _TestUtil.getTempDir("testIndex");
42 // use a simple FSDir here, to be sure to have SimpleFSInputs
43 dir = new SimpleFSDirectory(file,null);
47 public void tearDown() throws Exception {
52 /** Creates a file of the specified size with random data. */
53 private void createRandomFile(Directory dir, String name, int size)
56 IndexOutput os = dir.createOutput(name);
57 for (int i=0; i<size; i++) {
58 byte b = (byte) (Math.random() * 256);
64 /** Creates a file of the specified size with sequential data. The first
65 * byte is written as the start byte provided. All subsequent bytes are
66 * computed as start + offset where offset is the number of the byte.
68 private void createSequenceFile(Directory dir,
74 IndexOutput os = dir.createOutput(name);
75 for (int i=0; i < size; i++) {
83 private void assertSameStreams(String msg,
88 assertNotNull(msg + " null expected", expected);
89 assertNotNull(msg + " null test", test);
90 assertEquals(msg + " length", expected.length(), test.length());
91 assertEquals(msg + " position", expected.getFilePointer(),
92 test.getFilePointer());
94 byte expectedBuffer[] = new byte[512];
95 byte testBuffer[] = new byte[expectedBuffer.length];
97 long remainder = expected.length() - expected.getFilePointer();
98 while(remainder > 0) {
99 int readLen = (int) Math.min(remainder, expectedBuffer.length);
100 expected.readBytes(expectedBuffer, 0, readLen);
101 test.readBytes(testBuffer, 0, readLen);
102 assertEqualArrays(msg + ", remainder " + remainder, expectedBuffer,
103 testBuffer, 0, readLen);
104 remainder -= readLen;
109 private void assertSameStreams(String msg,
115 if(seekTo >= 0 && seekTo < expected.length())
117 expected.seek(seekTo);
119 assertSameStreams(msg + ", seek(mid)", expected, actual);
125 private void assertSameSeekBehavior(String msg,
132 assertSameStreams(msg + ", seek(0)", expected, actual, point);
135 point = expected.length() / 2l;
136 assertSameStreams(msg + ", seek(mid)", expected, actual, point);
139 point = expected.length() - 2;
140 assertSameStreams(msg + ", seek(end-2)", expected, actual, point);
143 point = expected.length() - 1;
144 assertSameStreams(msg + ", seek(end-1)", expected, actual, point);
147 point = expected.length();
148 assertSameStreams(msg + ", seek(end)", expected, actual, point);
151 point = expected.length() + 1;
152 assertSameStreams(msg + ", seek(end+1)", expected, actual, point);
156 private void assertEqualArrays(String msg,
162 assertNotNull(msg + " null expected", expected);
163 assertNotNull(msg + " null test", test);
165 for (int i=start; i<len; i++) {
166 assertEquals(msg + " " + i, expected[i], test[i]);
171 // ===========================================================
172 // Tests of the basic CompoundFile functionality
173 // ===========================================================
176 /** This test creates compound file based on a single file.
177 * Files of different sizes are tested: 0, 1, 10, 100 bytes.
179 public void testSingleFile() throws IOException {
180 int data[] = new int[] { 0, 1, 10, 100 };
181 for (int i=0; i<data.length; i++) {
182 String name = "t" + data[i];
183 createSequenceFile(dir, name, (byte) 0, data[i]);
184 CompoundFileWriter csw = new CompoundFileWriter(dir, name + ".cfs");
188 CompoundFileReader csr = new CompoundFileReader(dir, name + ".cfs");
189 IndexInput expected = dir.openInput(name);
190 IndexInput actual = csr.openInput(name);
191 assertSameStreams(name, expected, actual);
192 assertSameSeekBehavior(name, expected, actual);
200 /** This test creates compound file based on two files.
203 public void testTwoFiles() throws IOException {
204 createSequenceFile(dir, "d1", (byte) 0, 15);
205 createSequenceFile(dir, "d2", (byte) 0, 114);
207 CompoundFileWriter csw = new CompoundFileWriter(dir, "d.csf");
212 CompoundFileReader csr = new CompoundFileReader(dir, "d.csf");
213 IndexInput expected = dir.openInput("d1");
214 IndexInput actual = csr.openInput("d1");
215 assertSameStreams("d1", expected, actual);
216 assertSameSeekBehavior("d1", expected, actual);
220 expected = dir.openInput("d2");
221 actual = csr.openInput("d2");
222 assertSameStreams("d2", expected, actual);
223 assertSameSeekBehavior("d2", expected, actual);
229 /** This test creates a compound file based on a large number of files of
230 * various length. The file content is generated randomly. The sizes range
231 * from 0 to 1Mb. Some of the sizes are selected to test the buffering
232 * logic in the file reading code. For this the chunk variable is set to
233 * the length of the buffer used internally by the compound file logic.
235 public void testRandomFiles() throws IOException {
236 // Setup the test segment
237 String segment = "test";
238 int chunk = 1024; // internal buffer size used by the stream
239 createRandomFile(dir, segment + ".zero", 0);
240 createRandomFile(dir, segment + ".one", 1);
241 createRandomFile(dir, segment + ".ten", 10);
242 createRandomFile(dir, segment + ".hundred", 100);
243 createRandomFile(dir, segment + ".big1", chunk);
244 createRandomFile(dir, segment + ".big2", chunk - 1);
245 createRandomFile(dir, segment + ".big3", chunk + 1);
246 createRandomFile(dir, segment + ".big4", 3 * chunk);
247 createRandomFile(dir, segment + ".big5", 3 * chunk - 1);
248 createRandomFile(dir, segment + ".big6", 3 * chunk + 1);
249 createRandomFile(dir, segment + ".big7", 1000 * chunk);
251 // Setup extraneous files
252 createRandomFile(dir, "onetwothree", 100);
253 createRandomFile(dir, segment + ".notIn", 50);
254 createRandomFile(dir, segment + ".notIn2", 51);
257 CompoundFileWriter csw = new CompoundFileWriter(dir, "test.cfs");
258 final String data[] = new String[] {
259 ".zero", ".one", ".ten", ".hundred", ".big1", ".big2", ".big3",
260 ".big4", ".big5", ".big6", ".big7"
262 for (int i=0; i<data.length; i++) {
263 csw.addFile(segment + data[i]);
267 CompoundFileReader csr = new CompoundFileReader(dir, "test.cfs");
268 for (int i=0; i<data.length; i++) {
269 IndexInput check = dir.openInput(segment + data[i]);
270 IndexInput test = csr.openInput(segment + data[i]);
271 assertSameStreams(data[i], check, test);
272 assertSameSeekBehavior(data[i], check, test);
280 /** Setup a larger compound file with a number of components, each of
281 * which is a sequential file (so that we can easily tell that we are
282 * reading in the right byte). The methods sets up 20 files - f0 to f19,
283 * the size of each file is 1000 bytes.
285 private void setUp_2() throws IOException {
286 CompoundFileWriter cw = new CompoundFileWriter(dir, "f.comp");
287 for (int i=0; i<20; i++) {
288 createSequenceFile(dir, "f" + i, (byte) 0, 2000);
295 public void testReadAfterClose() throws IOException {
296 demo_FSIndexInputBug(dir, "test");
299 private void demo_FSIndexInputBug(Directory fsdir, String file)
302 // Setup the test file - we need more than 1024 bytes
303 IndexOutput os = fsdir.createOutput(file);
304 for(int i=0; i<2000; i++) {
305 os.writeByte((byte) i);
309 IndexInput in = fsdir.openInput(file);
311 // This read primes the buffer in IndexInput
317 // ERROR: this call should fail, but succeeds because the buffer
321 // ERROR: this call should fail, but succeeds for some reason as well
325 // OK: this call correctly fails. We are now past the 1024 internal
326 // buffer, so an actual IO is attempted, which fails
328 fail("expected readByte() to throw exception");
329 } catch (IOException e) {
330 // expected exception
335 static boolean isCSIndexInput(IndexInput is) {
336 return is instanceof CompoundFileReader.CSIndexInput;
339 static boolean isCSIndexInputOpen(IndexInput is) throws IOException {
340 if (isCSIndexInput(is)) {
341 CompoundFileReader.CSIndexInput cis =
342 (CompoundFileReader.CSIndexInput) is;
344 return _TestHelper.isSimpleFSIndexInputOpen(cis.base);
351 public void testClonedStreamsClosing() throws IOException {
353 CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
356 IndexInput expected = dir.openInput("f11");
358 // this test only works for FSIndexInput
359 assertTrue(_TestHelper.isSimpleFSIndexInput(expected));
360 assertTrue(_TestHelper.isSimpleFSIndexInputOpen(expected));
362 IndexInput one = cr.openInput("f11");
363 assertTrue(isCSIndexInputOpen(one));
365 IndexInput two = (IndexInput) one.clone();
366 assertTrue(isCSIndexInputOpen(two));
368 assertSameStreams("basic clone one", expected, one);
370 assertSameStreams("basic clone two", expected, two);
372 // Now close the first stream
374 assertTrue("Only close when cr is closed", isCSIndexInputOpen(one));
376 // The following should really fail since we couldn't expect to
377 // access a file once close has been called on it (regardless of
378 // buffering and/or clone magic)
381 assertSameStreams("basic clone two/2", expected, two);
384 // Now close the compound reader
386 assertFalse("Now closed one", isCSIndexInputOpen(one));
387 assertFalse("Now closed two", isCSIndexInputOpen(two));
389 // The following may also fail since the compound stream is closed
392 //assertSameStreams("basic clone two/3", expected, two);
395 // Now close the second clone
399 //assertSameStreams("basic clone two/4", expected, two);
405 /** This test opens two files from a compound stream and verifies that
406 * their file positions are independent of each other.
408 public void testRandomAccess() throws IOException {
410 CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
413 IndexInput e1 = dir.openInput("f11");
414 IndexInput e2 = dir.openInput("f3");
416 IndexInput a1 = cr.openInput("f11");
417 IndexInput a2 = dir.openInput("f3");
419 // Seek the first pair
422 assertEquals(100, e1.getFilePointer());
423 assertEquals(100, a1.getFilePointer());
424 byte be1 = e1.readByte();
425 byte ba1 = a1.readByte();
426 assertEquals(be1, ba1);
428 // Now seek the second pair
431 assertEquals(1027, e2.getFilePointer());
432 assertEquals(1027, a2.getFilePointer());
433 byte be2 = e2.readByte();
434 byte ba2 = a2.readByte();
435 assertEquals(be2, ba2);
437 // Now make sure the first one didn't move
438 assertEquals(101, e1.getFilePointer());
439 assertEquals(101, a1.getFilePointer());
442 assertEquals(be1, ba1);
444 // Now more the first one again, past the buffer length
447 assertEquals(1910, e1.getFilePointer());
448 assertEquals(1910, a1.getFilePointer());
451 assertEquals(be1, ba1);
453 // Now make sure the second set didn't move
454 assertEquals(1028, e2.getFilePointer());
455 assertEquals(1028, a2.getFilePointer());
458 assertEquals(be2, ba2);
460 // Move the second set back, again cross the buffer size
463 assertEquals(17, e2.getFilePointer());
464 assertEquals(17, a2.getFilePointer());
467 assertEquals(be2, ba2);
469 // Finally, make sure the first set didn't move
470 // Now make sure the first one didn't move
471 assertEquals(1911, e1.getFilePointer());
472 assertEquals(1911, a1.getFilePointer());
475 assertEquals(be1, ba1);
484 /** This test opens two files from a compound stream and verifies that
485 * their file positions are independent of each other.
487 public void testRandomAccessClones() throws IOException {
489 CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
492 IndexInput e1 = cr.openInput("f11");
493 IndexInput e2 = cr.openInput("f3");
495 IndexInput a1 = (IndexInput) e1.clone();
496 IndexInput a2 = (IndexInput) e2.clone();
498 // Seek the first pair
501 assertEquals(100, e1.getFilePointer());
502 assertEquals(100, a1.getFilePointer());
503 byte be1 = e1.readByte();
504 byte ba1 = a1.readByte();
505 assertEquals(be1, ba1);
507 // Now seek the second pair
510 assertEquals(1027, e2.getFilePointer());
511 assertEquals(1027, a2.getFilePointer());
512 byte be2 = e2.readByte();
513 byte ba2 = a2.readByte();
514 assertEquals(be2, ba2);
516 // Now make sure the first one didn't move
517 assertEquals(101, e1.getFilePointer());
518 assertEquals(101, a1.getFilePointer());
521 assertEquals(be1, ba1);
523 // Now more the first one again, past the buffer length
526 assertEquals(1910, e1.getFilePointer());
527 assertEquals(1910, a1.getFilePointer());
530 assertEquals(be1, ba1);
532 // Now make sure the second set didn't move
533 assertEquals(1028, e2.getFilePointer());
534 assertEquals(1028, a2.getFilePointer());
537 assertEquals(be2, ba2);
539 // Move the second set back, again cross the buffer size
542 assertEquals(17, e2.getFilePointer());
543 assertEquals(17, a2.getFilePointer());
546 assertEquals(be2, ba2);
548 // Finally, make sure the first set didn't move
549 // Now make sure the first one didn't move
550 assertEquals(1911, e1.getFilePointer());
551 assertEquals(1911, a1.getFilePointer());
554 assertEquals(be1, ba1);
564 public void testFileNotFound() throws IOException {
566 CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
570 cr.openInput("bogus");
571 fail("File not found");
573 } catch (IOException e) {
575 //System.out.println("SUCCESS: File Not Found: " + e);
582 public void testReadPastEOF() throws IOException {
584 CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
585 IndexInput is = cr.openInput("f2");
586 is.seek(is.length() - 10);
587 byte b[] = new byte[100];
588 is.readBytes(b, 0, 10);
592 fail("Single byte read past end of file");
593 } catch (IOException e) {
595 //System.out.println("SUCCESS: single byte read past end of file: " + e);
598 is.seek(is.length() - 10);
600 is.readBytes(b, 0, 50);
601 fail("Block read past end of file");
602 } catch (IOException e) {
604 //System.out.println("SUCCESS: block read past end of file: " + e);
611 /** This test that writes larger than the size of the buffer output
612 * will correctly increment the file pointer.
614 public void testLargeWrites() throws IOException {
615 IndexOutput os = dir.createOutput("testBufferStart.txt");
617 byte[] largeBuf = new byte[2048];
618 for (int i=0; i<largeBuf.length; i++) {
619 largeBuf[i] = (byte) (Math.random() * 256);
622 long currentPos = os.getFilePointer();
623 os.writeBytes(largeBuf, largeBuf.length);
626 assertEquals(currentPos + largeBuf.length, os.getFilePointer());
633 public void testAddExternalFile() throws IOException {
634 createSequenceFile(dir, "d1", (byte) 0, 15);
636 Directory newDir = newDirectory();
637 CompoundFileWriter csw = new CompoundFileWriter(newDir, "d.csf");
638 csw.addFile("d1", dir);
641 CompoundFileReader csr = new CompoundFileReader(newDir, "d.csf");
642 IndexInput expected = dir.openInput("d1");
643 IndexInput actual = csr.openInput("d1");
644 assertSameStreams("d1", expected, actual);
645 assertSameSeekBehavior("d1", expected, actual);
653 // Make sure we don't somehow use more than 1 descriptor
654 // when reading a CFS with many subs:
655 public void testManySubFiles() throws IOException {
657 final Directory d = newFSDirectory(_TestUtil.getTempDir("CFSManySubFiles"));
658 final int FILE_COUNT = 10000;
660 for(int fileIdx=0;fileIdx<FILE_COUNT;fileIdx++) {
661 IndexOutput out = d.createOutput("file." + fileIdx);
662 out.writeByte((byte) fileIdx);
666 final CompoundFileWriter cfw = new CompoundFileWriter(d, "c.cfs");
667 for(int fileIdx=0;fileIdx<FILE_COUNT;fileIdx++) {
668 cfw.addFile("file." + fileIdx);
672 final IndexInput[] ins = new IndexInput[FILE_COUNT];
673 final CompoundFileReader cfr = new CompoundFileReader(d, "c.cfs");
674 for(int fileIdx=0;fileIdx<FILE_COUNT;fileIdx++) {
675 ins[fileIdx] = cfr.openInput("file." + fileIdx);
678 for(int fileIdx=0;fileIdx<FILE_COUNT;fileIdx++) {
679 assertEquals((byte) fileIdx, ins[fileIdx].readByte());
682 for(int fileIdx=0;fileIdx<FILE_COUNT;fileIdx++) {
683 ins[fileIdx].close();