1 package org.apache.lucene.index;
4 * Licensed to the Apache Software Foundation (ASF) under one or more
5 * contributor license agreements. See the NOTICE file distributed with
6 * this work for additional information regarding copyright ownership.
7 * The ASF licenses this file to You under the Apache License, Version 2.0
8 * (the "License"); you may not use this file except in compliance with
9 * the License. You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
20 import java.io.IOException;
23 import org.apache.lucene.util.LuceneTestCase;
24 import junit.framework.TestSuite;
25 import junit.textui.TestRunner;
26 import org.apache.lucene.store.IndexOutput;
27 import org.apache.lucene.store.Directory;
28 import org.apache.lucene.store.IndexInput;
29 import org.apache.lucene.store.SimpleFSDirectory;
30 import org.apache.lucene.store._TestHelper;
31 import org.apache.lucene.util._TestUtil;
34 public class TestCompoundFile extends LuceneTestCase
36 /** Main for running test case by itself. */
37 public static void main(String args[]) {
38 TestRunner.run (new TestSuite(TestCompoundFile.class));
39 // TestRunner.run (new TestCompoundFile("testSingleFile"));
40 // TestRunner.run (new TestCompoundFile("testTwoFiles"));
41 // TestRunner.run (new TestCompoundFile("testRandomFiles"));
42 // TestRunner.run (new TestCompoundFile("testClonedStreamsClosing"));
43 // TestRunner.run (new TestCompoundFile("testReadAfterClose"));
44 // TestRunner.run (new TestCompoundFile("testRandomAccess"));
45 // TestRunner.run (new TestCompoundFile("testRandomAccessClones"));
46 // TestRunner.run (new TestCompoundFile("testFileNotFound"));
47 // TestRunner.run (new TestCompoundFile("testReadPastEOF"));
49 // TestRunner.run (new TestCompoundFile("testIWCreate"));
54 private Directory dir;
58 public void setUp() throws Exception {
60 File file = _TestUtil.getTempDir("testIndex");
61 // use a simple FSDir here, to be sure to have SimpleFSInputs
62 dir = new SimpleFSDirectory(file,null);
66 public void tearDown() throws Exception {
71 /** Creates a file of the specified size with random data. */
72 private void createRandomFile(Directory dir, String name, int size)
75 IndexOutput os = dir.createOutput(name);
76 for (int i=0; i<size; i++) {
77 byte b = (byte) (Math.random() * 256);
83 /** Creates a file of the specified size with sequential data. The first
84 * byte is written as the start byte provided. All subsequent bytes are
85 * computed as start + offset where offset is the number of the byte.
87 private void createSequenceFile(Directory dir,
93 IndexOutput os = dir.createOutput(name);
94 for (int i=0; i < size; i++) {
102 private void assertSameStreams(String msg,
107 assertNotNull(msg + " null expected", expected);
108 assertNotNull(msg + " null test", test);
109 assertEquals(msg + " length", expected.length(), test.length());
110 assertEquals(msg + " position", expected.getFilePointer(),
111 test.getFilePointer());
113 byte expectedBuffer[] = new byte[512];
114 byte testBuffer[] = new byte[expectedBuffer.length];
116 long remainder = expected.length() - expected.getFilePointer();
117 while(remainder > 0) {
118 int readLen = (int) Math.min(remainder, expectedBuffer.length);
119 expected.readBytes(expectedBuffer, 0, readLen);
120 test.readBytes(testBuffer, 0, readLen);
121 assertEqualArrays(msg + ", remainder " + remainder, expectedBuffer,
122 testBuffer, 0, readLen);
123 remainder -= readLen;
128 private void assertSameStreams(String msg,
134 if(seekTo >= 0 && seekTo < expected.length())
136 expected.seek(seekTo);
138 assertSameStreams(msg + ", seek(mid)", expected, actual);
144 private void assertSameSeekBehavior(String msg,
151 assertSameStreams(msg + ", seek(0)", expected, actual, point);
154 point = expected.length() / 2l;
155 assertSameStreams(msg + ", seek(mid)", expected, actual, point);
158 point = expected.length() - 2;
159 assertSameStreams(msg + ", seek(end-2)", expected, actual, point);
162 point = expected.length() - 1;
163 assertSameStreams(msg + ", seek(end-1)", expected, actual, point);
166 point = expected.length();
167 assertSameStreams(msg + ", seek(end)", expected, actual, point);
170 point = expected.length() + 1;
171 assertSameStreams(msg + ", seek(end+1)", expected, actual, point);
175 private void assertEqualArrays(String msg,
181 assertNotNull(msg + " null expected", expected);
182 assertNotNull(msg + " null test", test);
184 for (int i=start; i<len; i++) {
185 assertEquals(msg + " " + i, expected[i], test[i]);
190 // ===========================================================
191 // Tests of the basic CompoundFile functionality
192 // ===========================================================
195 /** This test creates compound file based on a single file.
196 * Files of different sizes are tested: 0, 1, 10, 100 bytes.
198 public void testSingleFile() throws IOException {
199 int data[] = new int[] { 0, 1, 10, 100 };
200 for (int i=0; i<data.length; i++) {
201 String name = "t" + data[i];
202 createSequenceFile(dir, name, (byte) 0, data[i]);
203 CompoundFileWriter csw = new CompoundFileWriter(dir, name + ".cfs");
207 CompoundFileReader csr = new CompoundFileReader(dir, name + ".cfs");
208 IndexInput expected = dir.openInput(name);
209 IndexInput actual = csr.openInput(name);
210 assertSameStreams(name, expected, actual);
211 assertSameSeekBehavior(name, expected, actual);
219 /** This test creates compound file based on two files.
222 public void testTwoFiles() throws IOException {
223 createSequenceFile(dir, "d1", (byte) 0, 15);
224 createSequenceFile(dir, "d2", (byte) 0, 114);
226 CompoundFileWriter csw = new CompoundFileWriter(dir, "d.csf");
231 CompoundFileReader csr = new CompoundFileReader(dir, "d.csf");
232 IndexInput expected = dir.openInput("d1");
233 IndexInput actual = csr.openInput("d1");
234 assertSameStreams("d1", expected, actual);
235 assertSameSeekBehavior("d1", expected, actual);
239 expected = dir.openInput("d2");
240 actual = csr.openInput("d2");
241 assertSameStreams("d2", expected, actual);
242 assertSameSeekBehavior("d2", expected, actual);
248 /** This test creates a compound file based on a large number of files of
249 * various length. The file content is generated randomly. The sizes range
250 * from 0 to 1Mb. Some of the sizes are selected to test the buffering
251 * logic in the file reading code. For this the chunk variable is set to
252 * the length of the buffer used internally by the compound file logic.
254 public void testRandomFiles() throws IOException {
255 // Setup the test segment
256 String segment = "test";
257 int chunk = 1024; // internal buffer size used by the stream
258 createRandomFile(dir, segment + ".zero", 0);
259 createRandomFile(dir, segment + ".one", 1);
260 createRandomFile(dir, segment + ".ten", 10);
261 createRandomFile(dir, segment + ".hundred", 100);
262 createRandomFile(dir, segment + ".big1", chunk);
263 createRandomFile(dir, segment + ".big2", chunk - 1);
264 createRandomFile(dir, segment + ".big3", chunk + 1);
265 createRandomFile(dir, segment + ".big4", 3 * chunk);
266 createRandomFile(dir, segment + ".big5", 3 * chunk - 1);
267 createRandomFile(dir, segment + ".big6", 3 * chunk + 1);
268 createRandomFile(dir, segment + ".big7", 1000 * chunk);
270 // Setup extraneous files
271 createRandomFile(dir, "onetwothree", 100);
272 createRandomFile(dir, segment + ".notIn", 50);
273 createRandomFile(dir, segment + ".notIn2", 51);
276 CompoundFileWriter csw = new CompoundFileWriter(dir, "test.cfs");
277 final String data[] = new String[] {
278 ".zero", ".one", ".ten", ".hundred", ".big1", ".big2", ".big3",
279 ".big4", ".big5", ".big6", ".big7"
281 for (int i=0; i<data.length; i++) {
282 csw.addFile(segment + data[i]);
286 CompoundFileReader csr = new CompoundFileReader(dir, "test.cfs");
287 for (int i=0; i<data.length; i++) {
288 IndexInput check = dir.openInput(segment + data[i]);
289 IndexInput test = csr.openInput(segment + data[i]);
290 assertSameStreams(data[i], check, test);
291 assertSameSeekBehavior(data[i], check, test);
299 /** Setup a larger compound file with a number of components, each of
300 * which is a sequential file (so that we can easily tell that we are
301 * reading in the right byte). The methods sets up 20 files - f0 to f19,
302 * the size of each file is 1000 bytes.
304 private void setUp_2() throws IOException {
305 CompoundFileWriter cw = new CompoundFileWriter(dir, "f.comp");
306 for (int i=0; i<20; i++) {
307 createSequenceFile(dir, "f" + i, (byte) 0, 2000);
314 public void testReadAfterClose() throws IOException {
315 demo_FSIndexInputBug(dir, "test");
318 private void demo_FSIndexInputBug(Directory fsdir, String file)
321 // Setup the test file - we need more than 1024 bytes
322 IndexOutput os = fsdir.createOutput(file);
323 for(int i=0; i<2000; i++) {
324 os.writeByte((byte) i);
328 IndexInput in = fsdir.openInput(file);
330 // This read primes the buffer in IndexInput
336 // ERROR: this call should fail, but succeeds because the buffer
340 // ERROR: this call should fail, but succeeds for some reason as well
344 // OK: this call correctly fails. We are now past the 1024 internal
345 // buffer, so an actual IO is attempted, which fails
347 fail("expected readByte() to throw exception");
348 } catch (IOException e) {
349 // expected exception
354 static boolean isCSIndexInput(IndexInput is) {
355 return is instanceof CompoundFileReader.CSIndexInput;
358 static boolean isCSIndexInputOpen(IndexInput is) throws IOException {
359 if (isCSIndexInput(is)) {
360 CompoundFileReader.CSIndexInput cis =
361 (CompoundFileReader.CSIndexInput) is;
363 return _TestHelper.isSimpleFSIndexInputOpen(cis.base);
370 public void testClonedStreamsClosing() throws IOException {
372 CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
375 IndexInput expected = dir.openInput("f11");
377 // this test only works for FSIndexInput
378 assertTrue(_TestHelper.isSimpleFSIndexInput(expected));
379 assertTrue(_TestHelper.isSimpleFSIndexInputOpen(expected));
381 IndexInput one = cr.openInput("f11");
382 assertTrue(isCSIndexInputOpen(one));
384 IndexInput two = (IndexInput) one.clone();
385 assertTrue(isCSIndexInputOpen(two));
387 assertSameStreams("basic clone one", expected, one);
389 assertSameStreams("basic clone two", expected, two);
391 // Now close the first stream
393 assertTrue("Only close when cr is closed", isCSIndexInputOpen(one));
395 // The following should really fail since we couldn't expect to
396 // access a file once close has been called on it (regardless of
397 // buffering and/or clone magic)
400 assertSameStreams("basic clone two/2", expected, two);
403 // Now close the compound reader
405 assertFalse("Now closed one", isCSIndexInputOpen(one));
406 assertFalse("Now closed two", isCSIndexInputOpen(two));
408 // The following may also fail since the compound stream is closed
411 //assertSameStreams("basic clone two/3", expected, two);
414 // Now close the second clone
418 //assertSameStreams("basic clone two/4", expected, two);
424 /** This test opens two files from a compound stream and verifies that
425 * their file positions are independent of each other.
427 public void testRandomAccess() throws IOException {
429 CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
432 IndexInput e1 = dir.openInput("f11");
433 IndexInput e2 = dir.openInput("f3");
435 IndexInput a1 = cr.openInput("f11");
436 IndexInput a2 = dir.openInput("f3");
438 // Seek the first pair
441 assertEquals(100, e1.getFilePointer());
442 assertEquals(100, a1.getFilePointer());
443 byte be1 = e1.readByte();
444 byte ba1 = a1.readByte();
445 assertEquals(be1, ba1);
447 // Now seek the second pair
450 assertEquals(1027, e2.getFilePointer());
451 assertEquals(1027, a2.getFilePointer());
452 byte be2 = e2.readByte();
453 byte ba2 = a2.readByte();
454 assertEquals(be2, ba2);
456 // Now make sure the first one didn't move
457 assertEquals(101, e1.getFilePointer());
458 assertEquals(101, a1.getFilePointer());
461 assertEquals(be1, ba1);
463 // Now more the first one again, past the buffer length
466 assertEquals(1910, e1.getFilePointer());
467 assertEquals(1910, a1.getFilePointer());
470 assertEquals(be1, ba1);
472 // Now make sure the second set didn't move
473 assertEquals(1028, e2.getFilePointer());
474 assertEquals(1028, a2.getFilePointer());
477 assertEquals(be2, ba2);
479 // Move the second set back, again cross the buffer size
482 assertEquals(17, e2.getFilePointer());
483 assertEquals(17, a2.getFilePointer());
486 assertEquals(be2, ba2);
488 // Finally, make sure the first set didn't move
489 // Now make sure the first one didn't move
490 assertEquals(1911, e1.getFilePointer());
491 assertEquals(1911, a1.getFilePointer());
494 assertEquals(be1, ba1);
503 /** This test opens two files from a compound stream and verifies that
504 * their file positions are independent of each other.
506 public void testRandomAccessClones() throws IOException {
508 CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
511 IndexInput e1 = cr.openInput("f11");
512 IndexInput e2 = cr.openInput("f3");
514 IndexInput a1 = (IndexInput) e1.clone();
515 IndexInput a2 = (IndexInput) e2.clone();
517 // Seek the first pair
520 assertEquals(100, e1.getFilePointer());
521 assertEquals(100, a1.getFilePointer());
522 byte be1 = e1.readByte();
523 byte ba1 = a1.readByte();
524 assertEquals(be1, ba1);
526 // Now seek the second pair
529 assertEquals(1027, e2.getFilePointer());
530 assertEquals(1027, a2.getFilePointer());
531 byte be2 = e2.readByte();
532 byte ba2 = a2.readByte();
533 assertEquals(be2, ba2);
535 // Now make sure the first one didn't move
536 assertEquals(101, e1.getFilePointer());
537 assertEquals(101, a1.getFilePointer());
540 assertEquals(be1, ba1);
542 // Now more the first one again, past the buffer length
545 assertEquals(1910, e1.getFilePointer());
546 assertEquals(1910, a1.getFilePointer());
549 assertEquals(be1, ba1);
551 // Now make sure the second set didn't move
552 assertEquals(1028, e2.getFilePointer());
553 assertEquals(1028, a2.getFilePointer());
556 assertEquals(be2, ba2);
558 // Move the second set back, again cross the buffer size
561 assertEquals(17, e2.getFilePointer());
562 assertEquals(17, a2.getFilePointer());
565 assertEquals(be2, ba2);
567 // Finally, make sure the first set didn't move
568 // Now make sure the first one didn't move
569 assertEquals(1911, e1.getFilePointer());
570 assertEquals(1911, a1.getFilePointer());
573 assertEquals(be1, ba1);
583 public void testFileNotFound() throws IOException {
585 CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
589 cr.openInput("bogus");
590 fail("File not found");
592 } catch (IOException e) {
594 //System.out.println("SUCCESS: File Not Found: " + e);
601 public void testReadPastEOF() throws IOException {
603 CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
604 IndexInput is = cr.openInput("f2");
605 is.seek(is.length() - 10);
606 byte b[] = new byte[100];
607 is.readBytes(b, 0, 10);
611 fail("Single byte read past end of file");
612 } catch (IOException e) {
614 //System.out.println("SUCCESS: single byte read past end of file: " + e);
617 is.seek(is.length() - 10);
619 is.readBytes(b, 0, 50);
620 fail("Block read past end of file");
621 } catch (IOException e) {
623 //System.out.println("SUCCESS: block read past end of file: " + e);
630 /** This test that writes larger than the size of the buffer output
631 * will correctly increment the file pointer.
633 public void testLargeWrites() throws IOException {
634 IndexOutput os = dir.createOutput("testBufferStart.txt");
636 byte[] largeBuf = new byte[2048];
637 for (int i=0; i<largeBuf.length; i++) {
638 largeBuf[i] = (byte) (Math.random() * 256);
641 long currentPos = os.getFilePointer();
642 os.writeBytes(largeBuf, largeBuf.length);
645 assertEquals(currentPos + largeBuf.length, os.getFilePointer());
652 public void testAddExternalFile() throws IOException {
653 createSequenceFile(dir, "d1", (byte) 0, 15);
655 Directory newDir = newDirectory();
656 CompoundFileWriter csw = new CompoundFileWriter(newDir, "d.csf");
657 csw.addFile("d1", dir);
660 CompoundFileReader csr = new CompoundFileReader(newDir, "d.csf");
661 IndexInput expected = dir.openInput("d1");
662 IndexInput actual = csr.openInput("d1");
663 assertSameStreams("d1", expected, actual);
664 assertSameSeekBehavior("d1", expected, actual);