1 package org.apache.lucene.store;
4 * Licensed to the Apache Software Foundation (ASF) under one or more
5 * contributor license agreements. See the NOTICE file distributed with
6 * this work for additional information regarding copyright ownership.
7 * The ASF licenses this file to You under the Apache License, Version 2.0
8 * (the "License"); you may not use this file except in compliance with
9 * the License. You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
20 import java.io.Closeable;
21 import java.io.FileNotFoundException;
22 import java.io.IOException;
23 import java.util.ArrayList;
24 import java.util.Collection;
25 import java.util.Collections;
26 import java.util.HashMap;
27 import java.util.HashSet;
28 import java.util.IdentityHashMap;
29 import java.util.Iterator;
31 import java.util.Random;
34 import org.apache.lucene.index.IndexReader;
35 import org.apache.lucene.util.LuceneTestCase;
36 import org.apache.lucene.util.ThrottledIndexOutput;
37 import org.apache.lucene.util._TestUtil;
40 * This is a Directory Wrapper that adds methods
41 * intended to be used only by unit tests.
42 * It also adds a number of features useful for testing:
44 * <li> Instances created by {@link LuceneTestCase#newDirectory()} are tracked
45 * to ensure they are closed by the test.
46 * <li> When a MockDirectoryWrapper is closed, it will throw an exception if
47 * it has any open files against it (with a stacktrace indicating where
48 * they were opened from).
49 * <li> When a MockDirectoryWrapper is closed, it runs CheckIndex to test if
50 * the index was corrupted.
51 * <li> MockDirectoryWrapper simulates some "features" of Windows, such as
52 * refusing to write/delete to open files.
56 public class MockDirectoryWrapper extends Directory {
57 final Directory delegate;
60 // Max actual bytes used. This is set by MockRAMOutputStream:
62 double randomIOExceptionRate;
64 boolean noDeleteOpenFile = true;
65 boolean preventDoubleWrite = true;
66 boolean checkIndexOnClose = true;
67 boolean trackDiskUsage = false;
68 private Set<String> unSyncedFiles;
69 private Set<String> createdFiles;
70 private Set<String> openFilesForWrite = new HashSet<String>();
71 Set<String> openLocks = Collections.synchronizedSet(new HashSet<String>());
72 volatile boolean crashed;
73 private ThrottledIndexOutput throttledOutput;
74 private Throttling throttling = Throttling.SOMETIMES;
76 // use this for tracking files for crash.
77 // additionally: provides debugging information in case you leave one open
78 private Map<Closeable,Exception> openFileHandles = Collections.synchronizedMap(new IdentityHashMap<Closeable,Exception>());
80 // NOTE: we cannot initialize the Map here due to the
81 // order in which our constructor actually does this
82 // member initialization vs when it calls super. It seems
83 // like super is called, then our members are initialized:
84 private Map<String,Integer> openFiles;
86 // Only tracked if noDeleteOpenFile is true: if an attempt
87 // is made to delete an open file, we enroll it here.
88 private Set<String> openFilesDeleted;
90 private synchronized void init() {
91 if (openFiles == null) {
92 openFiles = new HashMap<String,Integer>();
93 openFilesDeleted = new HashSet<String>();
96 if (createdFiles == null)
97 createdFiles = new HashSet<String>();
98 if (unSyncedFiles == null)
99 unSyncedFiles = new HashSet<String>();
102 public MockDirectoryWrapper(Random random, Directory delegate) {
103 this.delegate = delegate;
104 // must make a private random since our methods are
105 // called from different threads; else test failures may
106 // not be reproducible from the original seed
107 this.randomState = new Random(random.nextInt());
108 this.throttledOutput = new ThrottledIndexOutput(ThrottledIndexOutput
109 .mBitsToBytes(40 + randomState.nextInt(10)), 5 + randomState.nextInt(5), null);
110 // force wrapping of lockfactory
112 setLockFactory(new MockLockFactoryWrapper(this, delegate.getLockFactory()));
113 } catch (IOException e) {
114 throw new RuntimeException(e);
119 public void setTrackDiskUsage(boolean v) {
123 /** If set to true, we throw an IOException if the same
124 * file is opened by createOutput, ever. */
125 public void setPreventDoubleWrite(boolean value) {
126 preventDoubleWrite = value;
131 public void sync(String name) throws IOException {
133 maybeThrowDeterministicException();
135 throw new IOException("cannot sync after crash");
136 unSyncedFiles.remove(name);
140 public static enum Throttling {
141 /** always emulate a slow hard disk. could be very slow! */
143 /** sometimes (2% of the time) emulate a slow hard disk. */
145 /** never throttle output */
149 public void setThrottling(Throttling throttling) {
150 this.throttling = throttling;
154 public synchronized void sync(Collection<String> names) throws IOException {
156 for (String name : names)
157 maybeThrowDeterministicException();
159 throw new IOException("cannot sync after crash");
160 unSyncedFiles.removeAll(names);
161 delegate.sync(names);
165 public String toString() {
167 return "MockDirWrapper(" + delegate + ")";
170 public synchronized final long sizeInBytes() throws IOException {
171 if (delegate instanceof RAMDirectory)
172 return ((RAMDirectory) delegate).sizeInBytes();
176 for (String file : delegate.listAll())
177 size += delegate.fileLength(file);
182 /** Simulates a crash of OS or machine by overwriting
184 public synchronized void crash() throws IOException {
186 openFiles = new HashMap<String,Integer>();
187 openFilesForWrite = new HashSet<String>();
188 openFilesDeleted = new HashSet<String>();
189 Iterator<String> it = unSyncedFiles.iterator();
190 unSyncedFiles = new HashSet<String>();
191 // first force-close all files, so we can corrupt on windows etc.
192 // clone the file map, as these guys want to remove themselves on close.
193 Map<Closeable,Exception> m = new IdentityHashMap<Closeable,Exception>(openFileHandles);
194 for (Closeable f : m.keySet())
197 } catch (Exception ignored) {}
200 while(it.hasNext()) {
201 String name = it.next();
202 if (count % 3 == 0) {
203 deleteFile(name, true);
204 } else if (count % 3 == 1) {
205 // Zero out file entirely
206 long length = fileLength(name);
207 byte[] zeroes = new byte[256];
209 IndexOutput out = delegate.createOutput(name);
210 while(upto < length) {
211 final int limit = (int) Math.min(length-upto, zeroes.length);
212 out.writeBytes(zeroes, 0, limit);
216 } else if (count % 3 == 2) {
217 // Truncate the file:
218 IndexOutput out = delegate.createOutput(name);
219 out.setLength(fileLength(name)/2);
226 public synchronized void clearCrash() throws IOException {
231 public void setMaxSizeInBytes(long maxSize) {
232 this.maxSize = maxSize;
234 public long getMaxSizeInBytes() {
239 * Returns the peek actual storage used (bytes) in this
242 public long getMaxUsedSizeInBytes() {
243 return this.maxUsedSize;
245 public void resetMaxUsedSizeInBytes() throws IOException {
246 this.maxUsedSize = getRecomputedActualSizeInBytes();
250 * Emulate windows whereby deleting an open file is not
251 * allowed (raise IOException).
253 public void setNoDeleteOpenFile(boolean value) {
254 this.noDeleteOpenFile = value;
256 public boolean getNoDeleteOpenFile() {
257 return noDeleteOpenFile;
261 * Set whether or not checkindex should be run
264 public void setCheckIndexOnClose(boolean value) {
265 this.checkIndexOnClose = value;
268 public boolean getCheckIndexOnClose() {
269 return checkIndexOnClose;
272 * If 0.0, no exceptions will be thrown. Else this should
273 * be a double 0.0 - 1.0. We will randomly throw an
274 * IOException on the first write to an OutputStream based
275 * on this probability.
277 public void setRandomIOExceptionRate(double rate) {
278 randomIOExceptionRate = rate;
280 public double getRandomIOExceptionRate() {
281 return randomIOExceptionRate;
284 void maybeThrowIOException() throws IOException {
285 if (randomIOExceptionRate > 0.0) {
286 int number = Math.abs(randomState.nextInt() % 1000);
287 if (number < randomIOExceptionRate*1000) {
288 if (LuceneTestCase.VERBOSE) {
289 System.out.println(Thread.currentThread().getName() + ": MockDirectoryWrapper: now throw random exception");
290 new Throwable().printStackTrace(System.out);
292 throw new IOException("a random IOException");
298 public synchronized void deleteFile(String name) throws IOException {
300 deleteFile(name, false);
303 // sets the cause of the incoming ioe to be the stack
304 // trace when the offending file name was opened
305 private synchronized IOException fillOpenTrace(IOException ioe, String name, boolean input) {
306 for(Map.Entry<Closeable,Exception> ent : openFileHandles.entrySet()) {
307 if (input && ent.getKey() instanceof MockIndexInputWrapper && ((MockIndexInputWrapper) ent.getKey()).name.equals(name)) {
308 ioe.initCause(ent.getValue());
310 } else if (!input && ent.getKey() instanceof MockIndexOutputWrapper && ((MockIndexOutputWrapper) ent.getKey()).name.equals(name)) {
311 ioe.initCause(ent.getValue());
318 private void maybeYield() {
319 if (randomState.nextBoolean()) {
324 private synchronized void deleteFile(String name, boolean forced) throws IOException {
327 maybeThrowDeterministicException();
329 if (crashed && !forced)
330 throw new IOException("cannot delete after crash");
332 if (unSyncedFiles.contains(name))
333 unSyncedFiles.remove(name);
334 if (!forced && noDeleteOpenFile) {
335 if (openFiles.containsKey(name)) {
336 openFilesDeleted.add(name);
337 throw fillOpenTrace(new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open: cannot delete"), name, true);
339 openFilesDeleted.remove(name);
342 delegate.deleteFile(name);
345 public synchronized Set<String> getOpenDeletedFiles() {
346 return new HashSet<String>(openFilesDeleted);
350 public synchronized IndexOutput createOutput(String name) throws IOException {
353 throw new IOException("cannot createOutput after crash");
356 if (preventDoubleWrite && createdFiles.contains(name) && !name.equals("segments.gen"))
357 throw new IOException("file \"" + name + "\" was already written to");
359 if (noDeleteOpenFile && openFiles.containsKey(name))
360 throw new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open: cannot overwrite");
363 throw new IOException("cannot createOutput after crash");
364 unSyncedFiles.add(name);
365 createdFiles.add(name);
367 if (delegate instanceof RAMDirectory) {
368 RAMDirectory ramdir = (RAMDirectory) delegate;
369 RAMFile file = new RAMFile(ramdir);
370 RAMFile existing = ramdir.fileMap.get(name);
372 // Enforce write once:
373 if (existing!=null && !name.equals("segments.gen") && preventDoubleWrite)
374 throw new IOException("file " + name + " already exists");
376 if (existing!=null) {
377 ramdir.sizeInBytes.getAndAdd(-existing.sizeInBytes);
378 existing.directory = null;
380 ramdir.fileMap.put(name, file);
384 //System.out.println(Thread.currentThread().getName() + ": MDW: create " + name);
385 IndexOutput io = new MockIndexOutputWrapper(this, delegate.createOutput(name), name);
386 addFileHandle(io, name, false);
387 openFilesForWrite.add(name);
389 // throttling REALLY slows down tests, so don't do it very often for SOMETIMES.
390 if (throttling == Throttling.ALWAYS ||
391 (throttling == Throttling.SOMETIMES && randomState.nextInt(50) == 0)) {
392 if (LuceneTestCase.VERBOSE) {
393 System.out.println("MockDirectoryWrapper: throttling indexOutput");
395 return throttledOutput.newFromDelegate(io);
401 private void addFileHandle(Closeable c, String name, boolean input) {
402 Integer v = openFiles.get(name);
404 v = Integer.valueOf(v.intValue()+1);
405 openFiles.put(name, v);
407 openFiles.put(name, Integer.valueOf(1));
410 openFileHandles.put(c, new RuntimeException("unclosed Index" + (input ? "Input" : "Output") + ": " + name));
414 public synchronized IndexInput openInput(String name) throws IOException {
416 if (!delegate.fileExists(name))
417 throw new FileNotFoundException(name);
419 // cannot open a file for input if it's still open for
420 // output, except for segments.gen and segments_N
421 if (openFilesForWrite.contains(name) && !name.startsWith("segments")) {
422 throw fillOpenTrace(new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open for writing"), name, false);
425 IndexInput ii = new MockIndexInputWrapper(this, name, delegate.openInput(name));
426 addFileHandle(ii, name, true);
430 /** Provided for testing purposes. Use sizeInBytes() instead. */
431 public synchronized final long getRecomputedSizeInBytes() throws IOException {
432 if (!(delegate instanceof RAMDirectory))
433 return sizeInBytes();
435 for(final RAMFile file: ((RAMDirectory)delegate).fileMap.values()) {
436 size += file.getSizeInBytes();
441 /** Like getRecomputedSizeInBytes(), but, uses actual file
442 * lengths rather than buffer allocations (which are
443 * quantized up to nearest
444 * RAMOutputStream.BUFFER_SIZE (now 1024) bytes.
447 public final synchronized long getRecomputedActualSizeInBytes() throws IOException {
448 if (!(delegate instanceof RAMDirectory))
449 return sizeInBytes();
451 for (final RAMFile file : ((RAMDirectory)delegate).fileMap.values())
457 public synchronized void close() throws IOException {
459 if (openFiles == null) {
460 openFiles = new HashMap<String,Integer>();
461 openFilesDeleted = new HashSet<String>();
463 if (noDeleteOpenFile && openFiles.size() > 0) {
464 // print the first one as its very verbose otherwise
465 Exception cause = null;
466 Iterator<Exception> stacktraces = openFileHandles.values().iterator();
467 if (stacktraces.hasNext())
468 cause = stacktraces.next();
469 // RuntimeException instead of IOException because
470 // super() does not throw IOException currently:
471 throw new RuntimeException("MockDirectoryWrapper: cannot close: there are still open files: " + openFiles, cause);
473 if (noDeleteOpenFile && openLocks.size() > 0) {
474 throw new RuntimeException("MockDirectoryWrapper: cannot close: there are still open locks: " + openLocks);
477 if (checkIndexOnClose && IndexReader.indexExists(this)) {
478 if (LuceneTestCase.VERBOSE) {
479 System.out.println("\nNOTE: MockDirectoryWrapper: now run CheckIndex");
481 _TestUtil.checkIndex(this);
486 private synchronized void removeOpenFile(Closeable c, String name) {
487 Integer v = openFiles.get(name);
488 // Could be null when crash() was called
490 if (v.intValue() == 1) {
491 openFiles.remove(name);
492 openFilesDeleted.remove(name);
494 v = Integer.valueOf(v.intValue()-1);
495 openFiles.put(name, v);
499 openFileHandles.remove(c);
502 public synchronized void removeIndexOutput(IndexOutput out, String name) {
503 openFilesForWrite.remove(name);
504 removeOpenFile(out, name);
507 public synchronized void removeIndexInput(IndexInput in, String name) {
508 removeOpenFile(in, name);
513 public synchronized boolean isOpen() {
518 * Objects that represent fail-able conditions. Objects of a derived
519 * class are created and registered with the mock directory. After
520 * register, each object will be invoked once for each first write
521 * of a file, giving the object a chance to throw an IOException.
523 public static class Failure {
525 * eval is called on the first write of every new file.
527 public void eval(MockDirectoryWrapper dir) throws IOException { }
530 * reset should set the state of the failure to its default
531 * (freshly constructed) state. Reset is convenient for tests
532 * that want to create one failure object and then reuse it in
533 * multiple cases. This, combined with the fact that Failure
534 * subclasses are often anonymous classes makes reset difficult to
537 * A typical example of use is
538 * Failure failure = new Failure() { ... };
540 * mock.failOn(failure.reset())
542 public Failure reset() { return this; }
544 protected boolean doFail;
546 public void setDoFail() {
550 public void clearDoFail() {
555 ArrayList<Failure> failures;
558 * add a Failure object to the list of objects to be evaluated
559 * at every potential failure point
561 synchronized public void failOn(Failure fail) {
562 if (failures == null) {
563 failures = new ArrayList<Failure>();
569 * Iterate through the failures list, giving each object a
570 * chance to throw an IOE
572 synchronized void maybeThrowDeterministicException() throws IOException {
573 if (failures != null) {
574 for(int i = 0; i < failures.size(); i++) {
575 failures.get(i).eval(this);
581 public synchronized String[] listAll() throws IOException {
583 return delegate.listAll();
587 public synchronized boolean fileExists(String name) throws IOException {
589 return delegate.fileExists(name);
593 public synchronized long fileModified(String name) throws IOException {
595 return delegate.fileModified(name);
600 /* @deprecated Lucene never uses this API; it will be
602 public synchronized void touchFile(String name) throws IOException {
604 delegate.touchFile(name);
608 public synchronized long fileLength(String name) throws IOException {
610 return delegate.fileLength(name);
614 public synchronized Lock makeLock(String name) {
616 return delegate.makeLock(name);
620 public synchronized void clearLock(String name) throws IOException {
622 delegate.clearLock(name);
626 public synchronized void setLockFactory(LockFactory lockFactory) throws IOException {
628 delegate.setLockFactory(lockFactory);
632 public synchronized LockFactory getLockFactory() {
634 return delegate.getLockFactory();
638 public synchronized String getLockID() {
640 return delegate.getLockID();
644 public synchronized void copy(Directory to, String src, String dest) throws IOException {
646 delegate.copy(to, src, dest);