1 package org.apache.lucene.store;
4 * Licensed to the Apache Software Foundation (ASF) under one or more
5 * contributor license agreements. See the NOTICE file distributed with
6 * this work for additional information regarding copyright ownership.
7 * The ASF licenses this file to You under the Apache License, Version 2.0
8 * (the "License"); you may not use this file except in compliance with
9 * the License. You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
20 import java.io.Closeable;
21 import java.io.FileNotFoundException;
22 import java.io.IOException;
23 import java.util.ArrayList;
24 import java.util.Collection;
25 import java.util.Collections;
26 import java.util.HashMap;
27 import java.util.HashSet;
28 import java.util.IdentityHashMap;
29 import java.util.Iterator;
31 import java.util.Random;
34 import org.apache.lucene.index.IndexReader;
35 import org.apache.lucene.util.LuceneTestCase;
36 import org.apache.lucene.util.ThrottledIndexOutput;
37 import org.apache.lucene.util._TestUtil;
40 * This is a Directory Wrapper that adds methods
41 * intended to be used only by unit tests.
42 * It also adds a number of features useful for testing:
44 * <li> Instances created by {@link LuceneTestCase#newDirectory()} are tracked
45 * to ensure they are closed by the test.
46 * <li> When a MockDirectoryWrapper is closed, it will throw an exception if
47 * it has any open files against it (with a stacktrace indicating where
48 * they were opened from).
49 * <li> When a MockDirectoryWrapper is closed, it runs CheckIndex to test if
50 * the index was corrupted.
51 * <li> MockDirectoryWrapper simulates some "features" of Windows, such as
52 * refusing to write/delete to open files.
56 public class MockDirectoryWrapper extends Directory {
57 final Directory delegate;
60 // Max actual bytes used. This is set by MockRAMOutputStream:
62 double randomIOExceptionRate;
64 boolean noDeleteOpenFile = true;
65 boolean preventDoubleWrite = true;
66 boolean checkIndexOnClose = true;
67 boolean trackDiskUsage = false;
68 private Set<String> unSyncedFiles;
69 private Set<String> createdFiles;
70 private Set<String> openFilesForWrite = new HashSet<String>();
71 Set<String> openLocks = Collections.synchronizedSet(new HashSet<String>());
72 volatile boolean crashed;
73 private ThrottledIndexOutput throttledOutput;
74 private Throttling throttling = Throttling.SOMETIMES;
76 // use this for tracking files for crash.
77 // additionally: provides debugging information in case you leave one open
78 private Map<Closeable,Exception> openFileHandles = Collections.synchronizedMap(new IdentityHashMap<Closeable,Exception>());
80 // NOTE: we cannot initialize the Map here due to the
81 // order in which our constructor actually does this
82 // member initialization vs when it calls super. It seems
83 // like super is called, then our members are initialized:
84 private Map<String,Integer> openFiles;
86 // Only tracked if noDeleteOpenFile is true: if an attempt
87 // is made to delete an open file, we enroll it here.
88 private Set<String> openFilesDeleted;
90 private synchronized void init() {
91 if (openFiles == null) {
92 openFiles = new HashMap<String,Integer>();
93 openFilesDeleted = new HashSet<String>();
96 if (createdFiles == null)
97 createdFiles = new HashSet<String>();
98 if (unSyncedFiles == null)
99 unSyncedFiles = new HashSet<String>();
102 public MockDirectoryWrapper(Random random, Directory delegate) {
103 this.delegate = delegate;
104 // must make a private random since our methods are
105 // called from different threads; else test failures may
106 // not be reproducible from the original seed
107 this.randomState = new Random(random.nextInt());
108 this.throttledOutput = new ThrottledIndexOutput(ThrottledIndexOutput
109 .mBitsToBytes(40 + randomState.nextInt(10)), 5 + randomState.nextInt(5), null);
110 // force wrapping of lockfactory
112 setLockFactory(new MockLockFactoryWrapper(this, delegate.getLockFactory()));
113 } catch (IOException e) {
114 throw new RuntimeException(e);
119 public void setTrackDiskUsage(boolean v) {
123 /** If set to true, we throw an IOException if the same
124 * file is opened by createOutput, ever. */
125 public void setPreventDoubleWrite(boolean value) {
126 preventDoubleWrite = value;
131 public void sync(String name) throws IOException {
133 maybeThrowDeterministicException();
135 throw new IOException("cannot sync after crash");
136 unSyncedFiles.remove(name);
140 public static enum Throttling {
141 /** always emulate a slow hard disk. could be very slow! */
143 /** sometimes (2% of the time) emulate a slow hard disk. */
145 /** never throttle output */
149 public void setThrottling(Throttling throttling) {
150 this.throttling = throttling;
154 public synchronized void sync(Collection<String> names) throws IOException {
156 for (String name : names)
157 maybeThrowDeterministicException();
159 throw new IOException("cannot sync after crash");
160 unSyncedFiles.removeAll(names);
161 delegate.sync(names);
165 public String toString() {
166 // NOTE: do not maybeYield here, since it consumes
167 // randomness and can thus (unexpectedly during
168 // debugging) change the behavior of a seed
170 return "MockDirWrapper(" + delegate + ")";
173 public synchronized final long sizeInBytes() throws IOException {
174 if (delegate instanceof RAMDirectory)
175 return ((RAMDirectory) delegate).sizeInBytes();
179 for (String file : delegate.listAll())
180 size += delegate.fileLength(file);
185 /** Simulates a crash of OS or machine by overwriting
187 public synchronized void crash() throws IOException {
189 openFiles = new HashMap<String,Integer>();
190 openFilesForWrite = new HashSet<String>();
191 openFilesDeleted = new HashSet<String>();
192 Iterator<String> it = unSyncedFiles.iterator();
193 unSyncedFiles = new HashSet<String>();
194 // first force-close all files, so we can corrupt on windows etc.
195 // clone the file map, as these guys want to remove themselves on close.
196 Map<Closeable,Exception> m = new IdentityHashMap<Closeable,Exception>(openFileHandles);
197 for (Closeable f : m.keySet())
200 } catch (Exception ignored) {}
203 while(it.hasNext()) {
204 String name = it.next();
205 if (count % 3 == 0) {
206 deleteFile(name, true);
207 } else if (count % 3 == 1) {
208 // Zero out file entirely
209 long length = fileLength(name);
210 byte[] zeroes = new byte[256];
212 IndexOutput out = delegate.createOutput(name);
213 while(upto < length) {
214 final int limit = (int) Math.min(length-upto, zeroes.length);
215 out.writeBytes(zeroes, 0, limit);
219 } else if (count % 3 == 2) {
220 // Truncate the file:
221 IndexOutput out = delegate.createOutput(name);
222 out.setLength(fileLength(name)/2);
229 public synchronized void clearCrash() throws IOException {
234 public void setMaxSizeInBytes(long maxSize) {
235 this.maxSize = maxSize;
237 public long getMaxSizeInBytes() {
242 * Returns the peek actual storage used (bytes) in this
245 public long getMaxUsedSizeInBytes() {
246 return this.maxUsedSize;
248 public void resetMaxUsedSizeInBytes() throws IOException {
249 this.maxUsedSize = getRecomputedActualSizeInBytes();
253 * Emulate windows whereby deleting an open file is not
254 * allowed (raise IOException).
256 public void setNoDeleteOpenFile(boolean value) {
257 this.noDeleteOpenFile = value;
259 public boolean getNoDeleteOpenFile() {
260 return noDeleteOpenFile;
264 * Set whether or not checkindex should be run
267 public void setCheckIndexOnClose(boolean value) {
268 this.checkIndexOnClose = value;
271 public boolean getCheckIndexOnClose() {
272 return checkIndexOnClose;
275 * If 0.0, no exceptions will be thrown. Else this should
276 * be a double 0.0 - 1.0. We will randomly throw an
277 * IOException on the first write to an OutputStream based
278 * on this probability.
280 public void setRandomIOExceptionRate(double rate) {
281 randomIOExceptionRate = rate;
283 public double getRandomIOExceptionRate() {
284 return randomIOExceptionRate;
287 void maybeThrowIOException() throws IOException {
288 if (randomIOExceptionRate > 0.0) {
289 int number = Math.abs(randomState.nextInt() % 1000);
290 if (number < randomIOExceptionRate*1000) {
291 if (LuceneTestCase.VERBOSE) {
292 System.out.println(Thread.currentThread().getName() + ": MockDirectoryWrapper: now throw random exception");
293 new Throwable().printStackTrace(System.out);
295 throw new IOException("a random IOException");
301 public synchronized void deleteFile(String name) throws IOException {
303 deleteFile(name, false);
306 // sets the cause of the incoming ioe to be the stack
307 // trace when the offending file name was opened
308 private synchronized IOException fillOpenTrace(IOException ioe, String name, boolean input) {
309 for(Map.Entry<Closeable,Exception> ent : openFileHandles.entrySet()) {
310 if (input && ent.getKey() instanceof MockIndexInputWrapper && ((MockIndexInputWrapper) ent.getKey()).name.equals(name)) {
311 ioe.initCause(ent.getValue());
313 } else if (!input && ent.getKey() instanceof MockIndexOutputWrapper && ((MockIndexOutputWrapper) ent.getKey()).name.equals(name)) {
314 ioe.initCause(ent.getValue());
321 private void maybeYield() {
322 if (randomState.nextBoolean()) {
327 private synchronized void deleteFile(String name, boolean forced) throws IOException {
330 maybeThrowDeterministicException();
332 if (crashed && !forced)
333 throw new IOException("cannot delete after crash");
335 if (unSyncedFiles.contains(name))
336 unSyncedFiles.remove(name);
337 if (!forced && noDeleteOpenFile) {
338 if (openFiles.containsKey(name)) {
339 openFilesDeleted.add(name);
340 throw fillOpenTrace(new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open: cannot delete"), name, true);
342 openFilesDeleted.remove(name);
345 delegate.deleteFile(name);
348 public synchronized Set<String> getOpenDeletedFiles() {
349 return new HashSet<String>(openFilesDeleted);
353 public synchronized IndexOutput createOutput(String name) throws IOException {
356 throw new IOException("cannot createOutput after crash");
359 if (preventDoubleWrite && createdFiles.contains(name) && !name.equals("segments.gen"))
360 throw new IOException("file \"" + name + "\" was already written to");
362 if (noDeleteOpenFile && openFiles.containsKey(name))
363 throw new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open: cannot overwrite");
366 throw new IOException("cannot createOutput after crash");
367 unSyncedFiles.add(name);
368 createdFiles.add(name);
370 if (delegate instanceof RAMDirectory) {
371 RAMDirectory ramdir = (RAMDirectory) delegate;
372 RAMFile file = new RAMFile(ramdir);
373 RAMFile existing = ramdir.fileMap.get(name);
375 // Enforce write once:
376 if (existing!=null && !name.equals("segments.gen") && preventDoubleWrite)
377 throw new IOException("file " + name + " already exists");
379 if (existing!=null) {
380 ramdir.sizeInBytes.getAndAdd(-existing.sizeInBytes);
381 existing.directory = null;
383 ramdir.fileMap.put(name, file);
387 //System.out.println(Thread.currentThread().getName() + ": MDW: create " + name);
388 IndexOutput io = new MockIndexOutputWrapper(this, delegate.createOutput(name), name);
389 addFileHandle(io, name, false);
390 openFilesForWrite.add(name);
392 // throttling REALLY slows down tests, so don't do it very often for SOMETIMES.
393 if (throttling == Throttling.ALWAYS ||
394 (throttling == Throttling.SOMETIMES && randomState.nextInt(50) == 0)) {
395 if (LuceneTestCase.VERBOSE) {
396 System.out.println("MockDirectoryWrapper: throttling indexOutput");
398 return throttledOutput.newFromDelegate(io);
404 synchronized void addFileHandle(Closeable c, String name, boolean input) {
405 Integer v = openFiles.get(name);
407 v = Integer.valueOf(v.intValue()+1);
408 openFiles.put(name, v);
410 openFiles.put(name, Integer.valueOf(1));
413 openFileHandles.put(c, new RuntimeException("unclosed Index" + (input ? "Input" : "Output") + ": " + name));
417 public synchronized IndexInput openInput(String name) throws IOException {
419 if (!delegate.fileExists(name))
420 throw new FileNotFoundException(name);
422 // cannot open a file for input if it's still open for
423 // output, except for segments.gen and segments_N
424 if (openFilesForWrite.contains(name) && !name.startsWith("segments")) {
425 throw fillOpenTrace(new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open for writing"), name, false);
428 IndexInput ii = new MockIndexInputWrapper(this, name, delegate.openInput(name));
429 addFileHandle(ii, name, true);
433 /** Provided for testing purposes. Use sizeInBytes() instead. */
434 public synchronized final long getRecomputedSizeInBytes() throws IOException {
435 if (!(delegate instanceof RAMDirectory))
436 return sizeInBytes();
438 for(final RAMFile file: ((RAMDirectory)delegate).fileMap.values()) {
439 size += file.getSizeInBytes();
444 /** Like getRecomputedSizeInBytes(), but, uses actual file
445 * lengths rather than buffer allocations (which are
446 * quantized up to nearest
447 * RAMOutputStream.BUFFER_SIZE (now 1024) bytes.
450 public final synchronized long getRecomputedActualSizeInBytes() throws IOException {
451 if (!(delegate instanceof RAMDirectory))
452 return sizeInBytes();
454 for (final RAMFile file : ((RAMDirectory)delegate).fileMap.values())
460 public synchronized void close() throws IOException {
462 if (openFiles == null) {
463 openFiles = new HashMap<String,Integer>();
464 openFilesDeleted = new HashSet<String>();
466 if (noDeleteOpenFile && openFiles.size() > 0) {
467 // print the first one as its very verbose otherwise
468 Exception cause = null;
469 Iterator<Exception> stacktraces = openFileHandles.values().iterator();
470 if (stacktraces.hasNext())
471 cause = stacktraces.next();
472 // RuntimeException instead of IOException because
473 // super() does not throw IOException currently:
474 throw new RuntimeException("MockDirectoryWrapper: cannot close: there are still open files: " + openFiles, cause);
476 if (noDeleteOpenFile && openLocks.size() > 0) {
477 throw new RuntimeException("MockDirectoryWrapper: cannot close: there are still open locks: " + openLocks);
480 if (checkIndexOnClose && IndexReader.indexExists(this)) {
481 if (LuceneTestCase.VERBOSE) {
482 System.out.println("\nNOTE: MockDirectoryWrapper: now run CheckIndex");
484 _TestUtil.checkIndex(this);
489 synchronized void removeOpenFile(Closeable c, String name) {
490 Integer v = openFiles.get(name);
491 // Could be null when crash() was called
493 if (v.intValue() == 1) {
494 openFiles.remove(name);
495 openFilesDeleted.remove(name);
497 v = Integer.valueOf(v.intValue()-1);
498 openFiles.put(name, v);
502 openFileHandles.remove(c);
505 public synchronized void removeIndexOutput(IndexOutput out, String name) {
506 openFilesForWrite.remove(name);
507 removeOpenFile(out, name);
510 public synchronized void removeIndexInput(IndexInput in, String name) {
511 removeOpenFile(in, name);
516 public synchronized boolean isOpen() {
521 * Objects that represent fail-able conditions. Objects of a derived
522 * class are created and registered with the mock directory. After
523 * register, each object will be invoked once for each first write
524 * of a file, giving the object a chance to throw an IOException.
526 public static class Failure {
528 * eval is called on the first write of every new file.
530 public void eval(MockDirectoryWrapper dir) throws IOException { }
533 * reset should set the state of the failure to its default
534 * (freshly constructed) state. Reset is convenient for tests
535 * that want to create one failure object and then reuse it in
536 * multiple cases. This, combined with the fact that Failure
537 * subclasses are often anonymous classes makes reset difficult to
540 * A typical example of use is
541 * Failure failure = new Failure() { ... };
543 * mock.failOn(failure.reset())
545 public Failure reset() { return this; }
547 protected boolean doFail;
549 public void setDoFail() {
553 public void clearDoFail() {
558 ArrayList<Failure> failures;
561 * add a Failure object to the list of objects to be evaluated
562 * at every potential failure point
564 synchronized public void failOn(Failure fail) {
565 if (failures == null) {
566 failures = new ArrayList<Failure>();
572 * Iterate through the failures list, giving each object a
573 * chance to throw an IOE
575 synchronized void maybeThrowDeterministicException() throws IOException {
576 if (failures != null) {
577 for(int i = 0; i < failures.size(); i++) {
578 failures.get(i).eval(this);
584 public synchronized String[] listAll() throws IOException {
586 return delegate.listAll();
590 public synchronized boolean fileExists(String name) throws IOException {
592 return delegate.fileExists(name);
596 public synchronized long fileModified(String name) throws IOException {
598 return delegate.fileModified(name);
603 /* @deprecated Lucene never uses this API; it will be
605 public synchronized void touchFile(String name) throws IOException {
607 delegate.touchFile(name);
611 public synchronized long fileLength(String name) throws IOException {
613 return delegate.fileLength(name);
617 public synchronized Lock makeLock(String name) {
619 return delegate.makeLock(name);
623 public synchronized void clearLock(String name) throws IOException {
625 delegate.clearLock(name);
629 public synchronized void setLockFactory(LockFactory lockFactory) throws IOException {
631 delegate.setLockFactory(lockFactory);
635 public synchronized LockFactory getLockFactory() {
637 return delegate.getLockFactory();
641 public synchronized String getLockID() {
643 return delegate.getLockID();
647 public synchronized void copy(Directory to, String src, String dest) throws IOException {
649 delegate.copy(to, src, dest);