--- /dev/null
+package org.apache.lucene.store;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.Closeable;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.IdentityHashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.ThrottledIndexOutput;
+import org.apache.lucene.util._TestUtil;
+
+/**
+ * This is a Directory Wrapper that adds methods
+ * intended to be used only by unit tests.
+ * It also adds a number of features useful for testing:
+ * <ul>
+ * <li> Instances created by {@link LuceneTestCase#newDirectory()} are tracked
+ * to ensure they are closed by the test.
+ * <li> When a MockDirectoryWrapper is closed, it will throw an exception if
+ * it has any open files against it (with a stacktrace indicating where
+ * they were opened from).
+ * <li> When a MockDirectoryWrapper is closed, it runs CheckIndex to test if
+ * the index was corrupted.
+ * <li> MockDirectoryWrapper simulates some "features" of Windows, such as
+ * refusing to write/delete to open files.
+ * </ul>
+ */
+
+public class MockDirectoryWrapper extends Directory {
+ final Directory delegate;
+ long maxSize;
+
+ // Max actual bytes used. This is set by MockRAMOutputStream:
+ long maxUsedSize;
+ double randomIOExceptionRate;
+ Random randomState;
+ boolean noDeleteOpenFile = true;
+ boolean preventDoubleWrite = true;
+ boolean checkIndexOnClose = true;
+ boolean trackDiskUsage = false;
+ private Set<String> unSyncedFiles;
+ private Set<String> createdFiles;
+ private Set<String> openFilesForWrite = new HashSet<String>();
+ Set<String> openLocks = Collections.synchronizedSet(new HashSet<String>());
+ volatile boolean crashed;
+ private ThrottledIndexOutput throttledOutput;
+ private Throttling throttling = Throttling.SOMETIMES;
+
+ final AtomicInteger inputCloneCount = new AtomicInteger();
+
+ // use this for tracking files for crash.
+ // additionally: provides debugging information in case you leave one open
+ private Map<Closeable,Exception> openFileHandles = Collections.synchronizedMap(new IdentityHashMap<Closeable,Exception>());
+
+ // NOTE: we cannot initialize the Map here due to the
+ // order in which our constructor actually does this
+ // member initialization vs when it calls super. It seems
+ // like super is called, then our members are initialized:
+ private Map<String,Integer> openFiles;
+
+ // Only tracked if noDeleteOpenFile is true: if an attempt
+ // is made to delete an open file, we enroll it here.
+ private Set<String> openFilesDeleted;
+
+ private synchronized void init() {
+ if (openFiles == null) {
+ openFiles = new HashMap<String,Integer>();
+ openFilesDeleted = new HashSet<String>();
+ }
+
+ if (createdFiles == null)
+ createdFiles = new HashSet<String>();
+ if (unSyncedFiles == null)
+ unSyncedFiles = new HashSet<String>();
+ }
+
+ public MockDirectoryWrapper(Random random, Directory delegate) {
+ this.delegate = delegate;
+ // must make a private random since our methods are
+ // called from different threads; else test failures may
+ // not be reproducible from the original seed
+ this.randomState = new Random(random.nextInt());
+ this.throttledOutput = new ThrottledIndexOutput(ThrottledIndexOutput
+ .mBitsToBytes(40 + randomState.nextInt(10)), 5 + randomState.nextInt(5), null);
+ // force wrapping of lockfactory
+ try {
+ setLockFactory(new MockLockFactoryWrapper(this, delegate.getLockFactory()));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ init();
+ }
+
+ public int getInputCloneCount() {
+ return inputCloneCount.get();
+ }
+
+ public void setTrackDiskUsage(boolean v) {
+ trackDiskUsage = v;
+ }
+
+ /** If set to true, we throw an IOException if the same
+ * file is opened by createOutput, ever. */
+ public void setPreventDoubleWrite(boolean value) {
+ preventDoubleWrite = value;
+ }
+
+ @Deprecated
+ @Override
+ public void sync(String name) throws IOException {
+ maybeYield();
+ maybeThrowDeterministicException();
+ if (crashed)
+ throw new IOException("cannot sync after crash");
+ unSyncedFiles.remove(name);
+ delegate.sync(name);
+ }
+
+ public static enum Throttling {
+ /** always emulate a slow hard disk. could be very slow! */
+ ALWAYS,
+ /** sometimes (2% of the time) emulate a slow hard disk. */
+ SOMETIMES,
+ /** never throttle output */
+ NEVER
+ }
+
+ public void setThrottling(Throttling throttling) {
+ this.throttling = throttling;
+ }
+
+ @Override
+ public synchronized void sync(Collection<String> names) throws IOException {
+ maybeYield();
+ maybeThrowDeterministicException();
+ if (crashed)
+ throw new IOException("cannot sync after crash");
+ unSyncedFiles.removeAll(names);
+ delegate.sync(names);
+ }
+
+ @Override
+ public String toString() {
+ // NOTE: do not maybeYield here, since it consumes
+ // randomness and can thus (unexpectedly during
+ // debugging) change the behavior of a seed
+ // maybeYield();
+ return "MockDirWrapper(" + delegate + ")";
+ }
+
+ public synchronized final long sizeInBytes() throws IOException {
+ if (delegate instanceof RAMDirectory)
+ return ((RAMDirectory) delegate).sizeInBytes();
+ else {
+ // hack
+ long size = 0;
+ for (String file : delegate.listAll())
+ size += delegate.fileLength(file);
+ return size;
+ }
+ }
+
+ /** Simulates a crash of OS or machine by overwriting
+ * unsynced files. */
+ public synchronized void crash() throws IOException {
+ crashed = true;
+ openFiles = new HashMap<String,Integer>();
+ openFilesForWrite = new HashSet<String>();
+ openFilesDeleted = new HashSet<String>();
+ Iterator<String> it = unSyncedFiles.iterator();
+ unSyncedFiles = new HashSet<String>();
+ // first force-close all files, so we can corrupt on windows etc.
+ // clone the file map, as these guys want to remove themselves on close.
+ Map<Closeable,Exception> m = new IdentityHashMap<Closeable,Exception>(openFileHandles);
+ for (Closeable f : m.keySet())
+ try {
+ f.close();
+ } catch (Exception ignored) {}
+
+ int count = 0;
+ while(it.hasNext()) {
+ String name = it.next();
+ if (count % 3 == 0) {
+ deleteFile(name, true);
+ } else if (count % 3 == 1) {
+ // Zero out file entirely
+ long length = fileLength(name);
+ byte[] zeroes = new byte[256];
+ long upto = 0;
+ IndexOutput out = delegate.createOutput(name);
+ while(upto < length) {
+ final int limit = (int) Math.min(length-upto, zeroes.length);
+ out.writeBytes(zeroes, 0, limit);
+ upto += limit;
+ }
+ out.close();
+ } else if (count % 3 == 2) {
+ // Truncate the file:
+ IndexOutput out = delegate.createOutput(name);
+ out.setLength(fileLength(name)/2);
+ out.close();
+ }
+ count++;
+ }
+ }
+
+ public synchronized void clearCrash() throws IOException {
+ crashed = false;
+ openLocks.clear();
+ }
+
+ public void setMaxSizeInBytes(long maxSize) {
+ this.maxSize = maxSize;
+ }
+ public long getMaxSizeInBytes() {
+ return this.maxSize;
+ }
+
+ /**
+ * Returns the peek actual storage used (bytes) in this
+ * directory.
+ */
+ public long getMaxUsedSizeInBytes() {
+ return this.maxUsedSize;
+ }
+ public void resetMaxUsedSizeInBytes() throws IOException {
+ this.maxUsedSize = getRecomputedActualSizeInBytes();
+ }
+
+ /**
+ * Emulate windows whereby deleting an open file is not
+ * allowed (raise IOException).
+ */
+ public void setNoDeleteOpenFile(boolean value) {
+ this.noDeleteOpenFile = value;
+ }
+ public boolean getNoDeleteOpenFile() {
+ return noDeleteOpenFile;
+ }
+
+ /**
+ * Set whether or not checkindex should be run
+ * on close
+ */
+ public void setCheckIndexOnClose(boolean value) {
+ this.checkIndexOnClose = value;
+ }
+
+ public boolean getCheckIndexOnClose() {
+ return checkIndexOnClose;
+ }
+ /**
+ * If 0.0, no exceptions will be thrown. Else this should
+ * be a double 0.0 - 1.0. We will randomly throw an
+ * IOException on the first write to an OutputStream based
+ * on this probability.
+ */
+ public void setRandomIOExceptionRate(double rate) {
+ randomIOExceptionRate = rate;
+ }
+ public double getRandomIOExceptionRate() {
+ return randomIOExceptionRate;
+ }
+
+ void maybeThrowIOException() throws IOException {
+ if (randomIOExceptionRate > 0.0) {
+ int number = Math.abs(randomState.nextInt() % 1000);
+ if (number < randomIOExceptionRate*1000) {
+ if (LuceneTestCase.VERBOSE) {
+ System.out.println(Thread.currentThread().getName() + ": MockDirectoryWrapper: now throw random exception");
+ new Throwable().printStackTrace(System.out);
+ }
+ throw new IOException("a random IOException");
+ }
+ }
+ }
+
+ @Override
+ public synchronized void deleteFile(String name) throws IOException {
+ maybeYield();
+ deleteFile(name, false);
+ }
+
+ // sets the cause of the incoming ioe to be the stack
+ // trace when the offending file name was opened
+ private synchronized IOException fillOpenTrace(IOException ioe, String name, boolean input) {
+ for(Map.Entry<Closeable,Exception> ent : openFileHandles.entrySet()) {
+ if (input && ent.getKey() instanceof MockIndexInputWrapper && ((MockIndexInputWrapper) ent.getKey()).name.equals(name)) {
+ ioe.initCause(ent.getValue());
+ break;
+ } else if (!input && ent.getKey() instanceof MockIndexOutputWrapper && ((MockIndexOutputWrapper) ent.getKey()).name.equals(name)) {
+ ioe.initCause(ent.getValue());
+ break;
+ }
+ }
+ return ioe;
+ }
+
+ private void maybeYield() {
+ if (randomState.nextBoolean()) {
+ Thread.yield();
+ }
+ }
+
+ private synchronized void deleteFile(String name, boolean forced) throws IOException {
+ maybeYield();
+
+ maybeThrowDeterministicException();
+
+ if (crashed && !forced)
+ throw new IOException("cannot delete after crash");
+
+ if (unSyncedFiles.contains(name))
+ unSyncedFiles.remove(name);
+ if (!forced && noDeleteOpenFile) {
+ if (openFiles.containsKey(name)) {
+ openFilesDeleted.add(name);
+ throw fillOpenTrace(new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open: cannot delete"), name, true);
+ } else {
+ openFilesDeleted.remove(name);
+ }
+ }
+ delegate.deleteFile(name);
+ }
+
+ public synchronized Set<String> getOpenDeletedFiles() {
+ return new HashSet<String>(openFilesDeleted);
+ }
+
+ private boolean failOnCreateOutput = true;
+
+ public void setFailOnCreateOutput(boolean v) {
+ failOnCreateOutput = v;
+ }
+
+ @Override
+ public synchronized IndexOutput createOutput(String name) throws IOException {
+ maybeYield();
+ if (failOnCreateOutput) {
+ maybeThrowDeterministicException();
+ }
+ if (crashed)
+ throw new IOException("cannot createOutput after crash");
+ init();
+ synchronized(this) {
+ if (preventDoubleWrite && createdFiles.contains(name) && !name.equals("segments.gen"))
+ throw new IOException("file \"" + name + "\" was already written to");
+ }
+ if (noDeleteOpenFile && openFiles.containsKey(name))
+ throw new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open: cannot overwrite");
+
+ if (crashed)
+ throw new IOException("cannot createOutput after crash");
+ unSyncedFiles.add(name);
+ createdFiles.add(name);
+
+ if (delegate instanceof RAMDirectory) {
+ RAMDirectory ramdir = (RAMDirectory) delegate;
+ RAMFile file = new RAMFile(ramdir);
+ RAMFile existing = ramdir.fileMap.get(name);
+
+ // Enforce write once:
+ if (existing!=null && !name.equals("segments.gen") && preventDoubleWrite)
+ throw new IOException("file " + name + " already exists");
+ else {
+ if (existing!=null) {
+ ramdir.sizeInBytes.getAndAdd(-existing.sizeInBytes);
+ existing.directory = null;
+ }
+ ramdir.fileMap.put(name, file);
+ }
+ }
+
+ //System.out.println(Thread.currentThread().getName() + ": MDW: create " + name);
+ IndexOutput io = new MockIndexOutputWrapper(this, delegate.createOutput(name), name);
+ addFileHandle(io, name, false);
+ openFilesForWrite.add(name);
+
+ // throttling REALLY slows down tests, so don't do it very often for SOMETIMES.
+ if (throttling == Throttling.ALWAYS ||
+ (throttling == Throttling.SOMETIMES && randomState.nextInt(50) == 0)) {
+ if (LuceneTestCase.VERBOSE) {
+ System.out.println("MockDirectoryWrapper: throttling indexOutput");
+ }
+ return throttledOutput.newFromDelegate(io);
+ } else {
+ return io;
+ }
+ }
+
+ synchronized void addFileHandle(Closeable c, String name, boolean input) {
+ Integer v = openFiles.get(name);
+ if (v != null) {
+ v = Integer.valueOf(v.intValue()+1);
+ openFiles.put(name, v);
+ } else {
+ openFiles.put(name, Integer.valueOf(1));
+ }
+
+ openFileHandles.put(c, new RuntimeException("unclosed Index" + (input ? "Input" : "Output") + ": " + name));
+ }
+
+ private boolean failOnOpenInput = true;
+
+ public void setFailOnOpenInput(boolean v) {
+ failOnOpenInput = v;
+ }
+
+ @Override
+ public synchronized IndexInput openInput(String name) throws IOException {
+ maybeYield();
+ if (failOnOpenInput) {
+ maybeThrowDeterministicException();
+ }
+ if (!delegate.fileExists(name))
+ throw new FileNotFoundException(name);
+
+ // cannot open a file for input if it's still open for
+ // output, except for segments.gen and segments_N
+ if (openFilesForWrite.contains(name) && !name.startsWith("segments")) {
+ throw fillOpenTrace(new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open for writing"), name, false);
+ }
+
+ IndexInput ii = new MockIndexInputWrapper(this, name, delegate.openInput(name));
+ addFileHandle(ii, name, true);
+ return ii;
+ }
+
+ /** Provided for testing purposes. Use sizeInBytes() instead. */
+ public synchronized final long getRecomputedSizeInBytes() throws IOException {
+ if (!(delegate instanceof RAMDirectory))
+ return sizeInBytes();
+ long size = 0;
+ for(final RAMFile file: ((RAMDirectory)delegate).fileMap.values()) {
+ size += file.getSizeInBytes();
+ }
+ return size;
+ }
+
+ /** Like getRecomputedSizeInBytes(), but, uses actual file
+ * lengths rather than buffer allocations (which are
+ * quantized up to nearest
+ * RAMOutputStream.BUFFER_SIZE (now 1024) bytes.
+ */
+
+ public final synchronized long getRecomputedActualSizeInBytes() throws IOException {
+ if (!(delegate instanceof RAMDirectory))
+ return sizeInBytes();
+ long size = 0;
+ for (final RAMFile file : ((RAMDirectory)delegate).fileMap.values())
+ size += file.length;
+ return size;
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ maybeYield();
+ if (openFiles == null) {
+ openFiles = new HashMap<String,Integer>();
+ openFilesDeleted = new HashSet<String>();
+ }
+ if (noDeleteOpenFile && openFiles.size() > 0) {
+ // print the first one as its very verbose otherwise
+ Exception cause = null;
+ Iterator<Exception> stacktraces = openFileHandles.values().iterator();
+ if (stacktraces.hasNext())
+ cause = stacktraces.next();
+ // RuntimeException instead of IOException because
+ // super() does not throw IOException currently:
+ throw new RuntimeException("MockDirectoryWrapper: cannot close: there are still open files: " + openFiles, cause);
+ }
+ if (noDeleteOpenFile && openLocks.size() > 0) {
+ throw new RuntimeException("MockDirectoryWrapper: cannot close: there are still open locks: " + openLocks);
+ }
+ open = false;
+ if (checkIndexOnClose && IndexReader.indexExists(this)) {
+ if (LuceneTestCase.VERBOSE) {
+ System.out.println("\nNOTE: MockDirectoryWrapper: now run CheckIndex");
+ }
+ _TestUtil.checkIndex(this);
+ }
+ delegate.close();
+ }
+
+ synchronized void removeOpenFile(Closeable c, String name) {
+ Integer v = openFiles.get(name);
+ // Could be null when crash() was called
+ if (v != null) {
+ if (v.intValue() == 1) {
+ openFiles.remove(name);
+ openFilesDeleted.remove(name);
+ } else {
+ v = Integer.valueOf(v.intValue()-1);
+ openFiles.put(name, v);
+ }
+ }
+
+ openFileHandles.remove(c);
+ }
+
+ public synchronized void removeIndexOutput(IndexOutput out, String name) {
+ openFilesForWrite.remove(name);
+ removeOpenFile(out, name);
+ }
+
+ public synchronized void removeIndexInput(IndexInput in, String name) {
+ removeOpenFile(in, name);
+ }
+
+ boolean open = true;
+
+ public synchronized boolean isOpen() {
+ return open;
+ }
+
+ /**
+ * Objects that represent fail-able conditions. Objects of a derived
+ * class are created and registered with the mock directory. After
+ * register, each object will be invoked once for each first write
+ * of a file, giving the object a chance to throw an IOException.
+ */
+ public static class Failure {
+ /**
+ * eval is called on the first write of every new file.
+ */
+ public void eval(MockDirectoryWrapper dir) throws IOException { }
+
+ /**
+ * reset should set the state of the failure to its default
+ * (freshly constructed) state. Reset is convenient for tests
+ * that want to create one failure object and then reuse it in
+ * multiple cases. This, combined with the fact that Failure
+ * subclasses are often anonymous classes makes reset difficult to
+ * do otherwise.
+ *
+ * A typical example of use is
+ * Failure failure = new Failure() { ... };
+ * ...
+ * mock.failOn(failure.reset())
+ */
+ public Failure reset() { return this; }
+
+ protected boolean doFail;
+
+ public void setDoFail() {
+ doFail = true;
+ }
+
+ public void clearDoFail() {
+ doFail = false;
+ }
+ }
+
+ ArrayList<Failure> failures;
+
+ /**
+ * add a Failure object to the list of objects to be evaluated
+ * at every potential failure point
+ */
+ synchronized public void failOn(Failure fail) {
+ if (failures == null) {
+ failures = new ArrayList<Failure>();
+ }
+ failures.add(fail);
+ }
+
+ /**
+ * Iterate through the failures list, giving each object a
+ * chance to throw an IOE
+ */
+ synchronized void maybeThrowDeterministicException() throws IOException {
+ if (failures != null) {
+ for(int i = 0; i < failures.size(); i++) {
+ failures.get(i).eval(this);
+ }
+ }
+ }
+
+ @Override
+ public synchronized String[] listAll() throws IOException {
+ maybeYield();
+ return delegate.listAll();
+ }
+
+ @Override
+ public synchronized boolean fileExists(String name) throws IOException {
+ maybeYield();
+ return delegate.fileExists(name);
+ }
+
+ @Override
+ public synchronized long fileModified(String name) throws IOException {
+ maybeYield();
+ return delegate.fileModified(name);
+ }
+
+ @Override
+ @Deprecated
+ /* @deprecated Lucene never uses this API; it will be
+ * removed in 4.0. */
+ public synchronized void touchFile(String name) throws IOException {
+ maybeYield();
+ delegate.touchFile(name);
+ }
+
+ @Override
+ public synchronized long fileLength(String name) throws IOException {
+ maybeYield();
+ return delegate.fileLength(name);
+ }
+
+ @Override
+ public synchronized Lock makeLock(String name) {
+ maybeYield();
+ return delegate.makeLock(name);
+ }
+
+ @Override
+ public synchronized void clearLock(String name) throws IOException {
+ maybeYield();
+ delegate.clearLock(name);
+ }
+
+ @Override
+ public synchronized void setLockFactory(LockFactory lockFactory) throws IOException {
+ maybeYield();
+ delegate.setLockFactory(lockFactory);
+ }
+
+ @Override
+ public synchronized LockFactory getLockFactory() {
+ maybeYield();
+ return delegate.getLockFactory();
+ }
+
+ @Override
+ public synchronized String getLockID() {
+ maybeYield();
+ return delegate.getLockID();
+ }
+
+ @Override
+ public synchronized void copy(Directory to, String src, String dest) throws IOException {
+ maybeYield();
+ delegate.copy(to, src, dest);
+ }
+}
fnp / pylucene.git / blobdiff