+++ /dev/null
-package org.apache.lucene.store;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.Closeable;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.IdentityHashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Random;
-import java.util.Set;
-
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.ThrottledIndexOutput;
-import org.apache.lucene.util._TestUtil;
-
-/**
- * This is a Directory Wrapper that adds methods
- * intended to be used only by unit tests.
- * It also adds a number of features useful for testing:
- * <ul>
- * <li> Instances created by {@link LuceneTestCase#newDirectory()} are tracked
- * to ensure they are closed by the test.
- * <li> When a MockDirectoryWrapper is closed, it will throw an exception if
- * it has any open files against it (with a stacktrace indicating where
- * they were opened from).
- * <li> When a MockDirectoryWrapper is closed, it runs CheckIndex to test if
- * the index was corrupted.
- * <li> MockDirectoryWrapper simulates some "features" of Windows, such as
- * refusing to write/delete to open files.
- * </ul>
- */
-
-public class MockDirectoryWrapper extends Directory {
- final Directory delegate;
- long maxSize;
-
- // Max actual bytes used. This is set by MockRAMOutputStream:
- long maxUsedSize;
- double randomIOExceptionRate;
- Random randomState;
- boolean noDeleteOpenFile = true;
- boolean preventDoubleWrite = true;
- boolean checkIndexOnClose = true;
- boolean trackDiskUsage = false;
- private Set<String> unSyncedFiles;
- private Set<String> createdFiles;
- private Set<String> openFilesForWrite = new HashSet<String>();
- Set<String> openLocks = Collections.synchronizedSet(new HashSet<String>());
- volatile boolean crashed;
- private ThrottledIndexOutput throttledOutput;
- private Throttling throttling = Throttling.SOMETIMES;
-
- // use this for tracking files for crash.
- // additionally: provides debugging information in case you leave one open
- private Map<Closeable,Exception> openFileHandles = Collections.synchronizedMap(new IdentityHashMap<Closeable,Exception>());
-
- // NOTE: we cannot initialize the Map here due to the
- // order in which our constructor actually does this
- // member initialization vs when it calls super. It seems
- // like super is called, then our members are initialized:
- private Map<String,Integer> openFiles;
-
- // Only tracked if noDeleteOpenFile is true: if an attempt
- // is made to delete an open file, we enroll it here.
- private Set<String> openFilesDeleted;
-
- private synchronized void init() {
- if (openFiles == null) {
- openFiles = new HashMap<String,Integer>();
- openFilesDeleted = new HashSet<String>();
- }
-
- if (createdFiles == null)
- createdFiles = new HashSet<String>();
- if (unSyncedFiles == null)
- unSyncedFiles = new HashSet<String>();
- }
-
- public MockDirectoryWrapper(Random random, Directory delegate) {
- this.delegate = delegate;
- // must make a private random since our methods are
- // called from different threads; else test failures may
- // not be reproducible from the original seed
- this.randomState = new Random(random.nextInt());
- this.throttledOutput = new ThrottledIndexOutput(ThrottledIndexOutput
- .mBitsToBytes(40 + randomState.nextInt(10)), 5 + randomState.nextInt(5), null);
- // force wrapping of lockfactory
- try {
- setLockFactory(new MockLockFactoryWrapper(this, delegate.getLockFactory()));
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- init();
- }
-
- public void setTrackDiskUsage(boolean v) {
- trackDiskUsage = v;
- }
-
- /** If set to true, we throw an IOException if the same
- * file is opened by createOutput, ever. */
- public void setPreventDoubleWrite(boolean value) {
- preventDoubleWrite = value;
- }
-
- @Deprecated
- @Override
- public void sync(String name) throws IOException {
- maybeYield();
- maybeThrowDeterministicException();
- if (crashed)
- throw new IOException("cannot sync after crash");
- unSyncedFiles.remove(name);
- delegate.sync(name);
- }
-
- public static enum Throttling {
- /** always emulate a slow hard disk. could be very slow! */
- ALWAYS,
- /** sometimes (2% of the time) emulate a slow hard disk. */
- SOMETIMES,
- /** never throttle output */
- NEVER
- }
-
- public void setThrottling(Throttling throttling) {
- this.throttling = throttling;
- }
-
- @Override
- public synchronized void sync(Collection<String> names) throws IOException {
- maybeYield();
- for (String name : names)
- maybeThrowDeterministicException();
- if (crashed)
- throw new IOException("cannot sync after crash");
- unSyncedFiles.removeAll(names);
- delegate.sync(names);
- }
-
- @Override
- public String toString() {
- maybeYield();
- return "MockDirWrapper(" + delegate + ")";
- }
-
- public synchronized final long sizeInBytes() throws IOException {
- if (delegate instanceof RAMDirectory)
- return ((RAMDirectory) delegate).sizeInBytes();
- else {
- // hack
- long size = 0;
- for (String file : delegate.listAll())
- size += delegate.fileLength(file);
- return size;
- }
- }
-
- /** Simulates a crash of OS or machine by overwriting
- * unsynced files. */
- public synchronized void crash() throws IOException {
- crashed = true;
- openFiles = new HashMap<String,Integer>();
- openFilesForWrite = new HashSet<String>();
- openFilesDeleted = new HashSet<String>();
- Iterator<String> it = unSyncedFiles.iterator();
- unSyncedFiles = new HashSet<String>();
- // first force-close all files, so we can corrupt on windows etc.
- // clone the file map, as these guys want to remove themselves on close.
- Map<Closeable,Exception> m = new IdentityHashMap<Closeable,Exception>(openFileHandles);
- for (Closeable f : m.keySet())
- try {
- f.close();
- } catch (Exception ignored) {}
-
- int count = 0;
- while(it.hasNext()) {
- String name = it.next();
- if (count % 3 == 0) {
- deleteFile(name, true);
- } else if (count % 3 == 1) {
- // Zero out file entirely
- long length = fileLength(name);
- byte[] zeroes = new byte[256];
- long upto = 0;
- IndexOutput out = delegate.createOutput(name);
- while(upto < length) {
- final int limit = (int) Math.min(length-upto, zeroes.length);
- out.writeBytes(zeroes, 0, limit);
- upto += limit;
- }
- out.close();
- } else if (count % 3 == 2) {
- // Truncate the file:
- IndexOutput out = delegate.createOutput(name);
- out.setLength(fileLength(name)/2);
- out.close();
- }
- count++;
- }
- }
-
- public synchronized void clearCrash() throws IOException {
- crashed = false;
- openLocks.clear();
- }
-
- public void setMaxSizeInBytes(long maxSize) {
- this.maxSize = maxSize;
- }
- public long getMaxSizeInBytes() {
- return this.maxSize;
- }
-
- /**
- * Returns the peek actual storage used (bytes) in this
- * directory.
- */
- public long getMaxUsedSizeInBytes() {
- return this.maxUsedSize;
- }
- public void resetMaxUsedSizeInBytes() throws IOException {
- this.maxUsedSize = getRecomputedActualSizeInBytes();
- }
-
- /**
- * Emulate windows whereby deleting an open file is not
- * allowed (raise IOException).
- */
- public void setNoDeleteOpenFile(boolean value) {
- this.noDeleteOpenFile = value;
- }
- public boolean getNoDeleteOpenFile() {
- return noDeleteOpenFile;
- }
-
- /**
- * Set whether or not checkindex should be run
- * on close
- */
- public void setCheckIndexOnClose(boolean value) {
- this.checkIndexOnClose = value;
- }
-
- public boolean getCheckIndexOnClose() {
- return checkIndexOnClose;
- }
- /**
- * If 0.0, no exceptions will be thrown. Else this should
- * be a double 0.0 - 1.0. We will randomly throw an
- * IOException on the first write to an OutputStream based
- * on this probability.
- */
- public void setRandomIOExceptionRate(double rate) {
- randomIOExceptionRate = rate;
- }
- public double getRandomIOExceptionRate() {
- return randomIOExceptionRate;
- }
-
- void maybeThrowIOException() throws IOException {
- if (randomIOExceptionRate > 0.0) {
- int number = Math.abs(randomState.nextInt() % 1000);
- if (number < randomIOExceptionRate*1000) {
- if (LuceneTestCase.VERBOSE) {
- System.out.println(Thread.currentThread().getName() + ": MockDirectoryWrapper: now throw random exception");
- new Throwable().printStackTrace(System.out);
- }
- throw new IOException("a random IOException");
- }
- }
- }
-
- @Override
- public synchronized void deleteFile(String name) throws IOException {
- maybeYield();
- deleteFile(name, false);
- }
-
- // sets the cause of the incoming ioe to be the stack
- // trace when the offending file name was opened
- private synchronized IOException fillOpenTrace(IOException ioe, String name, boolean input) {
- for(Map.Entry<Closeable,Exception> ent : openFileHandles.entrySet()) {
- if (input && ent.getKey() instanceof MockIndexInputWrapper && ((MockIndexInputWrapper) ent.getKey()).name.equals(name)) {
- ioe.initCause(ent.getValue());
- break;
- } else if (!input && ent.getKey() instanceof MockIndexOutputWrapper && ((MockIndexOutputWrapper) ent.getKey()).name.equals(name)) {
- ioe.initCause(ent.getValue());
- break;
- }
- }
- return ioe;
- }
-
- private void maybeYield() {
- if (randomState.nextBoolean()) {
- Thread.yield();
- }
- }
-
- private synchronized void deleteFile(String name, boolean forced) throws IOException {
- maybeYield();
-
- maybeThrowDeterministicException();
-
- if (crashed && !forced)
- throw new IOException("cannot delete after crash");
-
- if (unSyncedFiles.contains(name))
- unSyncedFiles.remove(name);
- if (!forced && noDeleteOpenFile) {
- if (openFiles.containsKey(name)) {
- openFilesDeleted.add(name);
- throw fillOpenTrace(new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open: cannot delete"), name, true);
- } else {
- openFilesDeleted.remove(name);
- }
- }
- delegate.deleteFile(name);
- }
-
- public synchronized Set<String> getOpenDeletedFiles() {
- return new HashSet<String>(openFilesDeleted);
- }
-
- @Override
- public synchronized IndexOutput createOutput(String name) throws IOException {
- maybeYield();
- if (crashed)
- throw new IOException("cannot createOutput after crash");
- init();
- synchronized(this) {
- if (preventDoubleWrite && createdFiles.contains(name) && !name.equals("segments.gen"))
- throw new IOException("file \"" + name + "\" was already written to");
- }
- if (noDeleteOpenFile && openFiles.containsKey(name))
- throw new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open: cannot overwrite");
-
- if (crashed)
- throw new IOException("cannot createOutput after crash");
- unSyncedFiles.add(name);
- createdFiles.add(name);
-
- if (delegate instanceof RAMDirectory) {
- RAMDirectory ramdir = (RAMDirectory) delegate;
- RAMFile file = new RAMFile(ramdir);
- RAMFile existing = ramdir.fileMap.get(name);
-
- // Enforce write once:
- if (existing!=null && !name.equals("segments.gen") && preventDoubleWrite)
- throw new IOException("file " + name + " already exists");
- else {
- if (existing!=null) {
- ramdir.sizeInBytes.getAndAdd(-existing.sizeInBytes);
- existing.directory = null;
- }
- ramdir.fileMap.put(name, file);
- }
- }
-
- //System.out.println(Thread.currentThread().getName() + ": MDW: create " + name);
- IndexOutput io = new MockIndexOutputWrapper(this, delegate.createOutput(name), name);
- addFileHandle(io, name, false);
- openFilesForWrite.add(name);
-
- // throttling REALLY slows down tests, so don't do it very often for SOMETIMES.
- if (throttling == Throttling.ALWAYS ||
- (throttling == Throttling.SOMETIMES && randomState.nextInt(50) == 0)) {
- if (LuceneTestCase.VERBOSE) {
- System.out.println("MockDirectoryWrapper: throttling indexOutput");
- }
- return throttledOutput.newFromDelegate(io);
- } else {
- return io;
- }
- }
-
- private void addFileHandle(Closeable c, String name, boolean input) {
- Integer v = openFiles.get(name);
- if (v != null) {
- v = Integer.valueOf(v.intValue()+1);
- openFiles.put(name, v);
- } else {
- openFiles.put(name, Integer.valueOf(1));
- }
-
- openFileHandles.put(c, new RuntimeException("unclosed Index" + (input ? "Input" : "Output") + ": " + name));
- }
-
- @Override
- public synchronized IndexInput openInput(String name) throws IOException {
- maybeYield();
- if (!delegate.fileExists(name))
- throw new FileNotFoundException(name);
-
- // cannot open a file for input if it's still open for
- // output, except for segments.gen and segments_N
- if (openFilesForWrite.contains(name) && !name.startsWith("segments")) {
- throw fillOpenTrace(new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open for writing"), name, false);
- }
-
- IndexInput ii = new MockIndexInputWrapper(this, name, delegate.openInput(name));
- addFileHandle(ii, name, true);
- return ii;
- }
-
- /** Provided for testing purposes. Use sizeInBytes() instead. */
- public synchronized final long getRecomputedSizeInBytes() throws IOException {
- if (!(delegate instanceof RAMDirectory))
- return sizeInBytes();
- long size = 0;
- for(final RAMFile file: ((RAMDirectory)delegate).fileMap.values()) {
- size += file.getSizeInBytes();
- }
- return size;
- }
-
- /** Like getRecomputedSizeInBytes(), but, uses actual file
- * lengths rather than buffer allocations (which are
- * quantized up to nearest
- * RAMOutputStream.BUFFER_SIZE (now 1024) bytes.
- */
-
- public final synchronized long getRecomputedActualSizeInBytes() throws IOException {
- if (!(delegate instanceof RAMDirectory))
- return sizeInBytes();
- long size = 0;
- for (final RAMFile file : ((RAMDirectory)delegate).fileMap.values())
- size += file.length;
- return size;
- }
-
- @Override
- public synchronized void close() throws IOException {
- maybeYield();
- if (openFiles == null) {
- openFiles = new HashMap<String,Integer>();
- openFilesDeleted = new HashSet<String>();
- }
- if (noDeleteOpenFile && openFiles.size() > 0) {
- // print the first one as its very verbose otherwise
- Exception cause = null;
- Iterator<Exception> stacktraces = openFileHandles.values().iterator();
- if (stacktraces.hasNext())
- cause = stacktraces.next();
- // RuntimeException instead of IOException because
- // super() does not throw IOException currently:
- throw new RuntimeException("MockDirectoryWrapper: cannot close: there are still open files: " + openFiles, cause);
- }
- if (noDeleteOpenFile && openLocks.size() > 0) {
- throw new RuntimeException("MockDirectoryWrapper: cannot close: there are still open locks: " + openLocks);
- }
- open = false;
- if (checkIndexOnClose && IndexReader.indexExists(this)) {
- if (LuceneTestCase.VERBOSE) {
- System.out.println("\nNOTE: MockDirectoryWrapper: now run CheckIndex");
- }
- _TestUtil.checkIndex(this);
- }
- delegate.close();
- }
-
- private synchronized void removeOpenFile(Closeable c, String name) {
- Integer v = openFiles.get(name);
- // Could be null when crash() was called
- if (v != null) {
- if (v.intValue() == 1) {
- openFiles.remove(name);
- openFilesDeleted.remove(name);
- } else {
- v = Integer.valueOf(v.intValue()-1);
- openFiles.put(name, v);
- }
- }
-
- openFileHandles.remove(c);
- }
-
- public synchronized void removeIndexOutput(IndexOutput out, String name) {
- openFilesForWrite.remove(name);
- removeOpenFile(out, name);
- }
-
- public synchronized void removeIndexInput(IndexInput in, String name) {
- removeOpenFile(in, name);
- }
-
- boolean open = true;
-
- public synchronized boolean isOpen() {
- return open;
- }
-
- /**
- * Objects that represent fail-able conditions. Objects of a derived
- * class are created and registered with the mock directory. After
- * register, each object will be invoked once for each first write
- * of a file, giving the object a chance to throw an IOException.
- */
- public static class Failure {
- /**
- * eval is called on the first write of every new file.
- */
- public void eval(MockDirectoryWrapper dir) throws IOException { }
-
- /**
- * reset should set the state of the failure to its default
- * (freshly constructed) state. Reset is convenient for tests
- * that want to create one failure object and then reuse it in
- * multiple cases. This, combined with the fact that Failure
- * subclasses are often anonymous classes makes reset difficult to
- * do otherwise.
- *
- * A typical example of use is
- * Failure failure = new Failure() { ... };
- * ...
- * mock.failOn(failure.reset())
- */
- public Failure reset() { return this; }
-
- protected boolean doFail;
-
- public void setDoFail() {
- doFail = true;
- }
-
- public void clearDoFail() {
- doFail = false;
- }
- }
-
- ArrayList<Failure> failures;
-
- /**
- * add a Failure object to the list of objects to be evaluated
- * at every potential failure point
- */
- synchronized public void failOn(Failure fail) {
- if (failures == null) {
- failures = new ArrayList<Failure>();
- }
- failures.add(fail);
- }
-
- /**
- * Iterate through the failures list, giving each object a
- * chance to throw an IOE
- */
- synchronized void maybeThrowDeterministicException() throws IOException {
- if (failures != null) {
- for(int i = 0; i < failures.size(); i++) {
- failures.get(i).eval(this);
- }
- }
- }
-
- @Override
- public synchronized String[] listAll() throws IOException {
- maybeYield();
- return delegate.listAll();
- }
-
- @Override
- public synchronized boolean fileExists(String name) throws IOException {
- maybeYield();
- return delegate.fileExists(name);
- }
-
- @Override
- public synchronized long fileModified(String name) throws IOException {
- maybeYield();
- return delegate.fileModified(name);
- }
-
- @Override
- @Deprecated
- /* @deprecated Lucene never uses this API; it will be
- * removed in 4.0. */
- public synchronized void touchFile(String name) throws IOException {
- maybeYield();
- delegate.touchFile(name);
- }
-
- @Override
- public synchronized long fileLength(String name) throws IOException {
- maybeYield();
- return delegate.fileLength(name);
- }
-
- @Override
- public synchronized Lock makeLock(String name) {
- maybeYield();
- return delegate.makeLock(name);
- }
-
- @Override
- public synchronized void clearLock(String name) throws IOException {
- maybeYield();
- delegate.clearLock(name);
- }
-
- @Override
- public synchronized void setLockFactory(LockFactory lockFactory) throws IOException {
- maybeYield();
- delegate.setLockFactory(lockFactory);
- }
-
- @Override
- public synchronized LockFactory getLockFactory() {
- maybeYield();
- return delegate.getLockFactory();
- }
-
- @Override
- public synchronized String getLockID() {
- maybeYield();
- return delegate.getLockID();
- }
-
- @Override
- public synchronized void copy(Directory to, String src, String dest) throws IOException {
- maybeYield();
- delegate.copy(to, src, dest);
- }
-}
fnp / pylucene.git / blobdiff