+++ /dev/null
-package org.apache.lucene.store;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FilenameFilter;
-import java.io.IOException;
-import java.io.RandomAccessFile;
-
-import java.util.Collection;
-import static java.util.Collections.synchronizedSet;
-
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.concurrent.Future;
-
-import org.apache.lucene.util.ThreadInterruptedException;
-import org.apache.lucene.util.Constants;
-
-/**
- * <a name="subclasses"/>
- * Base class for Directory implementations that store index
- * files in the file system. There are currently three core
- * subclasses:
- *
- * <ul>
- *
- * <li> {@link SimpleFSDirectory} is a straightforward
- * implementation using java.io.RandomAccessFile.
- * However, it has poor concurrent performance
- * (multiple threads will bottleneck) as it
- * synchronizes when multiple threads read from the
- * same file.
- *
- * <li> {@link NIOFSDirectory} uses java.nio's
- * FileChannel's positional io when reading to avoid
- * synchronization when reading from the same file.
- * Unfortunately, due to a Windows-only <a
- * href="http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6265734">Sun
- * JRE bug</a> this is a poor choice for Windows, but
- * on all other platforms this is the preferred
- * choice. Applications using {@link Thread#interrupt()} or
- * {@link Future#cancel(boolean)} should use
- * {@link SimpleFSDirectory} instead. See {@link NIOFSDirectory} java doc
- * for details.
- *
- *
- *
- * <li> {@link MMapDirectory} uses memory-mapped IO when
- * reading. This is a good choice if you have plenty
- * of virtual memory relative to your index size, eg
- * if you are running on a 64 bit JRE, or you are
- * running on a 32 bit JRE but your index sizes are
- * small enough to fit into the virtual memory space.
- * Java has currently the limitation of not being able to
- * unmap files from user code. The files are unmapped, when GC
- * releases the byte buffers. Due to
- * <a href="http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4724038">
- * this bug</a> in Sun's JRE, MMapDirectory's {@link IndexInput#close}
- * is unable to close the underlying OS file handle. Only when
- * GC finally collects the underlying objects, which could be
- * quite some time later, will the file handle be closed.
- * This will consume additional transient disk usage: on Windows,
- * attempts to delete or overwrite the files will result in an
- * exception; on other platforms, which typically have a "delete on
- * last close" semantics, while such operations will succeed, the bytes
- * are still consuming space on disk. For many applications this
- * limitation is not a problem (e.g. if you have plenty of disk space,
- * and you don't rely on overwriting files on Windows) but it's still
- * an important limitation to be aware of. This class supplies a
- * (possibly dangerous) workaround mentioned in the bug report,
- * which may fail on non-Sun JVMs.
- *
- * Applications using {@link Thread#interrupt()} or
- * {@link Future#cancel(boolean)} should use
- * {@link SimpleFSDirectory} instead. See {@link MMapDirectory}
- * java doc for details.
- * </ul>
- *
- * Unfortunately, because of system peculiarities, there is
- * no single overall best implementation. Therefore, we've
- * added the {@link #open} method, to allow Lucene to choose
- * the best FSDirectory implementation given your
- * environment, and the known limitations of each
- * implementation. For users who have no reason to prefer a
- * specific implementation, it's best to simply use {@link
- * #open}. For all others, you should instantiate the
- * desired implementation directly.
- *
- * <p>The locking implementation is by default {@link
- * NativeFSLockFactory}, but can be changed by
- * passing in a custom {@link LockFactory} instance.
- *
- * @see Directory
- */
-public abstract class FSDirectory extends Directory {
-
- /**
- * Default read chunk size. This is a conditional default: on 32bit JVMs, it defaults to 100 MB. On 64bit JVMs, it's
- * <code>Integer.MAX_VALUE</code>.
- *
- * @see #setReadChunkSize
- */
- public static final int DEFAULT_READ_CHUNK_SIZE = Constants.JRE_IS_64BIT ? Integer.MAX_VALUE : 100 * 1024 * 1024;
-
- protected final File directory; // The underlying filesystem directory
- protected final Set<String> staleFiles = synchronizedSet(new HashSet<String>()); // Files written, but not yet sync'ed
- private int chunkSize = DEFAULT_READ_CHUNK_SIZE; // LUCENE-1566
-
- // returns the canonical version of the directory, creating it if it doesn't exist.
- private static File getCanonicalPath(File file) throws IOException {
- return new File(file.getCanonicalPath());
- }
-
- /** Create a new FSDirectory for the named location (ctor for subclasses).
- * @param path the path of the directory
- * @param lockFactory the lock factory to use, or null for the default
- * ({@link NativeFSLockFactory});
- * @throws IOException
- */
- protected FSDirectory(File path, LockFactory lockFactory) throws IOException {
- // new ctors use always NativeFSLockFactory as default:
- if (lockFactory == null) {
- lockFactory = new NativeFSLockFactory();
- }
- directory = getCanonicalPath(path);
-
- if (directory.exists() && !directory.isDirectory())
- throw new NoSuchDirectoryException("file '" + directory + "' exists but is not a directory");
-
- setLockFactory(lockFactory);
- }
-
- /** Creates an FSDirectory instance, trying to pick the
- * best implementation given the current environment.
- * The directory returned uses the {@link NativeFSLockFactory}.
- *
- * <p>Currently this returns {@link MMapDirectory} for most Solaris
- * and Windows 64-bit JREs, {@link NIOFSDirectory} for other
- * non-Windows JREs, and {@link SimpleFSDirectory} for other
- * JREs on Windows. It is highly recommended that you consult the
- * implementation's documentation for your platform before
- * using this method.
- *
- * <p><b>NOTE</b>: this method may suddenly change which
- * implementation is returned from release to release, in
- * the event that higher performance defaults become
- * possible; if the precise implementation is important to
- * your application, please instantiate it directly,
- * instead. For optimal performance you should consider using
- * {@link MMapDirectory} on 64 bit JVMs.
- *
- * <p>See <a href="#subclasses">above</a> */
- public static FSDirectory open(File path) throws IOException {
- return open(path, null);
- }
-
- /** Just like {@link #open(File)}, but allows you to
- * also specify a custom {@link LockFactory}. */
- public static FSDirectory open(File path, LockFactory lockFactory) throws IOException {
- if ((Constants.WINDOWS || Constants.SUN_OS || Constants.LINUX)
- && Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) {
- return new MMapDirectory(path, lockFactory);
- } else if (Constants.WINDOWS) {
- return new SimpleFSDirectory(path, lockFactory);
- } else {
- return new NIOFSDirectory(path, lockFactory);
- }
- }
-
- @Override
- public void setLockFactory(LockFactory lockFactory) throws IOException {
- super.setLockFactory(lockFactory);
-
- // for filesystem based LockFactory, delete the lockPrefix, if the locks are placed
- // in index dir. If no index dir is given, set ourselves
- if (lockFactory instanceof FSLockFactory) {
- final FSLockFactory lf = (FSLockFactory) lockFactory;
- final File dir = lf.getLockDir();
- // if the lock factory has no lockDir set, use the this directory as lockDir
- if (dir == null) {
- lf.setLockDir(directory);
- lf.setLockPrefix(null);
- } else if (dir.getCanonicalPath().equals(directory.getCanonicalPath())) {
- lf.setLockPrefix(null);
- }
- }
-
- }
-
- /** Lists all files (not subdirectories) in the
- * directory. This method never returns null (throws
- * {@link IOException} instead).
- *
- * @throws NoSuchDirectoryException if the directory
- * does not exist, or does exist but is not a
- * directory.
- * @throws IOException if list() returns null */
- public static String[] listAll(File dir) throws IOException {
- if (!dir.exists())
- throw new NoSuchDirectoryException("directory '" + dir + "' does not exist");
- else if (!dir.isDirectory())
- throw new NoSuchDirectoryException("file '" + dir + "' exists but is not a directory");
-
- // Exclude subdirs
- String[] result = dir.list(new FilenameFilter() {
- public boolean accept(File dir, String file) {
- return !new File(dir, file).isDirectory();
- }
- });
-
- if (result == null)
- throw new IOException("directory '" + dir + "' exists and is a directory, but cannot be listed: list() returned null");
-
- return result;
- }
-
- /** Lists all files (not subdirectories) in the
- * directory.
- * @see #listAll(File) */
- @Override
- public String[] listAll() throws IOException {
- ensureOpen();
- return listAll(directory);
- }
-
- /** Returns true iff a file with the given name exists. */
- @Override
- public boolean fileExists(String name) {
- ensureOpen();
- File file = new File(directory, name);
- return file.exists();
- }
-
- /** Returns the time the named file was last modified. */
- @Override
- public long fileModified(String name) {
- ensureOpen();
- File file = new File(directory, name);
- return file.lastModified();
- }
-
- /** Returns the time the named file was last modified. */
- public static long fileModified(File directory, String name) {
- File file = new File(directory, name);
- return file.lastModified();
- }
-
- /** Set the modified time of an existing file to now.
- * @deprecated Lucene never uses this API; it will be
- * removed in 4.0. */
- @Override
- @Deprecated
- public void touchFile(String name) {
- ensureOpen();
- File file = new File(directory, name);
- file.setLastModified(System.currentTimeMillis());
- }
-
- /** Returns the length in bytes of a file in the directory. */
- @Override
- public long fileLength(String name) throws IOException {
- ensureOpen();
- File file = new File(directory, name);
- final long len = file.length();
- if (len == 0 && !file.exists()) {
- throw new FileNotFoundException(name);
- } else {
- return len;
- }
- }
-
- /** Removes an existing file in the directory. */
- @Override
- public void deleteFile(String name) throws IOException {
- ensureOpen();
- File file = new File(directory, name);
- if (!file.delete())
- throw new IOException("Cannot delete " + file);
- staleFiles.remove(name);
- }
-
- /** Creates an IndexOutput for the file with the given name. */
- @Override
- public IndexOutput createOutput(String name) throws IOException {
- ensureOpen();
-
- ensureCanWrite(name);
- return new FSIndexOutput(this, name);
- }
-
- protected void ensureCanWrite(String name) throws IOException {
- if (!directory.exists())
- if (!directory.mkdirs())
- throw new IOException("Cannot create directory: " + directory);
-
- File file = new File(directory, name);
- if (file.exists() && !file.delete()) // delete existing, if any
- throw new IOException("Cannot overwrite: " + file);
- }
-
- protected void onIndexOutputClosed(FSIndexOutput io) {
- staleFiles.add(io.name);
- }
-
- @Deprecated
- @Override
- public void sync(String name) throws IOException {
- sync(Collections.singleton(name));
- }
-
- @Override
- public void sync(Collection<String> names) throws IOException {
- ensureOpen();
- Set<String> toSync = new HashSet<String>(names);
- toSync.retainAll(staleFiles);
-
- for (String name : toSync)
- fsync(name);
-
- staleFiles.removeAll(toSync);
- }
-
- // Inherit javadoc
- @Override
- public IndexInput openInput(String name) throws IOException {
- ensureOpen();
- return openInput(name, BufferedIndexInput.BUFFER_SIZE);
- }
-
- @Override
- public String getLockID() {
- ensureOpen();
- String dirName; // name to be hashed
- try {
- dirName = directory.getCanonicalPath();
- } catch (IOException e) {
- throw new RuntimeException(e.toString(), e);
- }
-
- int digest = 0;
- for(int charIDX=0;charIDX<dirName.length();charIDX++) {
- final char ch = dirName.charAt(charIDX);
- digest = 31 * digest + ch;
- }
- return "lucene-" + Integer.toHexString(digest);
- }
-
- /** Closes the store to future operations. */
- @Override
- public synchronized void close() {
- isOpen = false;
- }
-
- /** @deprecated Use {@link #getDirectory} instead. */
- @Deprecated
- public File getFile() {
- return getDirectory();
- }
-
- /** @return the underlying filesystem directory */
- public File getDirectory() {
- ensureOpen();
- return directory;
- }
-
- /** For debug output. */
- @Override
- public String toString() {
- return this.getClass().getName() + "@" + directory + " lockFactory=" + getLockFactory();
- }
-
- /**
- * Sets the maximum number of bytes read at once from the
- * underlying file during {@link IndexInput#readBytes}.
- * The default value is {@link #DEFAULT_READ_CHUNK_SIZE};
- *
- * <p> This was introduced due to <a
- * href="http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6478546">Sun
- * JVM Bug 6478546</a>, which throws an incorrect
- * OutOfMemoryError when attempting to read too many bytes
- * at once. It only happens on 32bit JVMs with a large
- * maximum heap size.</p>
- *
- * <p>Changes to this value will not impact any
- * already-opened {@link IndexInput}s. You should call
- * this before attempting to open an index on the
- * directory.</p>
- *
- * <p> <b>NOTE</b>: This value should be as large as
- * possible to reduce any possible performance impact. If
- * you still encounter an incorrect OutOfMemoryError,
- * trying lowering the chunk size.</p>
- */
- public final void setReadChunkSize(int chunkSize) {
- // LUCENE-1566
- if (chunkSize <= 0) {
- throw new IllegalArgumentException("chunkSize must be positive");
- }
- if (!Constants.JRE_IS_64BIT) {
- this.chunkSize = chunkSize;
- }
- }
-
- /**
- * The maximum number of bytes to read at once from the
- * underlying file during {@link IndexInput#readBytes}.
- * @see #setReadChunkSize
- */
- public final int getReadChunkSize() {
- // LUCENE-1566
- return chunkSize;
- }
-
- protected static class FSIndexOutput extends BufferedIndexOutput {
- private final FSDirectory parent;
- private final String name;
- private final RandomAccessFile file;
- private volatile boolean isOpen; // remember if the file is open, so that we don't try to close it more than once
-
- public FSIndexOutput(FSDirectory parent, String name) throws IOException {
- this.parent = parent;
- this.name = name;
- file = new RandomAccessFile(new File(parent.directory, name), "rw");
- isOpen = true;
- }
-
- /** output methods: */
- @Override
- public void flushBuffer(byte[] b, int offset, int size) throws IOException {
- file.write(b, offset, size);
- }
-
- @Override
- public void close() throws IOException {
- parent.onIndexOutputClosed(this);
- // only close the file if it has not been closed yet
- if (isOpen) {
- boolean success = false;
- try {
- super.close();
- success = true;
- } finally {
- isOpen = false;
- if (!success) {
- try {
- file.close();
- } catch (Throwable t) {
- // Suppress so we don't mask original exception
- }
- } else {
- file.close();
- }
- }
- }
- }
-
- /** Random-access methods */
- @Override
- public void seek(long pos) throws IOException {
- super.seek(pos);
- file.seek(pos);
- }
-
- @Override
- public long length() throws IOException {
- return file.length();
- }
-
- @Override
- public void setLength(long length) throws IOException {
- file.setLength(length);
- }
- }
-
- protected void fsync(String name) throws IOException {
- File fullFile = new File(directory, name);
- boolean success = false;
- int retryCount = 0;
- IOException exc = null;
- while (!success && retryCount < 5) {
- retryCount++;
- RandomAccessFile file = null;
- try {
- try {
- file = new RandomAccessFile(fullFile, "rw");
- file.getFD().sync();
- success = true;
- } finally {
- if (file != null)
- file.close();
- }
- } catch (IOException ioe) {
- if (exc == null)
- exc = ioe;
- try {
- // Pause 5 msec
- Thread.sleep(5);
- } catch (InterruptedException ie) {
- throw new ThreadInterruptedException(ie);
- }
- }
- }
- if (!success)
- // Throw original exception
- throw exc;
- }
-}
fnp / pylucene.git / blobdiff