1 package org.apache.lucene.store;
4 * Licensed to the Apache Software Foundation (ASF) under one or more
5 * contributor license agreements. See the NOTICE file distributed with
6 * this work for additional information regarding copyright ownership.
7 * The ASF licenses this file to You under the Apache License, Version 2.0
8 * (the "License"); you may not use this file except in compliance with
9 * the License. You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
21 import java.io.FileNotFoundException;
22 import java.io.FilenameFilter;
23 import java.io.IOException;
24 import java.io.RandomAccessFile;
26 import java.util.Collection;
27 import static java.util.Collections.synchronizedSet;
29 import java.util.Collections;
30 import java.util.HashSet;
32 import java.util.concurrent.Future;
34 import org.apache.lucene.util.ThreadInterruptedException;
35 import org.apache.lucene.util.Constants;
38 * <a name="subclasses"/>
39 * Base class for Directory implementations that store index
40 * files in the file system. There are currently three core
45 * <li> {@link SimpleFSDirectory} is a straightforward
46 * implementation using java.io.RandomAccessFile.
47 * However, it has poor concurrent performance
48 * (multiple threads will bottleneck) as it
49 * synchronizes when multiple threads read from the
52 * <li> {@link NIOFSDirectory} uses java.nio's
53 * FileChannel's positional io when reading to avoid
54 * synchronization when reading from the same file.
55 * Unfortunately, due to a Windows-only <a
56 * href="http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6265734">Sun
57 * JRE bug</a> this is a poor choice for Windows, but
58 * on all other platforms this is the preferred
59 * choice. Applications using {@link Thread#interrupt()} or
60 * {@link Future#cancel(boolean)} should use
61 * {@link SimpleFSDirectory} instead. See {@link NIOFSDirectory} java doc
66 * <li> {@link MMapDirectory} uses memory-mapped IO when
67 * reading. This is a good choice if you have plenty
68 * of virtual memory relative to your index size, eg
69 * if you are running on a 64 bit JRE, or you are
70 * running on a 32 bit JRE but your index sizes are
71 * small enough to fit into the virtual memory space.
72 * Java has currently the limitation of not being able to
73 * unmap files from user code. The files are unmapped, when GC
74 * releases the byte buffers. Due to
75 * <a href="http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4724038">
76 * this bug</a> in Sun's JRE, MMapDirectory's {@link IndexInput#close}
77 * is unable to close the underlying OS file handle. Only when
78 * GC finally collects the underlying objects, which could be
79 * quite some time later, will the file handle be closed.
80 * This will consume additional transient disk usage: on Windows,
81 * attempts to delete or overwrite the files will result in an
82 * exception; on other platforms, which typically have a "delete on
83 * last close" semantics, while such operations will succeed, the bytes
84 * are still consuming space on disk. For many applications this
85 * limitation is not a problem (e.g. if you have plenty of disk space,
86 * and you don't rely on overwriting files on Windows) but it's still
87 * an important limitation to be aware of. This class supplies a
88 * (possibly dangerous) workaround mentioned in the bug report,
89 * which may fail on non-Sun JVMs.
91 * Applications using {@link Thread#interrupt()} or
92 * {@link Future#cancel(boolean)} should use
93 * {@link SimpleFSDirectory} instead. See {@link MMapDirectory}
94 * java doc for details.
97 * Unfortunately, because of system peculiarities, there is
98 * no single overall best implementation. Therefore, we've
99 * added the {@link #open} method, to allow Lucene to choose
100 * the best FSDirectory implementation given your
101 * environment, and the known limitations of each
102 * implementation. For users who have no reason to prefer a
103 * specific implementation, it's best to simply use {@link
104 * #open}. For all others, you should instantiate the
105 * desired implementation directly.
107 * <p>The locking implementation is by default {@link
108 * NativeFSLockFactory}, but can be changed by
109 * passing in a custom {@link LockFactory} instance.
113 public abstract class FSDirectory extends Directory {
116 * Default read chunk size. This is a conditional default: on 32bit JVMs, it defaults to 100 MB. On 64bit JVMs, it's
117 * <code>Integer.MAX_VALUE</code>.
119 * @see #setReadChunkSize
121 public static final int DEFAULT_READ_CHUNK_SIZE = Constants.JRE_IS_64BIT ? Integer.MAX_VALUE : 100 * 1024 * 1024;
123 protected final File directory; // The underlying filesystem directory
124 protected final Set<String> staleFiles = synchronizedSet(new HashSet<String>()); // Files written, but not yet sync'ed
125 private int chunkSize = DEFAULT_READ_CHUNK_SIZE; // LUCENE-1566
127 // returns the canonical version of the directory, creating it if it doesn't exist.
128 private static File getCanonicalPath(File file) throws IOException {
129 return new File(file.getCanonicalPath());
132 /** Create a new FSDirectory for the named location (ctor for subclasses).
133 * @param path the path of the directory
134 * @param lockFactory the lock factory to use, or null for the default
135 * ({@link NativeFSLockFactory});
136 * @throws IOException
138 protected FSDirectory(File path, LockFactory lockFactory) throws IOException {
139 // new ctors use always NativeFSLockFactory as default:
140 if (lockFactory == null) {
141 lockFactory = new NativeFSLockFactory();
143 directory = getCanonicalPath(path);
145 if (directory.exists() && !directory.isDirectory())
146 throw new NoSuchDirectoryException("file '" + directory + "' exists but is not a directory");
148 setLockFactory(lockFactory);
151 /** Creates an FSDirectory instance, trying to pick the
152 * best implementation given the current environment.
153 * The directory returned uses the {@link NativeFSLockFactory}.
155 * <p>Currently this returns {@link MMapDirectory} for most Solaris
156 * and Windows 64-bit JREs, {@link NIOFSDirectory} for other
157 * non-Windows JREs, and {@link SimpleFSDirectory} for other
158 * JREs on Windows. It is highly recommended that you consult the
159 * implementation's documentation for your platform before
162 * <p><b>NOTE</b>: this method may suddenly change which
163 * implementation is returned from release to release, in
164 * the event that higher performance defaults become
165 * possible; if the precise implementation is important to
166 * your application, please instantiate it directly,
167 * instead. For optimal performance you should consider using
168 * {@link MMapDirectory} on 64 bit JVMs.
170 * <p>See <a href="#subclasses">above</a> */
171 public static FSDirectory open(File path) throws IOException {
172 return open(path, null);
175 /** Just like {@link #open(File)}, but allows you to
176 * also specify a custom {@link LockFactory}. */
177 public static FSDirectory open(File path, LockFactory lockFactory) throws IOException {
178 if ((Constants.WINDOWS || Constants.SUN_OS || Constants.LINUX)
179 && Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) {
180 return new MMapDirectory(path, lockFactory);
181 } else if (Constants.WINDOWS) {
182 return new SimpleFSDirectory(path, lockFactory);
184 return new NIOFSDirectory(path, lockFactory);
189 public void setLockFactory(LockFactory lockFactory) throws IOException {
190 super.setLockFactory(lockFactory);
192 // for filesystem based LockFactory, delete the lockPrefix, if the locks are placed
193 // in index dir. If no index dir is given, set ourselves
194 if (lockFactory instanceof FSLockFactory) {
195 final FSLockFactory lf = (FSLockFactory) lockFactory;
196 final File dir = lf.getLockDir();
197 // if the lock factory has no lockDir set, use the this directory as lockDir
199 lf.setLockDir(directory);
200 lf.setLockPrefix(null);
201 } else if (dir.getCanonicalPath().equals(directory.getCanonicalPath())) {
202 lf.setLockPrefix(null);
208 /** Lists all files (not subdirectories) in the
209 * directory. This method never returns null (throws
210 * {@link IOException} instead).
212 * @throws NoSuchDirectoryException if the directory
213 * does not exist, or does exist but is not a
215 * @throws IOException if list() returns null */
216 public static String[] listAll(File dir) throws IOException {
218 throw new NoSuchDirectoryException("directory '" + dir + "' does not exist");
219 else if (!dir.isDirectory())
220 throw new NoSuchDirectoryException("file '" + dir + "' exists but is not a directory");
223 String[] result = dir.list(new FilenameFilter() {
224 public boolean accept(File dir, String file) {
225 return !new File(dir, file).isDirectory();
230 throw new IOException("directory '" + dir + "' exists and is a directory, but cannot be listed: list() returned null");
235 /** Lists all files (not subdirectories) in the
237 * @see #listAll(File) */
239 public String[] listAll() throws IOException {
241 return listAll(directory);
244 /** Returns true iff a file with the given name exists. */
246 public boolean fileExists(String name) {
248 File file = new File(directory, name);
249 return file.exists();
252 /** Returns the time the named file was last modified. */
254 public long fileModified(String name) {
256 File file = new File(directory, name);
257 return file.lastModified();
260 /** Returns the time the named file was last modified. */
261 public static long fileModified(File directory, String name) {
262 File file = new File(directory, name);
263 return file.lastModified();
266 /** Set the modified time of an existing file to now.
267 * @deprecated Lucene never uses this API; it will be
271 public void touchFile(String name) {
273 File file = new File(directory, name);
274 file.setLastModified(System.currentTimeMillis());
277 /** Returns the length in bytes of a file in the directory. */
279 public long fileLength(String name) throws IOException {
281 File file = new File(directory, name);
282 final long len = file.length();
283 if (len == 0 && !file.exists()) {
284 throw new FileNotFoundException(name);
290 /** Removes an existing file in the directory. */
292 public void deleteFile(String name) throws IOException {
294 File file = new File(directory, name);
296 throw new IOException("Cannot delete " + file);
297 staleFiles.remove(name);
300 /** Creates an IndexOutput for the file with the given name. */
302 public IndexOutput createOutput(String name) throws IOException {
305 ensureCanWrite(name);
306 return new FSIndexOutput(this, name);
309 protected void ensureCanWrite(String name) throws IOException {
310 if (!directory.exists())
311 if (!directory.mkdirs())
312 throw new IOException("Cannot create directory: " + directory);
314 File file = new File(directory, name);
315 if (file.exists() && !file.delete()) // delete existing, if any
316 throw new IOException("Cannot overwrite: " + file);
319 protected void onIndexOutputClosed(FSIndexOutput io) {
320 staleFiles.add(io.name);
325 public void sync(String name) throws IOException {
326 sync(Collections.singleton(name));
330 public void sync(Collection<String> names) throws IOException {
332 Set<String> toSync = new HashSet<String>(names);
333 toSync.retainAll(staleFiles);
335 for (String name : toSync)
338 staleFiles.removeAll(toSync);
343 public IndexInput openInput(String name) throws IOException {
345 return openInput(name, BufferedIndexInput.BUFFER_SIZE);
349 public String getLockID() {
351 String dirName; // name to be hashed
353 dirName = directory.getCanonicalPath();
354 } catch (IOException e) {
355 throw new RuntimeException(e.toString(), e);
359 for(int charIDX=0;charIDX<dirName.length();charIDX++) {
360 final char ch = dirName.charAt(charIDX);
361 digest = 31 * digest + ch;
363 return "lucene-" + Integer.toHexString(digest);
366 /** Closes the store to future operations. */
368 public synchronized void close() {
372 /** @deprecated Use {@link #getDirectory} instead. */
374 public File getFile() {
375 return getDirectory();
378 /** @return the underlying filesystem directory */
379 public File getDirectory() {
384 /** For debug output. */
386 public String toString() {
387 return this.getClass().getName() + "@" + directory + " lockFactory=" + getLockFactory();
391 * Sets the maximum number of bytes read at once from the
392 * underlying file during {@link IndexInput#readBytes}.
393 * The default value is {@link #DEFAULT_READ_CHUNK_SIZE};
395 * <p> This was introduced due to <a
396 * href="http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6478546">Sun
397 * JVM Bug 6478546</a>, which throws an incorrect
398 * OutOfMemoryError when attempting to read too many bytes
399 * at once. It only happens on 32bit JVMs with a large
400 * maximum heap size.</p>
402 * <p>Changes to this value will not impact any
403 * already-opened {@link IndexInput}s. You should call
404 * this before attempting to open an index on the
407 * <p> <b>NOTE</b>: This value should be as large as
408 * possible to reduce any possible performance impact. If
409 * you still encounter an incorrect OutOfMemoryError,
410 * trying lowering the chunk size.</p>
412 public final void setReadChunkSize(int chunkSize) {
414 if (chunkSize <= 0) {
415 throw new IllegalArgumentException("chunkSize must be positive");
417 if (!Constants.JRE_IS_64BIT) {
418 this.chunkSize = chunkSize;
423 * The maximum number of bytes to read at once from the
424 * underlying file during {@link IndexInput#readBytes}.
425 * @see #setReadChunkSize
427 public final int getReadChunkSize() {
432 protected static class FSIndexOutput extends BufferedIndexOutput {
433 private final FSDirectory parent;
434 private final String name;
435 private final RandomAccessFile file;
436 private volatile boolean isOpen; // remember if the file is open, so that we don't try to close it more than once
438 public FSIndexOutput(FSDirectory parent, String name) throws IOException {
439 this.parent = parent;
441 file = new RandomAccessFile(new File(parent.directory, name), "rw");
445 /** output methods: */
447 public void flushBuffer(byte[] b, int offset, int size) throws IOException {
448 file.write(b, offset, size);
452 public void close() throws IOException {
453 parent.onIndexOutputClosed(this);
454 // only close the file if it has not been closed yet
456 boolean success = false;
465 } catch (Throwable t) {
466 // Suppress so we don't mask original exception
475 /** Random-access methods */
477 public void seek(long pos) throws IOException {
483 public long length() throws IOException {
484 return file.length();
488 public void setLength(long length) throws IOException {
489 file.setLength(length);
493 protected void fsync(String name) throws IOException {
494 File fullFile = new File(directory, name);
495 boolean success = false;
497 IOException exc = null;
498 while (!success && retryCount < 5) {
500 RandomAccessFile file = null;
503 file = new RandomAccessFile(fullFile, "rw");
510 } catch (IOException ioe) {
516 } catch (InterruptedException ie) {
517 throw new ThreadInterruptedException(ie);
522 // Throw original exception