--- /dev/null
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.ChecksumIndexOutput;
+import org.apache.lucene.store.ChecksumIndexInput;
+import org.apache.lucene.store.NoSuchDirectoryException;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.ThreadInterruptedException;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A collection of segmentInfo objects with methods for operating on
+ * those segments in relation to the file system.
+ *
+ * @lucene.experimental
+ */
+public final class SegmentInfos implements Cloneable, Iterable<SegmentInfo> {
+
+ /** The file format version, a negative number. */
+ /* Works since counter, the old 1st entry, is always >= 0 */
+ public static final int FORMAT = -1;
+
+ /** This format adds details used for lockless commits. It differs
+ * slightly from the previous format in that file names
+ * are never re-used (write once). Instead, each file is
+ * written to the next generation. For example,
+ * segments_1, segments_2, etc. This allows us to not use
+ * a commit lock. See <a
+ * href="http://lucene.apache.org/java/docs/fileformats.html">file
+ * formats</a> for details.
+ */
+ public static final int FORMAT_LOCKLESS = -2;
+
+ /** This format adds a "hasSingleNormFile" flag into each segment info.
+ * See <a href="http://issues.apache.org/jira/browse/LUCENE-756">LUCENE-756</a>
+ * for details.
+ */
+ public static final int FORMAT_SINGLE_NORM_FILE = -3;
+
+ /** This format allows multiple segments to share a single
+ * vectors and stored fields file. */
+ public static final int FORMAT_SHARED_DOC_STORE = -4;
+
+ /** This format adds a checksum at the end of the file to
+ * ensure all bytes were successfully written. */
+ public static final int FORMAT_CHECKSUM = -5;
+
+ /** This format adds the deletion count for each segment.
+ * This way IndexWriter can efficiently report numDocs(). */
+ public static final int FORMAT_DEL_COUNT = -6;
+
+ /** This format adds the boolean hasProx to record if any
+ * fields in the segment store prox information (ie, have
+ * omitTermFreqAndPositions==false) */
+ public static final int FORMAT_HAS_PROX = -7;
+
+ /** This format adds optional commit userData (String) storage. */
+ public static final int FORMAT_USER_DATA = -8;
+
+ /** This format adds optional per-segment String
+ * diagnostics storage, and switches userData to Map */
+ public static final int FORMAT_DIAGNOSTICS = -9;
+
+ /** Each segment records whether it has term vectors */
+ public static final int FORMAT_HAS_VECTORS = -10;
+
+ /** Each segment records the Lucene version that created it. */
+ public static final int FORMAT_3_1 = -11;
+
+ /* This must always point to the most recent file format. */
+ public static final int CURRENT_FORMAT = FORMAT_3_1;
+
+ public static final int FORMAT_MINIMUM = FORMAT;
+ public static final int FORMAT_MAXIMUM = CURRENT_FORMAT;
+
+ public int counter = 0; // used to name new segments
+ /**
+ * counts how often the index has been changed by adding or deleting docs.
+ * starting with the current time in milliseconds forces to create unique version numbers.
+ */
+ long version = System.currentTimeMillis();
+
+ private long generation = 0; // generation of the "segments_N" for the next commit
+ private long lastGeneration = 0; // generation of the "segments_N" file we last successfully read
+ // or wrote; this is normally the same as generation except if
+ // there was an IOException that had interrupted a commit
+
+ private Map<String,String> userData = Collections.<String,String>emptyMap(); // Opaque Map<String, String> that user can specify during IndexWriter.commit
+
+ private int format;
+
+ private List<SegmentInfo> segments = new ArrayList<SegmentInfo>();
+ private Set<SegmentInfo> segmentSet = new HashSet<SegmentInfo>();
+ private transient List<SegmentInfo> cachedUnmodifiableList;
+ private transient Set<SegmentInfo> cachedUnmodifiableSet;
+
+ /**
+ * If non-null, information about loading segments_N files
+ * will be printed here. @see #setInfoStream.
+ */
+ private static PrintStream infoStream = null;
+
+ public void setFormat(int format) {
+ this.format = format;
+ }
+
+ public int getFormat() {
+ return format;
+ }
+
+ public SegmentInfo info(int i) {
+ return segments.get(i);
+ }
+
+ /**
+ * Get the generation (N) of the current segments_N file
+ * from a list of files.
+ *
+ * @param files -- array of file names to check
+ */
+ public static long getCurrentSegmentGeneration(String[] files) {
+ if (files == null) {
+ return -1;
+ }
+ long max = -1;
+ for (int i = 0; i < files.length; i++) {
+ String file = files[i];
+ if (file.startsWith(IndexFileNames.SEGMENTS) && !file.equals(IndexFileNames.SEGMENTS_GEN)) {
+ long gen = generationFromSegmentsFileName(file);
+ if (gen > max) {
+ max = gen;
+ }
+ }
+ }
+ return max;
+ }
+
+ /**
+ * Get the generation (N) of the current segments_N file
+ * in the directory.
+ *
+ * @param directory -- directory to search for the latest segments_N file
+ */
+ public static long getCurrentSegmentGeneration(Directory directory) throws IOException {
+ try {
+ return getCurrentSegmentGeneration(directory.listAll());
+ } catch (NoSuchDirectoryException nsde) {
+ return -1;
+ }
+ }
+
+ /**
+ * Get the filename of the current segments_N file
+ * from a list of files.
+ *
+ * @param files -- array of file names to check
+ */
+
+ public static String getCurrentSegmentFileName(String[] files) throws IOException {
+ return IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
+ "",
+ getCurrentSegmentGeneration(files));
+ }
+
+ /**
+ * Get the filename of the current segments_N file
+ * in the directory.
+ *
+ * @param directory -- directory to search for the latest segments_N file
+ */
+ public static String getCurrentSegmentFileName(Directory directory) throws IOException {
+ return IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
+ "",
+ getCurrentSegmentGeneration(directory));
+ }
+
+ /**
+ * Get the segments_N filename in use by this segment infos.
+ */
+ public String getCurrentSegmentFileName() {
+ return IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
+ "",
+ lastGeneration);
+ }
+
+ /**
+ * Parse the generation off the segments file name and
+ * return it.
+ */
+ public static long generationFromSegmentsFileName(String fileName) {
+ if (fileName.equals(IndexFileNames.SEGMENTS)) {
+ return 0;
+ } else if (fileName.startsWith(IndexFileNames.SEGMENTS)) {
+ return Long.parseLong(fileName.substring(1+IndexFileNames.SEGMENTS.length()),
+ Character.MAX_RADIX);
+ } else {
+ throw new IllegalArgumentException("fileName \"" + fileName + "\" is not a segments file");
+ }
+ }
+
+
+ /**
+ * Get the next segments_N filename that will be written.
+ */
+ public String getNextSegmentFileName() {
+ long nextGeneration;
+
+ if (generation == -1) {
+ nextGeneration = 1;
+ } else {
+ nextGeneration = generation+1;
+ }
+ return IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
+ "",
+ nextGeneration);
+ }
+
+ /**
+ * Read a particular segmentFileName. Note that this may
+ * throw an IOException if a commit is in process.
+ *
+ * @param directory -- directory containing the segments file
+ * @param segmentFileName -- segment file to load
+ * @throws CorruptIndexException if the index is corrupt
+ * @throws IOException if there is a low-level IO error
+ */
+ public final void read(Directory directory, String segmentFileName) throws CorruptIndexException, IOException {
+ boolean success = false;
+
+ // Clear any previous segments:
+ this.clear();
+
+ ChecksumIndexInput input = new ChecksumIndexInput(directory.openInput(segmentFileName));
+
+ generation = generationFromSegmentsFileName(segmentFileName);
+
+ lastGeneration = generation;
+
+ try {
+ int format = input.readInt();
+ // check that it is a format we can understand
+ if (format > FORMAT_MINIMUM) {
+ throw new IndexFormatTooOldException(input, format,
+ FORMAT_MINIMUM, FORMAT_MAXIMUM);
+ }
+ if (format < FORMAT_MAXIMUM) {
+ throw new IndexFormatTooNewException(input, format,
+ FORMAT_MINIMUM, FORMAT_MAXIMUM);
+ }
+ version = input.readLong(); // read version
+ counter = input.readInt(); // read counter
+
+ for (int i = input.readInt(); i > 0; i--) { // read segmentInfos
+ SegmentInfo si = new SegmentInfo(directory, format, input);
+ if (si.getVersion() == null) {
+ // It's a pre-3.1 segment, upgrade its version to either 3.0 or 2.x
+ Directory dir = directory;
+ if (si.getDocStoreOffset() != -1) {
+ if (si.getDocStoreIsCompoundFile()) {
+ dir = new CompoundFileReader(dir, IndexFileNames.segmentFileName(
+ si.getDocStoreSegment(),
+ IndexFileNames.COMPOUND_FILE_STORE_EXTENSION), 1024);
+ }
+ } else if (si.getUseCompoundFile()) {
+ dir = new CompoundFileReader(dir, IndexFileNames.segmentFileName(
+ si.name, IndexFileNames.COMPOUND_FILE_EXTENSION), 1024);
+ }
+
+ try {
+ String store = si.getDocStoreOffset() != -1 ? si.getDocStoreSegment() : si.name;
+ si.setVersion(FieldsReader.detectCodeVersion(dir, store));
+ } finally {
+ // If we opened the directory, close it
+ if (dir != directory) dir.close();
+ }
+ }
+ add(si);
+ }
+
+ if(format >= 0){ // in old format the version number may be at the end of the file
+ if (input.getFilePointer() >= input.length())
+ version = System.currentTimeMillis(); // old file format without version number
+ else
+ version = input.readLong(); // read version
+ }
+
+ if (format <= FORMAT_USER_DATA) {
+ if (format <= FORMAT_DIAGNOSTICS) {
+ userData = input.readStringStringMap();
+ } else if (0 != input.readByte()) {
+ userData = Collections.singletonMap("userData", input.readString());
+ } else {
+ userData = Collections.<String,String>emptyMap();
+ }
+ } else {
+ userData = Collections.<String,String>emptyMap();
+ }
+
+ if (format <= FORMAT_CHECKSUM) {
+ final long checksumNow = input.getChecksum();
+ final long checksumThen = input.readLong();
+ if (checksumNow != checksumThen)
+ throw new CorruptIndexException("checksum mismatch in segments file (resource: " + input + ")");
+ }
+ success = true;
+ }
+ finally {
+ input.close();
+ if (!success) {
+ // Clear any segment infos we had loaded so we
+ // have a clean slate on retry:
+ this.clear();
+ }
+ }
+ }
+
+ /**
+ * This version of read uses the retry logic (for lock-less
+ * commits) to find the right segments file to load.
+ * @throws CorruptIndexException if the index is corrupt
+ * @throws IOException if there is a low-level IO error
+ */
+ public final void read(Directory directory) throws CorruptIndexException, IOException {
+
+ generation = lastGeneration = -1;
+
+ new FindSegmentsFile(directory) {
+
+ @Override
+ protected Object doBody(String segmentFileName) throws CorruptIndexException, IOException {
+ read(directory, segmentFileName);
+ return null;
+ }
+ }.run();
+ }
+
+ // Only non-null after prepareCommit has been called and
+ // before finishCommit is called
+ ChecksumIndexOutput pendingSegnOutput;
+
+ private final void write(Directory directory) throws IOException {
+
+ String segmentFileName = getNextSegmentFileName();
+
+ // Always advance the generation on write:
+ if (generation == -1) {
+ generation = 1;
+ } else {
+ generation++;
+ }
+
+ ChecksumIndexOutput segnOutput = new ChecksumIndexOutput(directory.createOutput(segmentFileName));
+
+ boolean success = false;
+
+ try {
+ segnOutput.writeInt(CURRENT_FORMAT); // write FORMAT
+ segnOutput.writeLong(version);
+ segnOutput.writeInt(counter); // write counter
+ segnOutput.writeInt(size()); // write infos
+ for (SegmentInfo si : this) {
+ si.write(segnOutput);
+ }
+ segnOutput.writeStringStringMap(userData);
+ segnOutput.prepareCommit();
+ pendingSegnOutput = segnOutput;
+ success = true;
+ } finally {
+ if (!success) {
+ // We hit an exception above; try to close the file
+ // but suppress any exception:
+ IOUtils.closeWhileHandlingException(segnOutput);
+ try {
+ // Try not to leave a truncated segments_N file in
+ // the index:
+ directory.deleteFile(segmentFileName);
+ } catch (Throwable t) {
+ // Suppress so we keep throwing the original exception
+ }
+ }
+ }
+ }
+
+ /** Prunes any segment whose docs are all deleted. */
+ public void pruneDeletedSegments() throws IOException {
+ for(final Iterator<SegmentInfo> it = segments.iterator(); it.hasNext();) {
+ final SegmentInfo info = it.next();
+ if (info.getDelCount() == info.docCount) {
+ it.remove();
+ segmentSet.remove(info);
+ }
+ }
+ assert segmentSet.size() == segments.size();
+ }
+
+ /**
+ * Returns a copy of this instance, also copying each
+ * SegmentInfo.
+ */
+
+ @Override
+ public Object clone() {
+ try {
+ final SegmentInfos sis = (SegmentInfos) super.clone();
+ // deep clone, first recreate all collections:
+ sis.segments = new ArrayList<SegmentInfo>(size());
+ sis.segmentSet = new HashSet<SegmentInfo>(size());
+ sis.cachedUnmodifiableList = null;
+ sis.cachedUnmodifiableSet = null;
+ for(final SegmentInfo info : this) {
+ // dont directly access segments, use add method!!!
+ sis.add((SegmentInfo) info.clone());
+ }
+ sis.userData = new HashMap<String,String>(userData);
+ return sis;
+ } catch (CloneNotSupportedException e) {
+ throw new RuntimeException("should not happen", e);
+ }
+ }
+
+ /**
+ * version number when this SegmentInfos was generated.
+ */
+ public long getVersion() {
+ return version;
+ }
+ public long getGeneration() {
+ return generation;
+ }
+ public long getLastGeneration() {
+ return lastGeneration;
+ }
+
+ /**
+ * Current version number from segments file.
+ * @throws CorruptIndexException if the index is corrupt
+ * @throws IOException if there is a low-level IO error
+ */
+ public static long readCurrentVersion(Directory directory)
+ throws CorruptIndexException, IOException {
+
+ // Fully read the segments file: this ensures that it's
+ // completely written so that if
+ // IndexWriter.prepareCommit has been called (but not
+ // yet commit), then the reader will still see itself as
+ // current:
+ SegmentInfos sis = new SegmentInfos();
+ sis.read(directory);
+ return sis.version;
+ }
+
+ /**
+ * Returns userData from latest segments file
+ * @throws CorruptIndexException if the index is corrupt
+ * @throws IOException if there is a low-level IO error
+ */
+ public static Map<String,String> readCurrentUserData(Directory directory)
+ throws CorruptIndexException, IOException {
+ SegmentInfos sis = new SegmentInfos();
+ sis.read(directory);
+ return sis.getUserData();
+ }
+
+ /** If non-null, information about retries when loading
+ * the segments file will be printed to this.
+ */
+ public static void setInfoStream(PrintStream infoStream) {
+ SegmentInfos.infoStream = infoStream;
+ }
+
+ /* Advanced configuration of retry logic in loading
+ segments_N file */
+ private static int defaultGenFileRetryCount = 10;
+ private static int defaultGenFileRetryPauseMsec = 50;
+ private static int defaultGenLookaheadCount = 10;
+
+ /**
+ * Advanced: set how many times to try loading the
+ * segments.gen file contents to determine current segment
+ * generation. This file is only referenced when the
+ * primary method (listing the directory) fails.
+ */
+ public static void setDefaultGenFileRetryCount(int count) {
+ defaultGenFileRetryCount = count;
+ }
+
+ /**
+ * @see #setDefaultGenFileRetryCount
+ */
+ public static int getDefaultGenFileRetryCount() {
+ return defaultGenFileRetryCount;
+ }
+
+ /**
+ * Advanced: set how many milliseconds to pause in between
+ * attempts to load the segments.gen file.
+ */
+ public static void setDefaultGenFileRetryPauseMsec(int msec) {
+ defaultGenFileRetryPauseMsec = msec;
+ }
+
+ /**
+ * @see #setDefaultGenFileRetryPauseMsec
+ */
+ public static int getDefaultGenFileRetryPauseMsec() {
+ return defaultGenFileRetryPauseMsec;
+ }
+
+ /**
+ * Advanced: set how many times to try incrementing the
+ * gen when loading the segments file. This only runs if
+ * the primary (listing directory) and secondary (opening
+ * segments.gen file) methods fail to find the segments
+ * file.
+ */
+ public static void setDefaultGenLookaheadCount(int count) {
+ defaultGenLookaheadCount = count;
+ }
+ /**
+ * @see #setDefaultGenLookaheadCount
+ */
+ public static int getDefaultGenLookahedCount() {
+ return defaultGenLookaheadCount;
+ }
+
+ /**
+ * @see #setInfoStream
+ */
+ public static PrintStream getInfoStream() {
+ return infoStream;
+ }
+
+ /**
+ * Prints the given message to the infoStream. Note, this method does not
+ * check for null infoStream. It assumes this check has been performed by the
+ * caller, which is recommended to avoid the (usually) expensive message
+ * creation.
+ */
+ private static void message(String message) {
+ infoStream.println("SIS [" + Thread.currentThread().getName() + "]: " + message);
+ }
+
+ /**
+ * Utility class for executing code that needs to do
+ * something with the current segments file. This is
+ * necessary with lock-less commits because from the time
+ * you locate the current segments file name, until you
+ * actually open it, read its contents, or check modified
+ * time, etc., it could have been deleted due to a writer
+ * commit finishing.
+ */
+ public abstract static class FindSegmentsFile {
+
+ final Directory directory;
+
+ public FindSegmentsFile(Directory directory) {
+ this.directory = directory;
+ }
+
+ public Object run() throws CorruptIndexException, IOException {
+ return run(null);
+ }
+
+ public Object run(IndexCommit commit) throws CorruptIndexException, IOException {
+ if (commit != null) {
+ if (directory != commit.getDirectory())
+ throw new IOException("the specified commit does not match the specified Directory");
+ return doBody(commit.getSegmentsFileName());
+ }
+
+ String segmentFileName = null;
+ long lastGen = -1;
+ long gen = 0;
+ int genLookaheadCount = 0;
+ IOException exc = null;
+ int retryCount = 0;
+
+ boolean useFirstMethod = true;
+
+ // Loop until we succeed in calling doBody() without
+ // hitting an IOException. An IOException most likely
+ // means a commit was in process and has finished, in
+ // the time it took us to load the now-old infos files
+ // (and segments files). It's also possible it's a
+ // true error (corrupt index). To distinguish these,
+ // on each retry we must see "forward progress" on
+ // which generation we are trying to load. If we
+ // don't, then the original error is real and we throw
+ // it.
+
+ // We have three methods for determining the current
+ // generation. We try the first two in parallel (when
+ // useFirstMethod is true), and fall back to the third
+ // when necessary.
+
+ while(true) {
+
+ if (useFirstMethod) {
+
+ // List the directory and use the highest
+ // segments_N file. This method works well as long
+ // as there is no stale caching on the directory
+ // contents (NOTE: NFS clients often have such stale
+ // caching):
+ String[] files = null;
+
+ long genA = -1;
+
+ files = directory.listAll();
+
+ if (files != null) {
+ genA = getCurrentSegmentGeneration(files);
+ }
+
+ if (infoStream != null) {
+ message("directory listing genA=" + genA);
+ }
+
+ // Also open segments.gen and read its
+ // contents. Then we take the larger of the two
+ // gens. This way, if either approach is hitting
+ // a stale cache (NFS) we have a better chance of
+ // getting the right generation.
+ long genB = -1;
+ for(int i=0;i<defaultGenFileRetryCount;i++) {
+ IndexInput genInput = null;
+ try {
+ genInput = directory.openInput(IndexFileNames.SEGMENTS_GEN);
+ } catch (FileNotFoundException e) {
+ if (infoStream != null) {
+ message("segments.gen open: FileNotFoundException " + e);
+ }
+ break;
+ } catch (IOException e) {
+ if (infoStream != null) {
+ message("segments.gen open: IOException " + e);
+ }
+ }
+
+ if (genInput != null) {
+ try {
+ int version = genInput.readInt();
+ if (version == FORMAT_LOCKLESS) {
+ long gen0 = genInput.readLong();
+ long gen1 = genInput.readLong();
+ if (infoStream != null) {
+ message("fallback check: " + gen0 + "; " + gen1);
+ }
+ if (gen0 == gen1) {
+ // The file is consistent.
+ genB = gen0;
+ break;
+ }
+ }
+ } catch (IOException err2) {
+ // will retry
+ } finally {
+ genInput.close();
+ }
+ }
+ try {
+ Thread.sleep(defaultGenFileRetryPauseMsec);
+ } catch (InterruptedException ie) {
+ throw new ThreadInterruptedException(ie);
+ }
+ }
+
+ if (infoStream != null) {
+ message(IndexFileNames.SEGMENTS_GEN + " check: genB=" + genB);
+ }
+
+ // Pick the larger of the two gen's:
+ if (genA > genB)
+ gen = genA;
+ else
+ gen = genB;
+
+ if (gen == -1) {
+ // Neither approach found a generation
+ throw new IndexNotFoundException("no segments* file found in " + directory + ": files: " + Arrays.toString(files));
+ }
+ }
+
+ if (useFirstMethod && lastGen == gen && retryCount >= 2) {
+ // Give up on first method -- this is 3rd cycle on
+ // listing directory and checking gen file to
+ // attempt to locate the segments file.
+ useFirstMethod = false;
+ }
+
+ // Second method: since both directory cache and
+ // file contents cache seem to be stale, just
+ // advance the generation.
+ if (!useFirstMethod) {
+ if (genLookaheadCount < defaultGenLookaheadCount) {
+ gen++;
+ genLookaheadCount++;
+ if (infoStream != null) {
+ message("look ahead increment gen to " + gen);
+ }
+ } else {
+ // All attempts have failed -- throw first exc:
+ throw exc;
+ }
+ } else if (lastGen == gen) {
+ // This means we're about to try the same
+ // segments_N last tried.
+ retryCount++;
+ } else {
+ // Segment file has advanced since our last loop
+ // (we made "progress"), so reset retryCount:
+ retryCount = 0;
+ }
+
+ lastGen = gen;
+
+ segmentFileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
+ "",
+ gen);
+
+ try {
+ Object v = doBody(segmentFileName);
+ if (infoStream != null) {
+ message("success on " + segmentFileName);
+ }
+ return v;
+ } catch (IOException err) {
+
+ // Save the original root cause:
+ if (exc == null) {
+ exc = err;
+ }
+
+ if (infoStream != null) {
+ message("primary Exception on '" + segmentFileName + "': " + err + "'; will retry: retryCount=" + retryCount + "; gen = " + gen);
+ }
+
+ if (gen > 1 && useFirstMethod && retryCount == 1) {
+
+ // This is our second time trying this same segments
+ // file (because retryCount is 1), and, there is
+ // possibly a segments_(N-1) (because gen > 1).
+ // So, check if the segments_(N-1) exists and
+ // try it if so:
+ String prevSegmentFileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
+ "",
+ gen-1);
+
+ final boolean prevExists;
+ prevExists = directory.fileExists(prevSegmentFileName);
+
+ if (prevExists) {
+ if (infoStream != null) {
+ message("fallback to prior segment file '" + prevSegmentFileName + "'");
+ }
+ try {
+ Object v = doBody(prevSegmentFileName);
+ if (infoStream != null) {
+ message("success on fallback " + prevSegmentFileName);
+ }
+ return v;
+ } catch (IOException err2) {
+ if (infoStream != null) {
+ message("secondary Exception on '" + prevSegmentFileName + "': " + err2 + "'; will retry");
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Subclass must implement this. The assumption is an
+ * IOException will be thrown if something goes wrong
+ * during the processing that could have been caused by
+ * a writer committing.
+ */
+ protected abstract Object doBody(String segmentFileName) throws CorruptIndexException, IOException;
+ }
+
+ /**
+ * Returns a new SegmentInfos containing the SegmentInfo
+ * instances in the specified range first (inclusive) to
+ * last (exclusive), so total number of segments returned
+ * is last-first.
+ * @deprecated use {@code asList().subList(first, last)}
+ * instead.
+ */
+ @Deprecated
+ public SegmentInfos range(int first, int last) {
+ SegmentInfos infos = new SegmentInfos();
+ infos.addAll(segments.subList(first, last));
+ return infos;
+ }
+
+ // Carry over generation numbers from another SegmentInfos
+ void updateGeneration(SegmentInfos other) {
+ lastGeneration = other.lastGeneration;
+ generation = other.generation;
+ }
+
+ final void rollbackCommit(Directory dir) throws IOException {
+ if (pendingSegnOutput != null) {
+ try {
+ pendingSegnOutput.close();
+ } catch (Throwable t) {
+ // Suppress so we keep throwing the original exception
+ // in our caller
+ }
+
+ // Must carefully compute fileName from "generation"
+ // since lastGeneration isn't incremented:
+ try {
+ final String segmentFileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
+ "",
+ generation);
+ dir.deleteFile(segmentFileName);
+ } catch (Throwable t) {
+ // Suppress so we keep throwing the original exception
+ // in our caller
+ }
+ pendingSegnOutput = null;
+ }
+ }
+
+ /** Call this to start a commit. This writes the new
+ * segments file, but writes an invalid checksum at the
+ * end, so that it is not visible to readers. Once this
+ * is called you must call {@link #finishCommit} to complete
+ * the commit or {@link #rollbackCommit} to abort it.
+ * <p>
+ * Note: {@link #changed()} should be called prior to this
+ * method if changes have been made to this {@link SegmentInfos} instance
+ * </p>
+ **/
+ final void prepareCommit(Directory dir) throws IOException {
+ if (pendingSegnOutput != null)
+ throw new IllegalStateException("prepareCommit was already called");
+ write(dir);
+ }
+
+ /** Returns all file names referenced by SegmentInfo
+ * instances matching the provided Directory (ie files
+ * associated with any "external" segments are skipped).
+ * The returned collection is recomputed on each
+ * invocation. */
+ public Collection<String> files(Directory dir, boolean includeSegmentsFile) throws IOException {
+ HashSet<String> files = new HashSet<String>();
+ if (includeSegmentsFile) {
+ files.add(getCurrentSegmentFileName());
+ }
+ final int size = size();
+ for(int i=0;i<size;i++) {
+ final SegmentInfo info = info(i);
+ if (info.dir == dir) {
+ files.addAll(info(i).files());
+ }
+ }
+ return files;
+ }
+
+ final void finishCommit(Directory dir) throws IOException {
+ if (pendingSegnOutput == null)
+ throw new IllegalStateException("prepareCommit was not called");
+ boolean success = false;
+ try {
+ pendingSegnOutput.finishCommit();
+ pendingSegnOutput.close();
+ pendingSegnOutput = null;
+ success = true;
+ } finally {
+ if (!success)
+ rollbackCommit(dir);
+ }
+
+ // NOTE: if we crash here, we have left a segments_N
+ // file in the directory in a possibly corrupt state (if
+ // some bytes made it to stable storage and others
+ // didn't). But, the segments_N file includes checksum
+ // at the end, which should catch this case. So when a
+ // reader tries to read it, it will throw a
+ // CorruptIndexException, which should cause the retry
+ // logic in SegmentInfos to kick in and load the last
+ // good (previous) segments_N-1 file.
+
+ final String fileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
+ "",
+ generation);
+ success = false;
+ try {
+ dir.sync(Collections.singleton(fileName));
+ success = true;
+ } finally {
+ if (!success) {
+ try {
+ dir.deleteFile(fileName);
+ } catch (Throwable t) {
+ // Suppress so we keep throwing the original exception
+ }
+ }
+ }
+
+ lastGeneration = generation;
+
+ try {
+ IndexOutput genOutput = dir.createOutput(IndexFileNames.SEGMENTS_GEN);
+ try {
+ genOutput.writeInt(FORMAT_LOCKLESS);
+ genOutput.writeLong(generation);
+ genOutput.writeLong(generation);
+ } finally {
+ genOutput.close();
+ }
+ } catch (ThreadInterruptedException t) {
+ throw t;
+ } catch (Throwable t) {
+ // It's OK if we fail to write this file since it's
+ // used only as one of the retry fallbacks.
+ }
+ }
+
+ /** Writes & syncs to the Directory dir, taking care to
+ * remove the segments file on exception
+ * <p>
+ * Note: {@link #changed()} should be called prior to this
+ * method if changes have been made to this {@link SegmentInfos} instance
+ * </p>
+ **/
+ final void commit(Directory dir) throws IOException {
+ prepareCommit(dir);
+ finishCommit(dir);
+ }
+
+ public String toString(Directory directory) {
+ StringBuilder buffer = new StringBuilder();
+ buffer.append(getCurrentSegmentFileName()).append(": ");
+ final int count = size();
+ for(int i = 0; i < count; i++) {
+ if (i > 0) {
+ buffer.append(' ');
+ }
+ final SegmentInfo info = info(i);
+ buffer.append(info.toString(directory, 0));
+ }
+ return buffer.toString();
+ }
+
+ public Map<String,String> getUserData() {
+ return userData;
+ }
+
+ void setUserData(Map<String,String> data) {
+ if (data == null) {
+ userData = Collections.<String,String>emptyMap();
+ } else {
+ userData = data;
+ }
+ }
+
+ /** Replaces all segments in this instance, but keeps
+ * generation, version, counter so that future commits
+ * remain write once.
+ */
+ void replace(SegmentInfos other) {
+ rollbackSegmentInfos(other.asList());
+ lastGeneration = other.lastGeneration;
+ }
+
+ /** Returns sum of all segment's docCounts. Note that
+ * this does not include deletions */
+ public int totalDocCount() {
+ int count = 0;
+ for(SegmentInfo info : this) {
+ count += info.docCount;
+ }
+ return count;
+ }
+
+ /** Call this before committing if changes have been made to the
+ * segments. */
+ public void changed() {
+ version++;
+ }
+
+ /** applies all changes caused by committing a merge to this SegmentInfos */
+ void applyMergeChanges(MergePolicy.OneMerge merge, boolean dropSegment) {
+ final Set<SegmentInfo> mergedAway = new HashSet<SegmentInfo>(merge.segments);
+ boolean inserted = false;
+ int newSegIdx = 0;
+ for (int segIdx = 0, cnt = segments.size(); segIdx < cnt; segIdx++) {
+ assert segIdx >= newSegIdx;
+ final SegmentInfo info = segments.get(segIdx);
+ if (mergedAway.contains(info)) {
+ if (!inserted && !dropSegment) {
+ segments.set(segIdx, merge.info);
+ inserted = true;
+ newSegIdx++;
+ }
+ } else {
+ segments.set(newSegIdx, info);
+ newSegIdx++;
+ }
+ }
+
+ // Either we found place to insert segment, or, we did
+ // not, but only because all segments we merged became
+ // deleted while we are merging, in which case it should
+ // be the case that the new segment is also all deleted,
+ // we insert it at the beginning if it should not be dropped:
+ if (!inserted && !dropSegment) {
+ segments.add(0, merge.info);
+ }
+
+ // the rest of the segments in list are duplicates, so don't remove from map, only list!
+ segments.subList(newSegIdx, segments.size()).clear();
+
+ // update the Set
+ if (!dropSegment) {
+ segmentSet.add(merge.info);
+ }
+ segmentSet.removeAll(mergedAway);
+
+ assert segmentSet.size() == segments.size();
+ }
+
+ List<SegmentInfo> createBackupSegmentInfos(boolean cloneChildren) {
+ if (cloneChildren) {
+ final List<SegmentInfo> list = new ArrayList<SegmentInfo>(size());
+ for(final SegmentInfo info : this) {
+ list.add((SegmentInfo) info.clone());
+ }
+ return list;
+ } else {
+ return new ArrayList<SegmentInfo>(segments);
+ }
+ }
+
+ void rollbackSegmentInfos(List<SegmentInfo> infos) {
+ this.clear();
+ this.addAll(infos);
+ }
+
+ /** Returns an <b>unmodifiable</b> {@link Iterator} of contained segments in order. */
+ // @Override (comment out until Java 6)
+ public Iterator<SegmentInfo> iterator() {
+ return asList().iterator();
+ }
+
+ /** Returns all contained segments as an <b>unmodifiable</b> {@link List} view. */
+ public List<SegmentInfo> asList() {
+ if (cachedUnmodifiableList == null) {
+ cachedUnmodifiableList = Collections.unmodifiableList(segments);
+ }
+ return cachedUnmodifiableList;
+ }
+
+ /** Returns all contained segments as an <b>unmodifiable</b> {@link Set} view.
+ * The iterator is not sorted, use {@link List} view or {@link #iterator} to get all segments in order. */
+ public Set<SegmentInfo> asSet() {
+ if (cachedUnmodifiableSet == null) {
+ cachedUnmodifiableSet = Collections.unmodifiableSet(segmentSet);
+ }
+ return cachedUnmodifiableSet;
+ }
+
+ public int size() {
+ return segments.size();
+ }
+
+ public void add(SegmentInfo si) {
+ if (segmentSet.contains(si)) {
+ throw new IllegalStateException("Cannot add the same segment two times to this SegmentInfos instance");
+ }
+ segments.add(si);
+ segmentSet.add(si);
+ assert segmentSet.size() == segments.size();
+ }
+
+ public void addAll(Iterable<SegmentInfo> sis) {
+ for (final SegmentInfo si : sis) {
+ this.add(si);
+ }
+ }
+
+ public void clear() {
+ segments.clear();
+ segmentSet.clear();
+ }
+
+ public void remove(SegmentInfo si) {
+ final int index = this.indexOf(si);
+ if (index >= 0) {
+ this.remove(index);
+ }
+ }
+
+ public void remove(int index) {
+ segmentSet.remove(segments.remove(index));
+ assert segmentSet.size() == segments.size();
+ }
+
+ public boolean contains(SegmentInfo si) {
+ return segmentSet.contains(si);
+ }
+
+ public int indexOf(SegmentInfo si) {
+ if (segmentSet.contains(si)) {
+ return segments.indexOf(si);
+ } else {
+ return -1;
+ }
+ }
+
+}
fnp / pylucene.git / blobdiff