X-Git-Url: https://git.mdrn.pl/pylucene.git/blobdiff_plain/a2e61f0c04805cfcb8706176758d1283c7e3a55c..aaeed5504b982cf3545252ab528713250aa33eed:/lucene-java-3.5.0/lucene/src/java/org/apache/lucene/index/SegmentInfos.java?ds=sidebyside

diff --git a/lucene-java-3.5.0/lucene/src/java/org/apache/lucene/index/SegmentInfos.java b/lucene-java-3.5.0/lucene/src/java/org/apache/lucene/index/SegmentInfos.java
new file mode 100644
index 0000000..46cf77a
--- /dev/null
+++ b/lucene-java-3.5.0/lucene/src/java/org/apache/lucene/index/SegmentInfos.java
@@ -0,0 +1,1147 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.ChecksumIndexOutput;
+import org.apache.lucene.store.ChecksumIndexInput;
+import org.apache.lucene.store.NoSuchDirectoryException;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.ThreadInterruptedException;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A collection of segmentInfo objects with methods for operating on
+ * those segments in relation to the file system.
+ * 
+ * @lucene.experimental
+ */
+public final class SegmentInfos implements Cloneable, Iterable<SegmentInfo> {
+
+  /** The file format version, a negative number. */
+  /* Works since counter, the old 1st entry, is always >= 0 */
+  public static final int FORMAT = -1;
+
+  /** This format adds details used for lockless commits.  It differs
+   * slightly from the previous format in that file names
+   * are never re-used (write once).  Instead, each file is
+   * written to the next generation.  For example,
+   * segments_1, segments_2, etc.  This allows us to not use
+   * a commit lock.  See <a
+   * href="http://lucene.apache.org/java/docs/fileformats.html">file
+   * formats</a> for details.
+   */
+  public static final int FORMAT_LOCKLESS = -2;
+
+  /** This format adds a "hasSingleNormFile" flag into each segment info.
+   * See <a href="http://issues.apache.org/jira/browse/LUCENE-756">LUCENE-756</a>
+   * for details.
+   */
+  public static final int FORMAT_SINGLE_NORM_FILE = -3;
+
+  /** This format allows multiple segments to share a single
+   * vectors and stored fields file. */
+  public static final int FORMAT_SHARED_DOC_STORE = -4;
+
+  /** This format adds a checksum at the end of the file to
+   *  ensure all bytes were successfully written. */
+  public static final int FORMAT_CHECKSUM = -5;
+
+  /** This format adds the deletion count for each segment.
+   *  This way IndexWriter can efficiently report numDocs(). */
+  public static final int FORMAT_DEL_COUNT = -6;
+
+  /** This format adds the boolean hasProx to record if any
+   *  fields in the segment store prox information (ie, have
+   *  omitTermFreqAndPositions==false) */
+  public static final int FORMAT_HAS_PROX = -7;
+
+  /** This format adds optional commit userData (String) storage. */
+  public static final int FORMAT_USER_DATA = -8;
+
+  /** This format adds optional per-segment String
+   *  diagnostics storage, and switches userData to Map */
+  public static final int FORMAT_DIAGNOSTICS = -9;
+
+  /** Each segment records whether it has term vectors */
+  public static final int FORMAT_HAS_VECTORS = -10;
+
+  /** Each segment records the Lucene version that created it. */
+  public static final int FORMAT_3_1 = -11;
+  
+  /* This must always point to the most recent file format. */
+  public static final int CURRENT_FORMAT = FORMAT_3_1;
+
+  public static final int FORMAT_MINIMUM = FORMAT;
+  public static final int FORMAT_MAXIMUM = CURRENT_FORMAT;
+  
+  public int counter = 0;    // used to name new segments
+  /**
+   * counts how often the index has been changed by adding or deleting docs.
+   * starting with the current time in milliseconds forces to create unique version numbers.
+   */
+  long version = System.currentTimeMillis();
+
+  private long generation = 0;     // generation of the "segments_N" for the next commit
+  private long lastGeneration = 0; // generation of the "segments_N" file we last successfully read
+                                   // or wrote; this is normally the same as generation except if
+                                   // there was an IOException that had interrupted a commit
+
+  private Map<String,String> userData = Collections.<String,String>emptyMap();       // Opaque Map<String, String> that user can specify during IndexWriter.commit
+
+  private int format;
+  
+  private List<SegmentInfo> segments = new ArrayList<SegmentInfo>();
+  private Set<SegmentInfo> segmentSet = new HashSet<SegmentInfo>();
+  private transient List<SegmentInfo> cachedUnmodifiableList;
+  private transient Set<SegmentInfo> cachedUnmodifiableSet;  
+  
+  /**
+   * If non-null, information about loading segments_N files
+   * will be printed here.  @see #setInfoStream.
+   */
+  private static PrintStream infoStream = null;
+
+  public void setFormat(int format) {
+    this.format = format;
+  }
+
+  public int getFormat() {
+    return format;
+  }
+
+  public SegmentInfo info(int i) {
+    return segments.get(i);
+  }
+
+  /**
+   * Get the generation (N) of the current segments_N file
+   * from a list of files.
+   *
+   * @param files -- array of file names to check
+   */
+  public static long getCurrentSegmentGeneration(String[] files) {
+    if (files == null) {
+      return -1;
+    }
+    long max = -1;
+    for (int i = 0; i < files.length; i++) {
+      String file = files[i];
+      if (file.startsWith(IndexFileNames.SEGMENTS) && !file.equals(IndexFileNames.SEGMENTS_GEN)) {
+        long gen = generationFromSegmentsFileName(file);
+        if (gen > max) {
+          max = gen;
+        }
+      }
+    }
+    return max;
+  }
+
+  /**
+   * Get the generation (N) of the current segments_N file
+   * in the directory.
+   *
+   * @param directory -- directory to search for the latest segments_N file
+   */
+  public static long getCurrentSegmentGeneration(Directory directory) throws IOException {
+    try {
+      return getCurrentSegmentGeneration(directory.listAll());
+    } catch (NoSuchDirectoryException nsde) {
+      return -1;
+    }
+  }
+
+  /**
+   * Get the filename of the current segments_N file
+   * from a list of files.
+   *
+   * @param files -- array of file names to check
+   */
+
+  public static String getCurrentSegmentFileName(String[] files) throws IOException {
+    return IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
+                                                 "",
+                                                 getCurrentSegmentGeneration(files));
+  }
+
+  /**
+   * Get the filename of the current segments_N file
+   * in the directory.
+   *
+   * @param directory -- directory to search for the latest segments_N file
+   */
+  public static String getCurrentSegmentFileName(Directory directory) throws IOException {
+    return IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
+                                                 "",
+                                                 getCurrentSegmentGeneration(directory));
+  }
+
+  /**
+   * Get the segments_N filename in use by this segment infos.
+   */
+  public String getCurrentSegmentFileName() {
+    return IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
+                                                 "",
+                                                 lastGeneration);
+  }
+
+  /**
+   * Parse the generation off the segments file name and
+   * return it.
+   */
+  public static long generationFromSegmentsFileName(String fileName) {
+    if (fileName.equals(IndexFileNames.SEGMENTS)) {
+      return 0;
+    } else if (fileName.startsWith(IndexFileNames.SEGMENTS)) {
+      return Long.parseLong(fileName.substring(1+IndexFileNames.SEGMENTS.length()),
+                            Character.MAX_RADIX);
+    } else {
+      throw new IllegalArgumentException("fileName \"" + fileName + "\" is not a segments file");
+    }
+  }
+
+
+  /**
+   * Get the next segments_N filename that will be written.
+   */
+  public String getNextSegmentFileName() {
+    long nextGeneration;
+
+    if (generation == -1) {
+      nextGeneration = 1;
+    } else {
+      nextGeneration = generation+1;
+    }
+    return IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
+                                                 "",
+                                                 nextGeneration);
+  }
+
+  /**
+   * Read a particular segmentFileName.  Note that this may
+   * throw an IOException if a commit is in process.
+   *
+   * @param directory -- directory containing the segments file
+   * @param segmentFileName -- segment file to load
+   * @throws CorruptIndexException if the index is corrupt
+   * @throws IOException if there is a low-level IO error
+   */
+  public final void read(Directory directory, String segmentFileName) throws CorruptIndexException, IOException {
+    boolean success = false;
+
+    // Clear any previous segments:
+    this.clear();
+
+    ChecksumIndexInput input = new ChecksumIndexInput(directory.openInput(segmentFileName));
+
+    generation = generationFromSegmentsFileName(segmentFileName);
+
+    lastGeneration = generation;
+
+    try {
+      int format = input.readInt();
+      // check that it is a format we can understand
+      if (format > FORMAT_MINIMUM) {
+        throw new IndexFormatTooOldException(input, format,
+          FORMAT_MINIMUM, FORMAT_MAXIMUM);
+      }
+      if (format < FORMAT_MAXIMUM) {
+        throw new IndexFormatTooNewException(input, format,
+          FORMAT_MINIMUM, FORMAT_MAXIMUM);
+      }
+      version = input.readLong(); // read version
+      counter = input.readInt(); // read counter
+      
+      for (int i = input.readInt(); i > 0; i--) { // read segmentInfos
+        SegmentInfo si = new SegmentInfo(directory, format, input);
+        if (si.getVersion() == null) {
+          // It's a pre-3.1 segment, upgrade its version to either 3.0 or 2.x
+          Directory dir = directory;
+          if (si.getDocStoreOffset() != -1) {
+            if (si.getDocStoreIsCompoundFile()) {
+              dir = new CompoundFileReader(dir, IndexFileNames.segmentFileName(
+                  si.getDocStoreSegment(),
+                  IndexFileNames.COMPOUND_FILE_STORE_EXTENSION), 1024);
+            }
+          } else if (si.getUseCompoundFile()) {
+            dir = new CompoundFileReader(dir, IndexFileNames.segmentFileName(
+                si.name, IndexFileNames.COMPOUND_FILE_EXTENSION), 1024);
+          }
+
+          try {
+            String store = si.getDocStoreOffset() != -1 ? si.getDocStoreSegment() : si.name;
+            si.setVersion(FieldsReader.detectCodeVersion(dir, store));
+          } finally {
+            // If we opened the directory, close it
+            if (dir != directory) dir.close();
+          }
+        }
+        add(si);
+      }
+      
+      if(format >= 0){    // in old format the version number may be at the end of the file
+        if (input.getFilePointer() >= input.length())
+          version = System.currentTimeMillis(); // old file format without version number
+        else
+          version = input.readLong(); // read version
+      }
+
+      if (format <= FORMAT_USER_DATA) {
+        if (format <= FORMAT_DIAGNOSTICS) {
+          userData = input.readStringStringMap();
+        } else if (0 != input.readByte()) {
+          userData = Collections.singletonMap("userData", input.readString());
+        } else {
+          userData = Collections.<String,String>emptyMap();
+        }
+      } else {
+        userData = Collections.<String,String>emptyMap();
+      }
+
+      if (format <= FORMAT_CHECKSUM) {
+        final long checksumNow = input.getChecksum();
+        final long checksumThen = input.readLong();
+        if (checksumNow != checksumThen)
+          throw new CorruptIndexException("checksum mismatch in segments file (resource: " + input + ")");
+      }
+      success = true;
+    }
+    finally {
+      input.close();
+      if (!success) {
+        // Clear any segment infos we had loaded so we
+        // have a clean slate on retry:
+        this.clear();
+      }
+    }
+  }
+
+  /**
+   * This version of read uses the retry logic (for lock-less
+   * commits) to find the right segments file to load.
+   * @throws CorruptIndexException if the index is corrupt
+   * @throws IOException if there is a low-level IO error
+   */
+  public final void read(Directory directory) throws CorruptIndexException, IOException {
+
+    generation = lastGeneration = -1;
+
+    new FindSegmentsFile(directory) {
+
+      @Override
+      protected Object doBody(String segmentFileName) throws CorruptIndexException, IOException {
+        read(directory, segmentFileName);
+        return null;
+      }
+    }.run();
+  }
+
+  // Only non-null after prepareCommit has been called and
+  // before finishCommit is called
+  ChecksumIndexOutput pendingSegnOutput;
+
+  private final void write(Directory directory) throws IOException {
+
+    String segmentFileName = getNextSegmentFileName();
+
+    // Always advance the generation on write:
+    if (generation == -1) {
+      generation = 1;
+    } else {
+      generation++;
+    }
+
+    ChecksumIndexOutput segnOutput = new ChecksumIndexOutput(directory.createOutput(segmentFileName));
+
+    boolean success = false;
+
+    try {
+      segnOutput.writeInt(CURRENT_FORMAT); // write FORMAT
+      segnOutput.writeLong(version); 
+      segnOutput.writeInt(counter); // write counter
+      segnOutput.writeInt(size()); // write infos
+      for (SegmentInfo si : this) {
+        si.write(segnOutput);
+      }
+      segnOutput.writeStringStringMap(userData);
+      segnOutput.prepareCommit();
+      pendingSegnOutput = segnOutput;
+      success = true;
+    } finally {
+      if (!success) {
+        // We hit an exception above; try to close the file
+        // but suppress any exception:
+        IOUtils.closeWhileHandlingException(segnOutput);
+        try {
+          // Try not to leave a truncated segments_N file in
+          // the index:
+          directory.deleteFile(segmentFileName);
+        } catch (Throwable t) {
+          // Suppress so we keep throwing the original exception
+        }
+      }
+    }
+  }
+
+  /** Prunes any segment whose docs are all deleted. */
+  public void pruneDeletedSegments() throws IOException {
+    for(final Iterator<SegmentInfo> it = segments.iterator(); it.hasNext();) {
+      final SegmentInfo info = it.next();
+      if (info.getDelCount() == info.docCount) {
+        it.remove();
+        segmentSet.remove(info);
+      }
+    }
+    assert segmentSet.size() == segments.size();
+  }
+
+  /**
+   * Returns a copy of this instance, also copying each
+   * SegmentInfo.
+   */
+  
+  @Override
+  public Object clone() {
+    try {
+      final SegmentInfos sis = (SegmentInfos) super.clone();
+      // deep clone, first recreate all collections:
+      sis.segments = new ArrayList<SegmentInfo>(size());
+      sis.segmentSet = new HashSet<SegmentInfo>(size());
+      sis.cachedUnmodifiableList = null;
+      sis.cachedUnmodifiableSet = null;
+      for(final SegmentInfo info : this) {
+        // dont directly access segments, use add method!!!
+        sis.add((SegmentInfo) info.clone());
+      }
+      sis.userData = new HashMap<String,String>(userData);
+      return sis;
+    } catch (CloneNotSupportedException e) {
+      throw new RuntimeException("should not happen", e);
+    }
+  }
+
+  /**
+   * version number when this SegmentInfos was generated.
+   */
+  public long getVersion() {
+    return version;
+  }
+  public long getGeneration() {
+    return generation;
+  }
+  public long getLastGeneration() {
+    return lastGeneration;
+  }
+
+  /**
+   * Current version number from segments file.
+   * @throws CorruptIndexException if the index is corrupt
+   * @throws IOException if there is a low-level IO error
+   */
+  public static long readCurrentVersion(Directory directory)
+    throws CorruptIndexException, IOException {
+
+    // Fully read the segments file: this ensures that it's
+    // completely written so that if
+    // IndexWriter.prepareCommit has been called (but not
+    // yet commit), then the reader will still see itself as
+    // current:
+    SegmentInfos sis = new SegmentInfos();
+    sis.read(directory);
+    return sis.version;
+  }
+
+  /**
+   * Returns userData from latest segments file
+   * @throws CorruptIndexException if the index is corrupt
+   * @throws IOException if there is a low-level IO error
+   */
+  public static Map<String,String> readCurrentUserData(Directory directory)
+    throws CorruptIndexException, IOException {
+    SegmentInfos sis = new SegmentInfos();
+    sis.read(directory);
+    return sis.getUserData();
+  }
+
+  /** If non-null, information about retries when loading
+   * the segments file will be printed to this.
+   */
+  public static void setInfoStream(PrintStream infoStream) {
+    SegmentInfos.infoStream = infoStream;
+  }
+
+  /* Advanced configuration of retry logic in loading
+     segments_N file */
+  private static int defaultGenFileRetryCount = 10;
+  private static int defaultGenFileRetryPauseMsec = 50;
+  private static int defaultGenLookaheadCount = 10;
+
+  /**
+   * Advanced: set how many times to try loading the
+   * segments.gen file contents to determine current segment
+   * generation.  This file is only referenced when the
+   * primary method (listing the directory) fails.
+   */
+  public static void setDefaultGenFileRetryCount(int count) {
+    defaultGenFileRetryCount = count;
+  }
+
+  /**
+   * @see #setDefaultGenFileRetryCount
+   */
+  public static int getDefaultGenFileRetryCount() {
+    return defaultGenFileRetryCount;
+  }
+
+  /**
+   * Advanced: set how many milliseconds to pause in between
+   * attempts to load the segments.gen file.
+   */
+  public static void setDefaultGenFileRetryPauseMsec(int msec) {
+    defaultGenFileRetryPauseMsec = msec;
+  }
+
+  /**
+   * @see #setDefaultGenFileRetryPauseMsec
+   */
+  public static int getDefaultGenFileRetryPauseMsec() {
+    return defaultGenFileRetryPauseMsec;
+  }
+
+  /**
+   * Advanced: set how many times to try incrementing the
+   * gen when loading the segments file.  This only runs if
+   * the primary (listing directory) and secondary (opening
+   * segments.gen file) methods fail to find the segments
+   * file.
+   */
+  public static void setDefaultGenLookaheadCount(int count) {
+    defaultGenLookaheadCount = count;
+  }
+  /**
+   * @see #setDefaultGenLookaheadCount
+   */
+  public static int getDefaultGenLookahedCount() {
+    return defaultGenLookaheadCount;
+  }
+
+  /**
+   * @see #setInfoStream
+   */
+  public static PrintStream getInfoStream() {
+    return infoStream;
+  }
+
+  /**
+   * Prints the given message to the infoStream. Note, this method does not
+   * check for null infoStream. It assumes this check has been performed by the
+   * caller, which is recommended to avoid the (usually) expensive message
+   * creation.
+   */
+  private static void message(String message) {
+    infoStream.println("SIS [" + Thread.currentThread().getName() + "]: " + message);
+  }
+
+  /**
+   * Utility class for executing code that needs to do
+   * something with the current segments file.  This is
+   * necessary with lock-less commits because from the time
+   * you locate the current segments file name, until you
+   * actually open it, read its contents, or check modified
+   * time, etc., it could have been deleted due to a writer
+   * commit finishing.
+   */
+  public abstract static class FindSegmentsFile {
+    
+    final Directory directory;
+
+    public FindSegmentsFile(Directory directory) {
+      this.directory = directory;
+    }
+
+    public Object run() throws CorruptIndexException, IOException {
+      return run(null);
+    }
+    
+    public Object run(IndexCommit commit) throws CorruptIndexException, IOException {
+      if (commit != null) {
+        if (directory != commit.getDirectory())
+          throw new IOException("the specified commit does not match the specified Directory");
+        return doBody(commit.getSegmentsFileName());
+      }
+
+      String segmentFileName = null;
+      long lastGen = -1;
+      long gen = 0;
+      int genLookaheadCount = 0;
+      IOException exc = null;
+      int retryCount = 0;
+
+      boolean useFirstMethod = true;
+
+      // Loop until we succeed in calling doBody() without
+      // hitting an IOException.  An IOException most likely
+      // means a commit was in process and has finished, in
+      // the time it took us to load the now-old infos files
+      // (and segments files).  It's also possible it's a
+      // true error (corrupt index).  To distinguish these,
+      // on each retry we must see "forward progress" on
+      // which generation we are trying to load.  If we
+      // don't, then the original error is real and we throw
+      // it.
+      
+      // We have three methods for determining the current
+      // generation.  We try the first two in parallel (when
+      // useFirstMethod is true), and fall back to the third
+      // when necessary.
+
+      while(true) {
+
+        if (useFirstMethod) {
+
+          // List the directory and use the highest
+          // segments_N file.  This method works well as long
+          // as there is no stale caching on the directory
+          // contents (NOTE: NFS clients often have such stale
+          // caching):
+          String[] files = null;
+
+          long genA = -1;
+
+          files = directory.listAll();
+          
+          if (files != null) {
+            genA = getCurrentSegmentGeneration(files);
+          }
+          
+          if (infoStream != null) {
+            message("directory listing genA=" + genA);
+          }
+
+          // Also open segments.gen and read its
+          // contents.  Then we take the larger of the two
+          // gens.  This way, if either approach is hitting
+          // a stale cache (NFS) we have a better chance of
+          // getting the right generation.
+          long genB = -1;
+          for(int i=0;i<defaultGenFileRetryCount;i++) {
+            IndexInput genInput = null;
+            try {
+              genInput = directory.openInput(IndexFileNames.SEGMENTS_GEN);
+            } catch (FileNotFoundException e) {
+              if (infoStream != null) {
+                message("segments.gen open: FileNotFoundException " + e);
+              }
+              break;
+            } catch (IOException e) {
+              if (infoStream != null) {
+                message("segments.gen open: IOException " + e);
+              }
+            }
+  
+            if (genInput != null) {
+              try {
+                int version = genInput.readInt();
+                if (version == FORMAT_LOCKLESS) {
+                  long gen0 = genInput.readLong();
+                  long gen1 = genInput.readLong();
+                  if (infoStream != null) {
+                    message("fallback check: " + gen0 + "; " + gen1);
+                  }
+                  if (gen0 == gen1) {
+                    // The file is consistent.
+                    genB = gen0;
+                    break;
+                  }
+                }
+              } catch (IOException err2) {
+                // will retry
+              } finally {
+                genInput.close();
+              }
+            }
+            try {
+              Thread.sleep(defaultGenFileRetryPauseMsec);
+            } catch (InterruptedException ie) {
+              throw new ThreadInterruptedException(ie);
+            }
+          }
+
+          if (infoStream != null) {
+            message(IndexFileNames.SEGMENTS_GEN + " check: genB=" + genB);
+          }
+
+          // Pick the larger of the two gen's:
+          if (genA > genB)
+            gen = genA;
+          else
+            gen = genB;
+
+          if (gen == -1) {
+            // Neither approach found a generation
+            throw new IndexNotFoundException("no segments* file found in " + directory + ": files: " + Arrays.toString(files));
+          }
+        }
+
+        if (useFirstMethod && lastGen == gen && retryCount >= 2) {
+          // Give up on first method -- this is 3rd cycle on
+          // listing directory and checking gen file to
+          // attempt to locate the segments file.
+          useFirstMethod = false;
+        }
+
+        // Second method: since both directory cache and
+        // file contents cache seem to be stale, just
+        // advance the generation.
+        if (!useFirstMethod) {
+          if (genLookaheadCount < defaultGenLookaheadCount) {
+            gen++;
+            genLookaheadCount++;
+            if (infoStream != null) {
+              message("look ahead increment gen to " + gen);
+            }
+          } else {
+            // All attempts have failed -- throw first exc:
+            throw exc;
+          }
+        } else if (lastGen == gen) {
+          // This means we're about to try the same
+          // segments_N last tried.
+          retryCount++;
+        } else {
+          // Segment file has advanced since our last loop
+          // (we made "progress"), so reset retryCount:
+          retryCount = 0;
+        }
+
+        lastGen = gen;
+
+        segmentFileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
+                                                                "",
+                                                                gen);
+
+        try {
+          Object v = doBody(segmentFileName);
+          if (infoStream != null) {
+            message("success on " + segmentFileName);
+          }
+          return v;
+        } catch (IOException err) {
+
+          // Save the original root cause:
+          if (exc == null) {
+            exc = err;
+          }
+
+          if (infoStream != null) {
+            message("primary Exception on '" + segmentFileName + "': " + err + "'; will retry: retryCount=" + retryCount + "; gen = " + gen);
+          }
+
+          if (gen > 1 && useFirstMethod && retryCount == 1) {
+
+            // This is our second time trying this same segments
+            // file (because retryCount is 1), and, there is
+            // possibly a segments_(N-1) (because gen > 1).
+            // So, check if the segments_(N-1) exists and
+            // try it if so:
+            String prevSegmentFileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
+                                                                               "",
+                                                                               gen-1);
+
+            final boolean prevExists;
+            prevExists = directory.fileExists(prevSegmentFileName);
+
+            if (prevExists) {
+              if (infoStream != null) {
+                message("fallback to prior segment file '" + prevSegmentFileName + "'");
+              }
+              try {
+                Object v = doBody(prevSegmentFileName);
+                if (infoStream != null) {
+                  message("success on fallback " + prevSegmentFileName);
+                }
+                return v;
+              } catch (IOException err2) {
+                if (infoStream != null) {
+                  message("secondary Exception on '" + prevSegmentFileName + "': " + err2 + "'; will retry");
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+
+    /**
+     * Subclass must implement this.  The assumption is an
+     * IOException will be thrown if something goes wrong
+     * during the processing that could have been caused by
+     * a writer committing.
+     */
+    protected abstract Object doBody(String segmentFileName) throws CorruptIndexException, IOException;
+  }
+
+  /**
+   * Returns a new SegmentInfos containing the SegmentInfo
+   * instances in the specified range first (inclusive) to
+   * last (exclusive), so total number of segments returned
+   * is last-first.
+   * @deprecated use {@code asList().subList(first, last)}
+   * instead.
+   */
+  @Deprecated
+  public SegmentInfos range(int first, int last) {
+    SegmentInfos infos = new SegmentInfos();
+    infos.addAll(segments.subList(first, last));
+    return infos;
+  }
+
+  // Carry over generation numbers from another SegmentInfos
+  void updateGeneration(SegmentInfos other) {
+    lastGeneration = other.lastGeneration;
+    generation = other.generation;
+  }
+
+  final void rollbackCommit(Directory dir) throws IOException {
+    if (pendingSegnOutput != null) {
+      try {
+        pendingSegnOutput.close();
+      } catch (Throwable t) {
+        // Suppress so we keep throwing the original exception
+        // in our caller
+      }
+
+      // Must carefully compute fileName from "generation"
+      // since lastGeneration isn't incremented:
+      try {
+        final String segmentFileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
+                                                                              "",
+                                                                             generation);
+        dir.deleteFile(segmentFileName);
+      } catch (Throwable t) {
+        // Suppress so we keep throwing the original exception
+        // in our caller
+      }
+      pendingSegnOutput = null;
+    }
+  }
+
+  /** Call this to start a commit.  This writes the new
+   *  segments file, but writes an invalid checksum at the
+   *  end, so that it is not visible to readers.  Once this
+   *  is called you must call {@link #finishCommit} to complete
+   *  the commit or {@link #rollbackCommit} to abort it.
+   *  <p>
+   *  Note: {@link #changed()} should be called prior to this
+   *  method if changes have been made to this {@link SegmentInfos} instance
+   *  </p>  
+   **/
+  final void prepareCommit(Directory dir) throws IOException {
+    if (pendingSegnOutput != null)
+      throw new IllegalStateException("prepareCommit was already called");
+    write(dir);
+  }
+
+  /** Returns all file names referenced by SegmentInfo
+   *  instances matching the provided Directory (ie files
+   *  associated with any "external" segments are skipped).
+   *  The returned collection is recomputed on each
+   *  invocation.  */
+  public Collection<String> files(Directory dir, boolean includeSegmentsFile) throws IOException {
+    HashSet<String> files = new HashSet<String>();
+    if (includeSegmentsFile) {
+      files.add(getCurrentSegmentFileName());
+    }
+    final int size = size();
+    for(int i=0;i<size;i++) {
+      final SegmentInfo info = info(i);
+      if (info.dir == dir) {
+        files.addAll(info(i).files());
+      }
+    }
+    return files;
+  }
+
+  final void finishCommit(Directory dir) throws IOException {
+    if (pendingSegnOutput == null)
+      throw new IllegalStateException("prepareCommit was not called");
+    boolean success = false;
+    try {
+      pendingSegnOutput.finishCommit();
+      pendingSegnOutput.close();
+      pendingSegnOutput = null;
+      success = true;
+    } finally {
+      if (!success)
+        rollbackCommit(dir);
+    }
+
+    // NOTE: if we crash here, we have left a segments_N
+    // file in the directory in a possibly corrupt state (if
+    // some bytes made it to stable storage and others
+    // didn't).  But, the segments_N file includes checksum
+    // at the end, which should catch this case.  So when a
+    // reader tries to read it, it will throw a
+    // CorruptIndexException, which should cause the retry
+    // logic in SegmentInfos to kick in and load the last
+    // good (previous) segments_N-1 file.
+
+    final String fileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
+                                                                  "",
+                                                                  generation);
+    success = false;
+    try {
+      dir.sync(Collections.singleton(fileName));
+      success = true;
+    } finally {
+      if (!success) {
+        try {
+          dir.deleteFile(fileName);
+        } catch (Throwable t) {
+          // Suppress so we keep throwing the original exception
+        }
+      }
+    }
+
+    lastGeneration = generation;
+
+    try {
+      IndexOutput genOutput = dir.createOutput(IndexFileNames.SEGMENTS_GEN);
+      try {
+        genOutput.writeInt(FORMAT_LOCKLESS);
+        genOutput.writeLong(generation);
+        genOutput.writeLong(generation);
+      } finally {
+        genOutput.close();
+      }
+    } catch (ThreadInterruptedException t) {
+      throw t;
+    } catch (Throwable t) {
+      // It's OK if we fail to write this file since it's
+      // used only as one of the retry fallbacks.
+    }
+  }
+
+  /** Writes & syncs to the Directory dir, taking care to
+   *  remove the segments file on exception
+   *  <p>
+   *  Note: {@link #changed()} should be called prior to this
+   *  method if changes have been made to this {@link SegmentInfos} instance
+   *  </p>  
+   **/
+  final void commit(Directory dir) throws IOException {
+    prepareCommit(dir);
+    finishCommit(dir);
+  }
+
+  public String toString(Directory directory) {
+    StringBuilder buffer = new StringBuilder();
+    buffer.append(getCurrentSegmentFileName()).append(": ");
+    final int count = size();
+    for(int i = 0; i < count; i++) {
+      if (i > 0) {
+        buffer.append(' ');
+      }
+      final SegmentInfo info = info(i);
+      buffer.append(info.toString(directory, 0));
+    }
+    return buffer.toString();
+  }
+
+  public Map<String,String> getUserData() {
+    return userData;
+  }
+
+  void setUserData(Map<String,String> data) {
+    if (data == null) {
+      userData = Collections.<String,String>emptyMap();
+    } else {
+      userData = data;
+    }
+  }
+
+  /** Replaces all segments in this instance, but keeps
+   *  generation, version, counter so that future commits
+   *  remain write once.
+   */
+  void replace(SegmentInfos other) {
+    rollbackSegmentInfos(other.asList());
+    lastGeneration = other.lastGeneration;
+  }
+
+  /** Returns sum of all segment's docCounts.  Note that
+   *  this does not include deletions */
+  public int totalDocCount() {
+    int count = 0;
+    for(SegmentInfo info : this) {
+      count += info.docCount;
+    }
+    return count;
+  }
+
+  /** Call this before committing if changes have been made to the
+   *  segments. */
+  public void changed() {
+    version++;
+  }
+  
+  /** applies all changes caused by committing a merge to this SegmentInfos */
+  void applyMergeChanges(MergePolicy.OneMerge merge, boolean dropSegment) {
+    final Set<SegmentInfo> mergedAway = new HashSet<SegmentInfo>(merge.segments);
+    boolean inserted = false;
+    int newSegIdx = 0;
+    for (int segIdx = 0, cnt = segments.size(); segIdx < cnt; segIdx++) {
+      assert segIdx >= newSegIdx;
+      final SegmentInfo info = segments.get(segIdx);
+      if (mergedAway.contains(info)) {
+        if (!inserted && !dropSegment) {
+          segments.set(segIdx, merge.info);
+          inserted = true;
+          newSegIdx++;
+        }
+      } else {
+        segments.set(newSegIdx, info);
+        newSegIdx++;
+      }
+    }
+
+    // Either we found place to insert segment, or, we did
+    // not, but only because all segments we merged became
+    // deleted while we are merging, in which case it should
+    // be the case that the new segment is also all deleted,
+    // we insert it at the beginning if it should not be dropped:
+    if (!inserted && !dropSegment) {
+      segments.add(0, merge.info);
+    }
+
+    // the rest of the segments in list are duplicates, so don't remove from map, only list!
+    segments.subList(newSegIdx, segments.size()).clear();
+    
+    // update the Set
+    if (!dropSegment) {
+      segmentSet.add(merge.info);
+    }
+    segmentSet.removeAll(mergedAway);
+    
+    assert segmentSet.size() == segments.size();
+  }
+
+  List<SegmentInfo> createBackupSegmentInfos(boolean cloneChildren) {
+    if (cloneChildren) {
+      final List<SegmentInfo> list = new ArrayList<SegmentInfo>(size());
+      for(final SegmentInfo info : this) {
+        list.add((SegmentInfo) info.clone());
+      }
+      return list;
+    } else {
+      return new ArrayList<SegmentInfo>(segments);
+    }
+  }
+  
+  void rollbackSegmentInfos(List<SegmentInfo> infos) {
+    this.clear();
+    this.addAll(infos);
+  }
+  
+  /** Returns an <b>unmodifiable</b> {@link Iterator} of contained segments in order. */
+  // @Override (comment out until Java 6)
+  public Iterator<SegmentInfo> iterator() {
+    return asList().iterator();
+  }
+  
+  /** Returns all contained segments as an <b>unmodifiable</b> {@link List} view. */
+  public List<SegmentInfo> asList() {
+    if (cachedUnmodifiableList == null) {
+      cachedUnmodifiableList = Collections.unmodifiableList(segments);
+    }
+    return cachedUnmodifiableList;
+  }
+  
+  /** Returns all contained segments as an <b>unmodifiable</b> {@link Set} view.
+   * The iterator is not sorted, use {@link List} view or {@link #iterator} to get all segments in order. */
+  public Set<SegmentInfo> asSet() {
+    if (cachedUnmodifiableSet == null) {
+      cachedUnmodifiableSet = Collections.unmodifiableSet(segmentSet);
+    }
+    return cachedUnmodifiableSet;
+  }
+  
+  public int size() {
+    return segments.size();
+  }
+
+  public void add(SegmentInfo si) {
+    if (segmentSet.contains(si)) {
+      throw new IllegalStateException("Cannot add the same segment two times to this SegmentInfos instance");
+    }
+    segments.add(si);
+    segmentSet.add(si);
+    assert segmentSet.size() == segments.size();
+  }
+  
+  public void addAll(Iterable<SegmentInfo> sis) {
+    for (final SegmentInfo si : sis) {
+      this.add(si);
+    }
+  }
+  
+  public void clear() {
+    segments.clear();
+    segmentSet.clear();
+  }
+  
+  public void remove(SegmentInfo si) {
+    final int index = this.indexOf(si);
+    if (index >= 0) {
+      this.remove(index);
+    }
+  }
+  
+  public void remove(int index) {
+    segmentSet.remove(segments.remove(index));
+    assert segmentSet.size() == segments.size();
+  }
+  
+  public boolean contains(SegmentInfo si) {
+    return segmentSet.contains(si);
+  }
+
+  public int indexOf(SegmentInfo si) {
+    if (segmentSet.contains(si)) {
+      return segments.indexOf(si);
+    } else {
+      return -1;
+    }
+  }
+
+}