pylucene 3.5.0-3

[pylucene.git] / lucene-java-3.5.0 / lucene / contrib / facet / src / java / org / apache / lucene / facet / taxonomy / directory / DirectoryTaxonomyReader.java

package org.apache.lucene.facet.taxonomy.directory;

import java.io.IOException;
import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.logging.Level;
import java.util.logging.Logger;

import org.apache.lucene.facet.taxonomy.CategoryPath;
import org.apache.lucene.facet.taxonomy.InconsistentTaxonomyException;
import org.apache.lucene.facet.taxonomy.TaxonomyReader;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermDocs;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.collections.LRUHashMap;

/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/**
 * A {@link TaxonomyReader} which retrieves stored taxonomy information from a
 * {@link Directory}.
 * <P>
 * Reading from the on-disk index on every method call is too slow, so this
 * implementation employs caching: Some methods cache recent requests and their
 * results, while other methods prefetch all the data into memory and then
 * provide answers directly from in-memory tables. See the documentation of
 * individual methods for comments on their performance.
 * 
 * @lucene.experimental
 */
public class DirectoryTaxonomyReader implements TaxonomyReader {

  private static final Logger logger = Logger.getLogger(DirectoryTaxonomyReader.class.getName());
  
  private IndexReader indexReader;

  // The following lock is used to allow multiple threads to read from the
  // index concurrently, while having them block during the very short
  // critical moment of refresh() (see comments below). Note, however, that
  // we only read from the index when we don't have the entry in our cache,
  // and the caches are locked separately.
  private ReadWriteLock indexReaderLock = new ReentrantReadWriteLock();

  // The following are the limited-size LRU caches used to cache the latest
  // results from getOrdinal() and getLabel().
  // Because LRUHashMap is not thread-safe, we need to synchronize on this
  // object when using it. Unfortunately, this is not optimal under heavy
  // contention because it means that while one thread is using the cache
  // (reading or modifying) others are blocked from using it - or even
  // starting to do benign things like calculating the hash function. A more
  // efficient approach would be to use a non-locking (as much as possible)
  // concurrent solution, along the lines of java.util.concurrent.ConcurrentHashMap
  // but with LRU semantics.
  // However, even in the current sub-optimal implementation we do not make
  // the mistake of locking out readers while waiting for disk in a cache
  // miss - below, we do not hold cache lock while reading missing data from
  // disk.
  private final LRUHashMap<String, Integer> ordinalCache;
  private final LRUHashMap<Integer, String> categoryCache;

  // getParent() needs to be extremely efficient, to the point that we need
  // to fetch all the data in advance into memory, and answer these calls
  // from memory. Currently we use a large integer array, which is
  // initialized when the taxonomy is opened, and potentially enlarged
  // when it is refresh()ed.
  // These arrays are not syncrhonized. Rather, the reference to the array
  // is volatile, and the only writing operation (refreshPrefetchArrays)
  // simply creates a new array and replaces the reference. The volatility
  // of the reference ensures the correct atomic replacement and its
  // visibility properties (the content of the array is visible when the
  // new reference is visible).
  private ParentArray parentArray;

  private char delimiter = Consts.DEFAULT_DELIMITER;

  private volatile boolean closed = false;
  
  /**
   * Open for reading a taxonomy stored in a given {@link Directory}.
   * @param directory
   *    The {@link Directory} in which to the taxonomy lives. Note that
   *    the taxonomy is read directly to that directory (not from a
   *    subdirectory of it).
   * @throws CorruptIndexException if the Taxonomy is corrupted.
   * @throws IOException if another error occurred.
   */
  public DirectoryTaxonomyReader(Directory directory) throws IOException {
    this.indexReader = openIndexReader(directory);

    // These are the default cache sizes; they can be configured after
    // construction with the cache's setMaxSize() method
    ordinalCache = new LRUHashMap<String, Integer>(4000);
    categoryCache = new LRUHashMap<Integer, String>(4000);

    // TODO (Facet): consider lazily create parent array when asked, not in the constructor
    parentArray = new ParentArray();
    parentArray.refresh(indexReader);
  }

  protected IndexReader openIndexReader(Directory directory) throws CorruptIndexException, IOException {
    return IndexReader.open(directory);
  }

  /**
   * @throws AlreadyClosedException if this IndexReader is closed
   */
  protected final void ensureOpen() throws AlreadyClosedException {
    if (indexReader.getRefCount() <= 0) {
      throw new AlreadyClosedException("this TaxonomyReader is closed");
    }
  }
  
  /**
   * setCacheSize controls the maximum allowed size of each of the caches
   * used by {@link #getPath(int)} and {@link #getOrdinal(CategoryPath)}.
   * <P>
   * Currently, if the given size is smaller than the current size of
   * a cache, it will not shrink, and rather we be limited to its current
   * size.
   * @param size the new maximum cache size, in number of entries.
   */
  public void setCacheSize(int size) {
    ensureOpen();
    synchronized(categoryCache) {
      categoryCache.setMaxSize(size);
    }
    synchronized(ordinalCache) {
      ordinalCache.setMaxSize(size);
    }
  }

  /**
   * setDelimiter changes the character that the taxonomy uses in its
   * internal storage as a delimiter between category components. Do not
   * use this method unless you really know what you are doing.
   * <P>
   * If you do use this method, make sure you call it before any other
   * methods that actually queries the taxonomy. Moreover, make sure you
   * always pass the same delimiter for all LuceneTaxonomyWriter and
   * LuceneTaxonomyReader objects you create.
   */
  public void setDelimiter(char delimiter) {
    ensureOpen();
    this.delimiter = delimiter;
  }

  public int getOrdinal(CategoryPath categoryPath) throws IOException {
    ensureOpen();
    if (categoryPath.length()==0) {
      return ROOT_ORDINAL;
    }
    String path = categoryPath.toString(delimiter);

    // First try to find the answer in the LRU cache:
    synchronized(ordinalCache) {
      Integer res = ordinalCache.get(path);
      if (res!=null) {
        return res.intValue();
      }
    }

    // If we're still here, we have a cache miss. We need to fetch the
    // value from disk, and then also put it in the cache:
    int ret = TaxonomyReader.INVALID_ORDINAL;
    try {
      indexReaderLock.readLock().lock();
      TermDocs docs = indexReader.termDocs(new Term(Consts.FULL, path));
      if (docs.next()) {
        ret = docs.doc();
      }
    } finally {
      indexReaderLock.readLock().unlock();
    }

    // Put the new value in the cache. Note that it is possible that while
    // we were doing the above fetching (without the cache locked), some
    // other thread already added the same category to the cache. We do
    // not care about this possibilty, as LRUCache replaces previous values
    // of the same keys (it doesn't store duplicates).
    synchronized(ordinalCache) {
      // GB: new Integer(int); creates a new object each and every time.
      // Integer.valueOf(int) might not (See JavaDoc). 
      ordinalCache.put(path, Integer.valueOf(ret));
    }

    return ret;
  }

  public CategoryPath getPath(int ordinal) throws CorruptIndexException, IOException {
    ensureOpen();
    // TODO (Facet): Currently, the LRU cache we use (categoryCache) holds
    // strings with delimiters, not CategoryPath objects, so even if
    // we have a cache hit, we need to process the string and build a new
    // CategoryPath object every time. What is preventing us from putting
    // the actual CategoryPath object in the cache is the fact that these
    // objects are mutable. So we should create an immutable (read-only)
    // interface that CategoryPath implements, and this method should
    // return this interface, not the writable CategoryPath.
    String label = getLabel(ordinal);
    if (label==null) {
      return null;  
    }
    return new CategoryPath(label, delimiter);
  }

  public boolean getPath(int ordinal, CategoryPath result) throws CorruptIndexException, IOException {
    ensureOpen();
    String label = getLabel(ordinal);
    if (label==null) {
      return false;
    }
    result.clear();
    result.add(label, delimiter);
    return true;
  }

  private String getLabel(int catID) throws CorruptIndexException, IOException {
    ensureOpen();
    // First try to find the answer in the LRU cache. It is very
    // unfortunate that we need to allocate an Integer object here -
    // it would have been better if we used a hash table specifically
    // designed for int keys...
    // GB: new Integer(int); creates a new object each and every time.
    // Integer.valueOf(int) might not (See JavaDoc). 
    Integer catIDInteger = Integer.valueOf(catID);

    synchronized(categoryCache) {
      String res = categoryCache.get(catIDInteger);
      if (res!=null) {
        return res;
      }
    }

    // If we're still here, we have a cache miss. We need to fetch the
    // value from disk, and then also put it in the cache:
    String ret;
    try {
      indexReaderLock.readLock().lock();
      // The taxonomy API dictates that if we get an invalid category
      // ID, we should return null, If we don't check this here, we
      // can some sort of an exception from the document() call below.
      // NOTE: Currently, we *do not* cache this return value; There
      // isn't much point to do so, because checking the validity of
      // the docid doesn't require disk access - just comparing with
      // the number indexReader.maxDoc().
      if (catID<0 || catID>=indexReader.maxDoc()) {
        return null;
      }
      ret = indexReader.document(catID, Consts.fullPathSelector)
      .get(Consts.FULL);
    } finally {
      indexReaderLock.readLock().unlock();
    }
    // Put the new value in the cache. Note that it is possible that while
    // we were doing the above fetching (without the cache locked), some
    // other thread already added the same category to the cache. We do
    // not care about this possibility, as LRUCache replaces previous
    // values of the same keys (it doesn't store duplicates).
    synchronized (categoryCache) {
      categoryCache.put(catIDInteger, ret);
    }

    return ret;
  }

  public int getParent(int ordinal) {
    ensureOpen();
    // Note how we don't need to hold the read lock to do the following,
    // because the array reference is volatile, ensuring the correct
    // visibility and ordering: if we get the new reference, the new
    // data is also visible to this thread.
    return getParentArray()[ordinal];
  }

  /**
   * getParentArray() returns an int array of size getSize() listing the
   * ordinal of the parent category of each category in the taxonomy.
   * <P>
   * The caller can hold on to the array it got indefinitely - it is
   * guaranteed that no-one else will modify it. The other side of the
   * same coin is that the caller must treat the array it got as read-only
   * and <B>not modify it</B>, because other callers might have gotten the
   * same array too, and getParent() calls are also answered from the
   * same array.
   * <P>
   * The getParentArray() call is extremely efficient, merely returning
   * a reference to an array that already exists. For a caller that plans
   * to call getParent() for many categories, using getParentArray() and
   * the array it returns is a somewhat faster approach because it avoids
   * the overhead of method calls and volatile dereferencing.
   * <P>
   * If you use getParentArray() instead of getParent(), remember that
   * the array you got is (naturally) not modified after a refresh(),
   * so you should always call getParentArray() again after a refresh().
   */

  public int[] getParentArray() {
    ensureOpen();
    // Note how we don't need to hold the read lock to do the following,
    // because the array reference is volatile, ensuring the correct
    // visibility and ordering: if we get the new reference, the new
    // data is also visible to this thread.
    return parentArray.getArray();
  }

  // Note that refresh() is synchronized (it is the only synchronized
  // method in this class) to ensure that it never gets called concurrently
  // with itself.
  public synchronized boolean refresh() throws IOException, InconsistentTaxonomyException {
    ensureOpen();
    /*
     * Since refresh() can be a lengthy operation, it is very important that we
     * avoid locking out all readers for its duration. This is why we don't hold
     * the indexReaderLock write lock for the entire duration of this method. In
     * fact, it is enough to hold it only during a single assignment! Other
     * comments in this method will explain this.
     */

    // note that the lengthy operation indexReader.reopen() does not
    // modify the reader, so we can do it without holding a lock. We can
    // safely read indexReader without holding the write lock, because
    // no other thread can be writing at this time (this method is the
    // only possible writer, and it is "synchronized" to avoid this case).
    IndexReader r2 = IndexReader.openIfChanged(indexReader);
    if (r2 == null) {
        return false; // no changes, nothing to do
    } 
    
    // validate that a refresh is valid at this point, i.e. that the taxonomy 
    // was not recreated since this reader was last opened or refresshed.
    String t1 = indexReader.getCommitUserData().get(DirectoryTaxonomyWriter.INDEX_CREATE_TIME);
    String t2 = r2.getCommitUserData().get(DirectoryTaxonomyWriter.INDEX_CREATE_TIME);
    if (t1==null) {
        if (t2!=null) {
                r2.close();
                throw new InconsistentTaxonomyException("Taxonomy was recreated at: "+t2);
        }
    } else if (!t1.equals(t2)) {
        r2.close();
        throw new InconsistentTaxonomyException("Taxonomy was recreated at: "+t2+"  !=  "+t1);
    }
    
      IndexReader oldreader = indexReader;
      // we can close the old searcher, but need to synchronize this
      // so that we don't close it in the middle that another routine
      // is reading from it.
      indexReaderLock.writeLock().lock();
      indexReader = r2;
      indexReaderLock.writeLock().unlock();
      // We can close the old reader, but need to be certain that we
      // don't close it while another method is reading from it.
      // Luckily, we can be certain of that even without putting the
      // oldreader.close() in the locked section. The reason is that
      // after lock() succeeded above, we know that all existing readers
      // had finished (this is what a read-write lock ensures). New
      // readers, starting after the unlock() we just did, already got
      // the new indexReader we set above. So nobody can be possibly
      // using the old indexReader, and we can close it:
      oldreader.close();

      // We prefetch some of the arrays to make requests much faster.
      // Let's refresh these prefetched arrays; This refresh is much
      // is made more efficient by assuming that it is enough to read
      // the values for new categories (old categories could not have been
      // changed or deleted)
      // Note that this this done without the write lock being held,
      // which means that it is possible that during a refresh(), a
      // reader will have some methods (like getOrdinal and getCategory)
      // return fresh information, while getParent()
      // (only to be prefetched now) still return older information.
      // We consider this to be acceptable. The important thing,
      // however, is that refreshPrefetchArrays() itself writes to
      // the arrays in a correct manner (see discussion there)
      parentArray.refresh(indexReader);

      // Remove any INVALID_ORDINAL values from the ordinal cache,
      // because it is possible those are now answered by the new data!
      Iterator<Entry<String, Integer>> i = ordinalCache.entrySet().iterator();
      while (i.hasNext()) {
        Entry<String, Integer> e = i.next();
        if (e.getValue().intValue() == INVALID_ORDINAL) {
          i.remove();
        }
      }
      return true;
    }

  public void close() throws IOException {
    if (!closed) {
      decRef();
      closed = true;
    }
  }
  
  /** Do the actual closing, free up resources */
  private void doClose() throws IOException {
    indexReader.close();
    closed = true;

    parentArray = null;
    childrenArrays = null;
    categoryCache.clear();
    ordinalCache.clear();
  }

  public int getSize() {
    ensureOpen();
    indexReaderLock.readLock().lock();
    try {
      return indexReader.numDocs();
    } finally {
      indexReaderLock.readLock().unlock();
    }
  }

  public Map<String, String> getCommitUserData() {
    ensureOpen();
    return indexReader.getCommitUserData();
  }
  
  private ChildrenArrays childrenArrays;
  Object childrenArraysRebuild = new Object();

  public ChildrenArrays getChildrenArrays() {
    ensureOpen();
    // Check if the taxonomy grew since we built the array, and if it
    // did, create new (and larger) arrays and fill them as required.
    // We do all this under a lock, two prevent to concurrent calls to
    // needlessly do the same array building at the same time.
    synchronized(childrenArraysRebuild) {
      int num = getSize();
      int first;
      if (childrenArrays==null) {
        first = 0;
      } else {
        first = childrenArrays.getYoungestChildArray().length;
      }
      // If the taxonomy hasn't grown, we can return the existing object
      // immediately
      if (first == num) {
        return childrenArrays;
      }
      // Otherwise, build new arrays for a new ChildrenArray object.
      // These arrays start with an enlarged copy of the previous arrays,
      // and then are modified to take into account the new categories:
      int[] newYoungestChildArray = new int[num];
      int[] newOlderSiblingArray = new int[num];
      // In Java 6, we could just do Arrays.copyOf()...
      if (childrenArrays!=null) {
        System.arraycopy(childrenArrays.getYoungestChildArray(), 0,
            newYoungestChildArray, 0, childrenArrays.getYoungestChildArray().length);
        System.arraycopy(childrenArrays.getOlderSiblingArray(), 0,
            newOlderSiblingArray, 0, childrenArrays.getOlderSiblingArray().length);
      }
      int[] parents = getParentArray();
      for (int i=first; i<num; i++) {
        newYoungestChildArray[i] = INVALID_ORDINAL;
      }
      // In the loop below we can ignore the root category (0) because
      // it has no parent
      if (first==0) {
        first = 1;
        newOlderSiblingArray[0] = INVALID_ORDINAL;
      }
      for (int i=first; i<num; i++) {
        // Note that parents[i] is always < i, so the right-hand-side of
        // the following line is already set when we get here.
        newOlderSiblingArray[i] = newYoungestChildArray[parents[i]];
        newYoungestChildArray[parents[i]] = i;
      }
      // Finally switch to the new arrays
      childrenArrays = new ChildrenArraysImpl(newYoungestChildArray,
          newOlderSiblingArray);
      return childrenArrays;
    }
  }

  public String toString(int max) {
    ensureOpen();
    StringBuilder sb = new StringBuilder();
    int upperl = Math.min(max, this.indexReader.maxDoc());
    for (int i = 0; i < upperl; i++) {
      try {
        CategoryPath category = this.getPath(i);
        if (category == null) {
          sb.append(i + ": NULL!! \n");
          continue;
        } 
        if (category.length() == 0) {
          sb.append(i + ": EMPTY STRING!! \n");
          continue;
        }
        sb.append(i +": "+category.toString()+"\n");
      } catch (IOException e) {
        if (logger.isLoggable(Level.FINEST)) {
          logger.log(Level.FINEST, e.getMessage(), e);
        }
      }
    }
    return sb.toString();
  }

  private static final class ChildrenArraysImpl implements ChildrenArrays {
    private int[] youngestChildArray, olderSiblingArray;
    public ChildrenArraysImpl(int[] youngestChildArray, int[] olderSiblingArray) {
      this.youngestChildArray = youngestChildArray;
      this.olderSiblingArray = olderSiblingArray;
    }
    public int[] getOlderSiblingArray() {
      return olderSiblingArray;
    }
    public int[] getYoungestChildArray() {
      return youngestChildArray;
    }    
  }

  /**
   * Expert:  This method is only for expert use.
   * Note also that any call to refresh() will invalidate the returned reader,
   * so the caller needs to take care of appropriate locking.
   * 
   * @return lucene indexReader
   */
  IndexReader getInternalIndexReader() {
    ensureOpen();
    return this.indexReader;
  }

  /**
   * Expert: decreases the refCount of this TaxonomyReader instance. 
   * If the refCount drops to 0, then pending changes (if any) are 
   * committed to the taxonomy index and this reader is closed. 
   * @throws IOException 
   */
  public void decRef() throws IOException {
    ensureOpen();
    if (indexReader.getRefCount() == 1) {
      // Do not decRef the indexReader - doClose does it by calling reader.close()
      doClose();
    } else {
      indexReader.decRef();
    }
  }
  
  /**
   * Expert: returns the current refCount for this taxonomy reader
   */
  public int getRefCount() {
    ensureOpen();
    return this.indexReader.getRefCount();
  }
  
  /**
   * Expert: increments the refCount of this TaxonomyReader instance. 
   * RefCounts are used to determine when a taxonomy reader can be closed 
   * safely, i.e. as soon as there are no more references. 
   * Be sure to always call a corresponding decRef(), in a finally clause; 
   * otherwise the reader may never be closed. 
   */
  public void incRef() {
    ensureOpen();
    this.indexReader.incRef();
  }
}