--- /dev/null
+package org.apache.lucene.facet.search;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.facet.FacetTestUtils;
+import org.apache.lucene.facet.FacetTestUtils.IndexTaxonomyReaderPair;
+import org.apache.lucene.facet.FacetTestUtils.IndexTaxonomyWriterPair;
+import org.apache.lucene.facet.example.ExampleResult;
+import org.apache.lucene.facet.example.TestMultiCLExample;
+import org.apache.lucene.facet.example.multiCL.MultiCLIndexer;
+import org.apache.lucene.facet.example.multiCL.MultiCLSearcher;
+import org.apache.lucene.facet.index.CategoryDocumentBuilder;
+import org.apache.lucene.facet.index.params.DefaultFacetIndexingParams;
+import org.apache.lucene.facet.index.params.FacetIndexingParams;
+import org.apache.lucene.facet.search.TotalFacetCounts.CreationType;
+import org.apache.lucene.facet.search.results.FacetResult;
+import org.apache.lucene.facet.search.results.FacetResultNode;
+import org.apache.lucene.facet.taxonomy.CategoryPath;
+import org.apache.lucene.facet.taxonomy.TaxonomyReader;
+import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
+import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
+import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.SlowRAMDirectory;
+import org.apache.lucene.util._TestUtil;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public class TestTotalFacetCountsCache extends LuceneTestCase {
+
+ static final TotalFacetCountsCache TFC = TotalFacetCountsCache.getSingleton();
+
+ /**
+ * Thread class to be used in tests for this method. This thread gets a TFC
+ * and records times.
+ */
+ private static class TFCThread extends Thread {
+ private final IndexReader r;
+ private final DirectoryTaxonomyReader tr;
+ private final FacetIndexingParams iParams;
+
+ TotalFacetCounts tfc;
+
+ public TFCThread(IndexReader r, DirectoryTaxonomyReader tr, FacetIndexingParams iParams) {
+ this.r = r;
+ this.tr = tr;
+ this.iParams = iParams;
+ }
+ @Override
+ public void run() {
+ try {
+ tfc = TFC.getTotalCounts(r, tr, iParams, null);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ /** Utility method to add a document and facets to an index/taxonomy. */
+ static void addFacets(FacetIndexingParams iParams, IndexWriter iw,
+ TaxonomyWriter tw, String... strings) throws IOException {
+ ArrayList<CategoryPath> cps = new ArrayList<CategoryPath>();
+ cps.add(new CategoryPath(strings));
+ CategoryDocumentBuilder builder = new CategoryDocumentBuilder(tw, iParams);
+ iw.addDocument(builder.setCategoryPaths(cps).build(new Document()));
+ }
+
+ /** Clears the cache and sets its size to one. */
+ static void initCache() {
+ TFC.clear();
+ TFC.setCacheSize(1); // Set to keep one in memory
+ }
+
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ initCache();
+ }
+
+ /** runs a few instances of {@link MultiCLSearcher} in parallel */
+ public void testGeneralSynchronization() throws Exception {
+ int numIters = atLeast(2);
+ for (int i = 0; i < numIters; i++) {
+ doTestGeneralSynchronization(_TestUtil.nextInt(random, 2, 4),
+ random.nextBoolean() ? -1 : _TestUtil.nextInt(random, 1, 10),
+ _TestUtil.nextInt(random, 0, 3));
+ }
+ }
+
+ /**
+ * Run many instances of {@link MultiCLSearcher} in parallel, results should
+ * be sane. Each instance has a random delay for reading bytes, to ensure
+ * that threads finish in different order than started.
+ */
+ @Test @Nightly
+ public void testGeneralSynchronizationBig() throws Exception {
+ int[] numThreads = new int[] { 2, 3, 5, 8 };
+ int[] sleepMillis = new int[] { -1, 1, 20, 33 };
+ int[] cacheSize = new int[] { 0,1,2,3,5 };
+ for (int size : cacheSize) {
+ for (int sleep : sleepMillis) {
+ for (int nThreads : numThreads) {
+ doTestGeneralSynchronization(nThreads, sleep, size);
+ }
+ }
+ }
+ }
+
+ private void doTestGeneralSynchronization(int numThreads, int sleepMillis,
+ int cacheSize) throws Exception, CorruptIndexException, IOException,
+ InterruptedException {
+ TFC.setCacheSize(cacheSize);
+ SlowRAMDirectory slowIndexDir = new SlowRAMDirectory(-1, random);
+ MockDirectoryWrapper indexDir = new MockDirectoryWrapper(random, slowIndexDir);
+ SlowRAMDirectory slowTaxoDir = new SlowRAMDirectory(-1, random);
+ MockDirectoryWrapper taxoDir = new MockDirectoryWrapper(random, slowTaxoDir);
+
+
+ // Index documents without the "slowness"
+ MultiCLIndexer.index(indexDir, taxoDir);
+
+ slowIndexDir.setSleepMillis(sleepMillis);
+ slowTaxoDir.setSleepMillis(sleepMillis);
+
+ // Open the slow readers
+ IndexReader slowIndexReader = IndexReader.open(indexDir);
+ TaxonomyReader slowTaxoReader = new DirectoryTaxonomyReader(taxoDir);
+
+ // Class to perform search and return results as threads
+ class Multi extends Thread {
+ private List<FacetResult> results;
+ private FacetIndexingParams iParams;
+ private IndexReader indexReader;
+ private TaxonomyReader taxoReader;
+
+ public Multi(IndexReader indexReader, TaxonomyReader taxoReader,
+ FacetIndexingParams iParams) {
+ this.indexReader = indexReader;
+ this.taxoReader = taxoReader;
+ this.iParams = iParams;
+ }
+
+ public ExampleResult getResults() {
+ ExampleResult exampleRes = new ExampleResult();
+ exampleRes.setFacetResults(results);
+ return exampleRes;
+ }
+
+ @Override
+ public void run() {
+ try {
+ results = MultiCLSearcher.searchWithFacets(indexReader, taxoReader, iParams);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ // Instantiate threads, but do not start them
+ Multi[] multis = new Multi[numThreads];
+ for (int i = 0; i < numThreads - 1; i++) {
+ multis[i] = new Multi(slowIndexReader, slowTaxoReader, MultiCLIndexer.MULTI_IPARAMS);
+ }
+ // The last thread uses ONLY the DefaultFacetIndexingParams so that
+ // it references a different TFC cache. This will still result
+ // in valid results, but will only search one of the category lists
+ // instead of all of them.
+ multis[numThreads - 1] = new Multi(slowIndexReader, slowTaxoReader, new DefaultFacetIndexingParams());
+
+ // Gentleman, start your engines
+ for (Multi m : multis) {
+ m.start();
+ }
+
+ // Wait for threads and get results
+ ExampleResult[] multiResults = new ExampleResult[numThreads];
+ for (int i = 0; i < numThreads; i++) {
+ multis[i].join();
+ multiResults[i] = multis[i].getResults();
+ }
+
+ // Each of the (numThreads-1) should have the same predictable
+ // results, which we test for here.
+ for (int i = 0; i < numThreads - 1; i++) {
+ ExampleResult eResults = multiResults[i];
+ TestMultiCLExample.assertCorrectMultiResults(eResults);
+ }
+
+ // The last thread, which only searched over the
+ // DefaultFacetIndexingParams,
+ // has its own results
+ ExampleResult eResults = multiResults[numThreads - 1];
+ List<FacetResult> results = eResults.getFacetResults();
+ assertEquals(3, results.size());
+ String[] expLabels = new String[] { "5", "5/5", "6/2" };
+ double[] expValues = new double[] { 0.0, 0.0, 1.0 };
+ for (int i = 0; i < 3; i++) {
+ FacetResult result = results.get(i);
+ assertNotNull("Result should not be null", result);
+ FacetResultNode resNode = result.getFacetResultNode();
+ assertEquals("Invalid label", expLabels[i], resNode.getLabel().toString());
+ assertEquals("Invalid value", expValues[i], resNode.getValue(), 0.0);
+ assertEquals("Invalid number of subresults", 0, resNode.getNumSubResults());
+ }
+ // we're done, close the index reader and the taxonomy.
+ slowIndexReader.close();
+ slowTaxoReader.close();
+ indexDir.close();
+ taxoDir.close();
+ }
+
+ /**
+ * Simple test to make sure the TotalFacetCountsManager updates the
+ * TotalFacetCounts array only when it is supposed to, and whether it
+ * is recomputed or read from disk.
+ */
+ @Test
+ public void testGenerationalConsistency() throws Exception {
+ // Create temporary RAMDirectories
+ Directory[][] dirs = FacetTestUtils.createIndexTaxonomyDirs(1);
+
+ // Create our index/taxonomy writers
+ IndexTaxonomyWriterPair[] writers = FacetTestUtils.createIndexTaxonomyWriterPair(dirs);
+ DefaultFacetIndexingParams iParams = new DefaultFacetIndexingParams();
+
+ // Add a facet to the index
+ addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "a", "b");
+
+ // Commit Changes
+ writers[0].indexWriter.commit();
+ writers[0].taxWriter.commit();
+
+ // Open readers
+ IndexTaxonomyReaderPair[] readers = FacetTestUtils.createIndexTaxonomyReaderPair(dirs);
+
+ // As this is the first time we have invoked the TotalFacetCountsManager,
+ // we should expect to compute and not read from disk.
+ TotalFacetCounts totalCounts =
+ TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams, null);
+ int prevGen = assertRecomputed(totalCounts, 0, "after first attempt to get it!");
+
+ // Repeating same operation should pull from the cache - not recomputed.
+ assertTrue("Should be obtained from cache at 2nd attempt",totalCounts ==
+ TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams, null));
+
+ // Repeat the same operation as above. but clear first - now should recompute again
+ initCache();
+ totalCounts = TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams, null);
+ prevGen = assertRecomputed(totalCounts, prevGen, "after cache clear, 3rd attempt to get it!");
+
+ //store to file
+ File outputFile = _TestUtil.createTempFile("test", "tmp", TEMP_DIR);
+ initCache();
+ TFC.store(outputFile, readers[0].indexReader, readers[0].taxReader, iParams, null);
+ totalCounts = TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams, null);
+ prevGen = assertRecomputed(totalCounts, prevGen, "after cache clear, 4th attempt to get it!");
+
+ //clear and load
+ initCache();
+ TFC.load(outputFile, readers[0].indexReader, readers[0].taxReader, iParams);
+ totalCounts = TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams, null);
+ prevGen = assertReadFromDisc(totalCounts, prevGen, "after 5th attempt to get it!");
+
+ // Add a new facet to the index, commit and refresh readers
+ addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "c", "d");
+ writers[0].indexWriter.close();
+ writers[0].taxWriter.close();
+
+ readers[0].taxReader.refresh();
+ IndexReader r2 = readers[0].indexReader.reopen();
+ // Hold on to the 'original' reader so we can do some checks with it
+ IndexReader origReader = null;
+
+ assertTrue("Reader must be updated!", readers[0].indexReader != r2);
+
+ // Set the 'original' reader
+ origReader = readers[0].indexReader;
+ // Set the new master index Reader
+ readers[0].indexReader = r2;
+
+ // Try to get total-counts the originalReader AGAIN, just for sanity. Should pull from the cache - not recomputed.
+ assertTrue("Should be obtained from cache at 6th attempt",totalCounts ==
+ TFC.getTotalCounts(origReader, readers[0].taxReader, iParams, null));
+
+ // now use the new reader - should recompute
+ totalCounts = TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams, null);
+ prevGen = assertRecomputed(totalCounts, prevGen, "after updating the index - 7th attempt!");
+
+ // try again - should not recompute
+ assertTrue("Should be obtained from cache at 8th attempt",totalCounts ==
+ TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams, null));
+
+ // delete a doc from the reader and commit - should recompute
+ origReader.close();
+ origReader = readers[0].indexReader;
+ readers[0].indexReader = IndexReader.open(origReader.directory(),false);
+ initCache();
+ totalCounts = TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams, null);
+ prevGen = assertRecomputed(totalCounts, prevGen, "after opening a writable reader - 9th attempt!");
+ // now do the delete
+ readers[0].indexReader.deleteDocument(1);
+ readers[0].indexReader.commit(null);
+ totalCounts = TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams, null);
+ prevGen = assertRecomputed(totalCounts, prevGen, "after deleting docs the index - 10th attempt!");
+
+ origReader.close();
+ readers[0].close();
+ r2.close();
+ outputFile.delete();
+ IOUtils.close(dirs[0]);
+ }
+
+ private int assertReadFromDisc(TotalFacetCounts totalCounts, int prevGen, String errMsg) {
+ assertEquals("should read from disk "+errMsg, CreationType.Loaded, totalCounts.createType4test);
+ int gen4test = totalCounts.gen4test;
+ assertTrue("should read from disk "+errMsg, gen4test > prevGen);
+ return gen4test;
+ }
+
+ private int assertRecomputed(TotalFacetCounts totalCounts, int prevGen, String errMsg) {
+ assertEquals("should recompute "+errMsg, CreationType.Computed, totalCounts.createType4test);
+ int gen4test = totalCounts.gen4test;
+ assertTrue("should recompute "+errMsg, gen4test > prevGen);
+ return gen4test;
+ }
+
+ /**
+ * This test is to address a bug in a previous version. If a TFC cache is
+ * written to disk, and then the taxonomy grows (but the index does not change),
+ * and then the TFC cache is re-read from disk, there will be an exception
+ * thrown, as the integers are read off of the disk according to taxonomy
+ * size, which has changed.
+ */
+ @Test
+ public void testGrowingTaxonomy() throws Exception {
+ // Create temporary RAMDirectories
+ Directory[][] dirs = FacetTestUtils.createIndexTaxonomyDirs(1);
+ // Create our index/taxonomy writers
+ IndexTaxonomyWriterPair[] writers = FacetTestUtils
+ .createIndexTaxonomyWriterPair(dirs);
+ DefaultFacetIndexingParams iParams = new DefaultFacetIndexingParams() {
+ @Override
+ protected int fixedPartitionSize() {
+ return 2;
+ }
+ };
+ // Add a facet to the index
+ addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "a", "b");
+ // Commit Changes
+ writers[0].indexWriter.commit();
+ writers[0].taxWriter.commit();
+
+ IndexTaxonomyReaderPair[] readers = FacetTestUtils.createIndexTaxonomyReaderPair(dirs);
+
+ // Create TFC and write cache to disk
+ File outputFile = _TestUtil.createTempFile("test", "tmp", TEMP_DIR);
+ TFC.store(outputFile, readers[0].indexReader, readers[0].taxReader, iParams, null);
+
+ // Make the taxonomy grow without touching the index
+ for (int i = 0; i < 10; i++) {
+ writers[0].taxWriter.addCategory(new CategoryPath("foo", Integer.toString(i)));
+ }
+ writers[0].taxWriter.commit();
+ readers[0].taxReader.refresh();
+
+ initCache();
+
+ // With the bug, this next call should result in an exception
+ TFC.load(outputFile, readers[0].indexReader, readers[0].taxReader, iParams);
+ TotalFacetCounts totalCounts = TFC.getTotalCounts(
+ readers[0].indexReader, readers[0].taxReader, iParams, null);
+ assertReadFromDisc(totalCounts, 0, "after reading from disk.");
+ outputFile.delete();
+ writers[0].close();
+ readers[0].close();
+ IOUtils.close(dirs[0]);
+ }
+
+ /**
+ * Test that a new TFC is only calculated and placed in memory (by two
+ * threads who want it at the same time) only once.
+ */
+ @Test
+ public void testMemoryCacheSynchronization() throws Exception {
+ SlowRAMDirectory indexDir = new SlowRAMDirectory(-1, null);
+ SlowRAMDirectory taxoDir = new SlowRAMDirectory(-1, null);
+
+ // Write index using 'normal' directories
+ IndexWriter w = new IndexWriter(indexDir, new IndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)));
+ DirectoryTaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir);
+ DefaultFacetIndexingParams iParams = new DefaultFacetIndexingParams();
+ // Add documents and facets
+ for (int i = 0; i < 1000; i++) {
+ addFacets(iParams, w, tw, "facet", Integer.toString(i));
+ }
+ w.close();
+ tw.close();
+
+ indexDir.setSleepMillis(1);
+ taxoDir.setSleepMillis(1);
+
+ IndexReader r = IndexReader.open(indexDir);
+ DirectoryTaxonomyReader tr = new DirectoryTaxonomyReader(taxoDir);
+
+ // Create and start threads. Thread1 should lock the cache and calculate
+ // the TFC array. The second thread should block until the first is
+ // done, then successfully retrieve from the cache without recalculating
+ // or reading from disk.
+ TFCThread tfcCalc1 = new TFCThread(r, tr, iParams);
+ TFCThread tfcCalc2 = new TFCThread(r, tr, iParams);
+ tfcCalc1.start();
+ // Give thread 1 a head start to ensure correct sequencing for testing
+ Thread.sleep(5);
+ tfcCalc2.start();
+
+ tfcCalc1.join();
+ tfcCalc2.join();
+
+ // Since this test ends up with references to the same TFC object, we
+ // can only test the times to make sure that they are the same.
+ assertRecomputed(tfcCalc1.tfc, 0, "thread 1 should recompute");
+ assertRecomputed(tfcCalc2.tfc, 0, "thread 2 should recompute");
+ assertTrue("Both results should be the same (as their inputs are the same objects)",
+ tfcCalc1.tfc == tfcCalc2.tfc);
+
+ r.close();
+ tr.close();
+ }
+
+ /**
+ * Simple test to make sure the TotalFacetCountsManager updates the
+ * TotalFacetCounts array only when it is supposed to, and whether it
+ * is recomputed or read from disk, but this time with TWO different
+ * TotalFacetCounts
+ */
+ @Test
+ public void testMultipleIndices() throws IOException {
+ // Create temporary RAMDirectories
+ Directory[][] dirs = FacetTestUtils.createIndexTaxonomyDirs(2);
+ // Create our index/taxonomy writers
+ IndexTaxonomyWriterPair[] writers = FacetTestUtils.createIndexTaxonomyWriterPair(dirs);
+ DefaultFacetIndexingParams iParams = new DefaultFacetIndexingParams();
+
+ // Add a facet to the index
+ addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "a", "b");
+ addFacets(iParams, writers[1].indexWriter, writers[1].taxWriter, "d", "e");
+ // Commit Changes
+ writers[0].indexWriter.commit();
+ writers[0].taxWriter.commit();
+ writers[1].indexWriter.commit();
+ writers[1].taxWriter.commit();
+
+ // Open two readers
+ IndexTaxonomyReaderPair[] readers = FacetTestUtils.createIndexTaxonomyReaderPair(dirs);
+
+ // As this is the first time we have invoked the TotalFacetCountsManager, we
+ // should expect to compute.
+ TotalFacetCounts totalCounts0 =
+ TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams, null);
+ int prevGen = -1;
+ prevGen = assertRecomputed(totalCounts0, prevGen, "after attempt 1");
+ assertTrue("attempt 1b for same input [0] shout find it in cache",
+ totalCounts0 == TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams, null));
+
+ // 2nd Reader - As this is the first time we have invoked the
+ // TotalFacetCountsManager, we should expect a state of NEW to be returned.
+ TotalFacetCounts totalCounts1 =
+ TFC.getTotalCounts(readers[1].indexReader, readers[1].taxReader, iParams, null);
+ prevGen = assertRecomputed(totalCounts1, prevGen, "after attempt 2");
+ assertTrue("attempt 2b for same input [1] shout find it in cache",
+ totalCounts1 == TFC.getTotalCounts(readers[1].indexReader, readers[1].taxReader, iParams, null));
+
+ // Right now cache size is one, so first TFC is gone and should be recomputed
+ totalCounts0 =
+ TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams, null);
+ prevGen = assertRecomputed(totalCounts0, prevGen, "after attempt 3");
+
+ // Similarly will recompute the second result
+ totalCounts1 =
+ TFC.getTotalCounts(readers[1].indexReader, readers[1].taxReader, iParams, null);
+ prevGen = assertRecomputed(totalCounts1, prevGen, "after attempt 4");
+
+ // Now we set the cache size to two, meaning both should exist in the
+ // cache simultaneously
+ TFC.setCacheSize(2);
+
+ // Re-compute totalCounts0 (was evicted from the cache when the cache was smaller)
+ totalCounts0 =
+ TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams, null);
+ prevGen = assertRecomputed(totalCounts0, prevGen, "after attempt 5");
+
+ // now both are in the larger cache and should not be recomputed
+ totalCounts1 = TFC.getTotalCounts(readers[1].indexReader,
+ readers[1].taxReader, iParams, null);
+ assertTrue("with cache of size 2 res no. 0 should come from cache",
+ totalCounts0 == TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams, null));
+ assertTrue("with cache of size 2 res no. 1 should come from cache",
+ totalCounts1 == TFC.getTotalCounts(readers[1].indexReader, readers[1].taxReader, iParams, null));
+
+ writers[0].close();
+ writers[1].close();
+ readers[0].close();
+ readers[1].close();
+ for (Directory[] dirset : dirs) {
+ IOUtils.close(dirset);
+ }
+ }
+
+}
fnp / pylucene.git / blobdiff