X-Git-Url: https://git.mdrn.pl/pylucene.git/blobdiff_plain/a2e61f0c04805cfcb8706176758d1283c7e3a55c..aaeed5504b982cf3545252ab528713250aa33eed:/lucene-java-3.4.0/lucene/src/test/org/apache/lucene/index/TestSnapshotDeletionPolicy.java diff --git a/lucene-java-3.4.0/lucene/src/test/org/apache/lucene/index/TestSnapshotDeletionPolicy.java b/lucene-java-3.4.0/lucene/src/test/org/apache/lucene/index/TestSnapshotDeletionPolicy.java deleted file mode 100644 index 37bdf23..0000000 --- a/lucene-java-3.4.0/lucene/src/test/org/apache/lucene/index/TestSnapshotDeletionPolicy.java +++ /dev/null @@ -1,445 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.util.Collection; -import java.util.Map; -import java.util.Random; -import java.io.IOException; - -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.analysis.KeywordAnalyzer; -import org.apache.lucene.analysis.MockAnalyzer; -import org.apache.lucene.analysis.standard.StandardAnalyzer; -import org.apache.lucene.index.IndexCommit; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.KeepOnlyLastCommitDeletionPolicy; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.SnapshotDeletionPolicy; -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.ThreadInterruptedException; -import org.junit.Test; - -// -// This was developed for Lucene In Action, -// http://lucenebook.com -// - -public class TestSnapshotDeletionPolicy extends LuceneTestCase { - public static final String INDEX_PATH = "test.snapshots"; - - protected IndexWriterConfig getConfig(Random random, IndexDeletionPolicy dp) { - IndexWriterConfig conf = newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)); - if (dp != null) { - conf.setIndexDeletionPolicy(dp); - } - return conf; - } - - protected void checkSnapshotExists(Directory dir, IndexCommit c) throws Exception { - String segFileName = c.getSegmentsFileName(); - assertTrue("segments file not found in directory: " + segFileName, dir.fileExists(segFileName)); - } - - protected void checkMaxDoc(IndexCommit commit, int expectedMaxDoc) throws Exception { - IndexReader reader = IndexReader.open(commit, true); - try { - assertEquals(expectedMaxDoc, reader.maxDoc()); - } finally { - reader.close(); - } - } - - protected void prepareIndexAndSnapshots(SnapshotDeletionPolicy sdp, - IndexWriter writer, int numSnapshots, String snapshotPrefix) - throws RuntimeException, IOException { - for (int i = 0; i < numSnapshots; i++) { - // create dummy document to trigger commit. - writer.addDocument(new Document()); - writer.commit(); - sdp.snapshot(snapshotPrefix + i); - } - } - - protected SnapshotDeletionPolicy getDeletionPolicy() throws IOException { - return getDeletionPolicy(null); - } - - protected SnapshotDeletionPolicy getDeletionPolicy(Map snapshots) throws IOException { - return new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy(), snapshots); - } - - protected void assertSnapshotExists(Directory dir, SnapshotDeletionPolicy sdp, int numSnapshots) throws Exception { - for (int i = 0; i < numSnapshots; i++) { - IndexCommit snapshot = sdp.getSnapshot("snapshot" + i); - checkMaxDoc(snapshot, i + 1); - checkSnapshotExists(dir, snapshot); - } - } - - @Test - public void testSnapshotDeletionPolicy() throws Exception { - Directory fsDir = newDirectory(); - runTest(random, fsDir); - fsDir.close(); - } - - private void runTest(Random random, Directory dir) throws Exception { - // Run for ~1 seconds - final long stopTime = System.currentTimeMillis() + 1000; - - SnapshotDeletionPolicy dp = getDeletionPolicy(); - final IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( - TEST_VERSION_CURRENT, new MockAnalyzer(random)).setIndexDeletionPolicy(dp) - .setMaxBufferedDocs(2)); - writer.commit(); - - final Thread t = new Thread() { - @Override - public void run() { - Document doc = new Document(); - doc.add(newField("content", "aaa", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS)); - do { - for(int i=0;i<27;i++) { - try { - writer.addDocument(doc); - } catch (Throwable t) { - t.printStackTrace(System.out); - fail("addDocument failed"); - } - if (i%2 == 0) { - try { - writer.commit(); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - } - try { - Thread.sleep(1); - } catch (InterruptedException ie) { - throw new ThreadInterruptedException(ie); - } - } while(System.currentTimeMillis() < stopTime); - } - }; - - t.start(); - - // While the above indexing thread is running, take many - // backups: - do { - backupIndex(dir, dp); - Thread.sleep(20); - } while(t.isAlive()); - - t.join(); - - // Add one more document to force writer to commit a - // final segment, so deletion policy has a chance to - // delete again: - Document doc = new Document(); - doc.add(newField("content", "aaa", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS)); - writer.addDocument(doc); - - // Make sure we don't have any leftover files in the - // directory: - writer.close(); - TestIndexWriter.assertNoUnreferencedFiles(dir, "some files were not deleted but should have been"); - } - - /** - * Example showing how to use the SnapshotDeletionPolicy to take a backup. - * This method does not really do a backup; instead, it reads every byte of - * every file just to test that the files indeed exist and are readable even - * while the index is changing. - */ - public void backupIndex(Directory dir, SnapshotDeletionPolicy dp) throws Exception { - // To backup an index we first take a snapshot: - try { - copyFiles(dir, dp.snapshot("id")); - } finally { - // Make sure to release the snapshot, otherwise these - // files will never be deleted during this IndexWriter - // session: - dp.release("id"); - } - } - - private void copyFiles(Directory dir, IndexCommit cp) throws Exception { - - // While we hold the snapshot, and nomatter how long - // we take to do the backup, the IndexWriter will - // never delete the files in the snapshot: - Collection files = cp.getFileNames(); - for (final String fileName : files) { - // NOTE: in a real backup you would not use - // readFile; you would need to use something else - // that copies the file to a backup location. This - // could even be a spawned shell process (eg "tar", - // "zip") that takes the list of files and builds a - // backup. - readFile(dir, fileName); - } - } - - byte[] buffer = new byte[4096]; - - private void readFile(Directory dir, String name) throws Exception { - IndexInput input = dir.openInput(name); - try { - long size = dir.fileLength(name); - long bytesLeft = size; - while (bytesLeft > 0) { - final int numToRead; - if (bytesLeft < buffer.length) - numToRead = (int) bytesLeft; - else - numToRead = buffer.length; - input.readBytes(buffer, 0, numToRead, false); - bytesLeft -= numToRead; - } - // Don't do this in your real backups! This is just - // to force a backup to take a somewhat long time, to - // make sure we are exercising the fact that the - // IndexWriter should not delete this file even when I - // take my time reading it. - Thread.sleep(1); - } finally { - input.close(); - } - } - - - @Test - public void testBasicSnapshots() throws Exception { - int numSnapshots = 3; - SnapshotDeletionPolicy sdp = getDeletionPolicy(); - - // Create 3 snapshots: snapshot0, snapshot1, snapshot2 - Directory dir = newDirectory(); - IndexWriter writer = new IndexWriter(dir, getConfig(random, sdp)); - prepareIndexAndSnapshots(sdp, writer, numSnapshots, "snapshot"); - writer.close(); - - assertSnapshotExists(dir, sdp, numSnapshots); - - // open a reader on a snapshot - should succeed. - IndexReader.open(sdp.getSnapshot("snapshot0"), true).close(); - - // open a new IndexWriter w/ no snapshots to keep and assert that all snapshots are gone. - sdp = getDeletionPolicy(); - writer = new IndexWriter(dir, getConfig(random, sdp)); - writer.deleteUnusedFiles(); - writer.close(); - assertEquals("no snapshots should exist", 1, IndexReader.listCommits(dir).size()); - - for (int i = 0; i < numSnapshots; i++) { - try { - sdp.getSnapshot("snapshot" + i); - fail("snapshot shouldn't have existed, but did: snapshot" + i); - } catch (IllegalStateException e) { - // expected - snapshot should not exist - } - } - dir.close(); - } - - @Test - public void testMultiThreadedSnapshotting() throws Exception { - Directory dir = newDirectory(); - final SnapshotDeletionPolicy sdp = getDeletionPolicy(); - final IndexWriter writer = new IndexWriter(dir, getConfig(random, sdp)); - - Thread[] threads = new Thread[10]; - for (int i = 0; i < threads.length; i++) { - threads[i] = new Thread() { - @Override - public void run() { - try { - writer.addDocument(new Document()); - writer.commit(); - sdp.snapshot(getName()); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - }; - threads[i].setName("t" + i); - } - - for (Thread t : threads) { - t.start(); - } - - for (Thread t : threads) { - t.join(); - } - - // Do one last commit, so that after we release all snapshots, we stay w/ one commit - writer.addDocument(new Document()); - writer.commit(); - - for (Thread t : threads) { - sdp.release(t.getName()); - writer.deleteUnusedFiles(); - } - assertEquals(1, IndexReader.listCommits(dir).size()); - writer.close(); - dir.close(); - } - - @Test - public void testRollbackToOldSnapshot() throws Exception { - int numSnapshots = 2; - Directory dir = newDirectory(); - SnapshotDeletionPolicy sdp = getDeletionPolicy(); - IndexWriter writer = new IndexWriter(dir, getConfig(random, sdp)); - prepareIndexAndSnapshots(sdp, writer, numSnapshots, "snapshot"); - writer.close(); - - // now open the writer on "snapshot0" - make sure it succeeds - writer = new IndexWriter(dir, getConfig(random, sdp).setIndexCommit(sdp.getSnapshot("snapshot0"))); - // this does the actual rollback - writer.commit(); - writer.deleteUnusedFiles(); - assertSnapshotExists(dir, sdp, numSnapshots - 1); - writer.close(); - - // but 'snapshot1' files will still exist (need to release snapshot before they can be deleted). - String segFileName = sdp.getSnapshot("snapshot1").getSegmentsFileName(); - assertTrue("snapshot files should exist in the directory: " + segFileName, dir.fileExists(segFileName)); - dir.close(); - } - - @Test - public void testReleaseSnapshot() throws Exception { - Directory dir = newDirectory(); - SnapshotDeletionPolicy sdp = getDeletionPolicy(); - IndexWriter writer = new IndexWriter(dir, getConfig(random, sdp)); - prepareIndexAndSnapshots(sdp, writer, 1, "snapshot"); - - // Create another commit - we must do that, because otherwise the "snapshot" - // files will still remain in the index, since it's the last commit. - writer.addDocument(new Document()); - writer.commit(); - - // Release - String snapId = "snapshot0"; - String segFileName = sdp.getSnapshot(snapId).getSegmentsFileName(); - sdp.release(snapId); - try { - sdp.getSnapshot(snapId); - fail("should not have succeeded to get an unsnapshotted id"); - } catch (IllegalStateException e) { - // expected - } - assertNull(sdp.getSnapshots().get(snapId)); - writer.deleteUnusedFiles(); - writer.close(); - assertFalse("segments file should not be found in dirctory: " + segFileName, dir.fileExists(segFileName)); - dir.close(); - } - - @Test - public void testExistingSnapshots() throws Exception { - // Tests the ability to construct a SDP from existing snapshots, and - // asserts that those snapshots/commit points are protected. - int numSnapshots = 3; - Directory dir = newDirectory(); - SnapshotDeletionPolicy sdp = getDeletionPolicy(); - IndexWriter writer = new IndexWriter(dir, getConfig(random, sdp)); - prepareIndexAndSnapshots(sdp, writer, numSnapshots, "snapshot"); - writer.close(); - - // Make a new policy and initialize with snapshots. - sdp = getDeletionPolicy(sdp.getSnapshots()); - writer = new IndexWriter(dir, getConfig(random, sdp)); - // attempt to delete unused files - the snapshotted files should not be deleted - writer.deleteUnusedFiles(); - writer.close(); - assertSnapshotExists(dir, sdp, numSnapshots); - dir.close(); - } - - @Test - public void testSnapshotLastCommitTwice() throws Exception { - Directory dir = newDirectory(); - SnapshotDeletionPolicy sdp = getDeletionPolicy(); - IndexWriter writer = new IndexWriter(dir, getConfig(random, sdp)); - writer.addDocument(new Document()); - writer.commit(); - - String s1 = "s1"; - String s2 = "s2"; - IndexCommit ic1 = sdp.snapshot(s1); - IndexCommit ic2 = sdp.snapshot(s2); - assertTrue(ic1 == ic2); // should be the same instance - - // create another commit - writer.addDocument(new Document()); - writer.commit(); - - // release "s1" should not delete "s2" - sdp.release(s1); - writer.deleteUnusedFiles(); - checkSnapshotExists(dir, ic2); - - writer.close(); - dir.close(); - } - - @Test - public void testMissingCommits() throws Exception { - // Tests the behavior of SDP when commits that are given at ctor are missing - // on onInit(). - Directory dir = newDirectory(); - SnapshotDeletionPolicy sdp = getDeletionPolicy(); - IndexWriter writer = new IndexWriter(dir, getConfig(random, sdp)); - writer.addDocument(new Document()); - writer.commit(); - IndexCommit ic = sdp.snapshot("s1"); - - // create another commit, not snapshotted. - writer.addDocument(new Document()); - writer.close(); - - // open a new writer w/ KeepOnlyLastCommit policy, so it will delete "s1" - // commit. - new IndexWriter(dir, getConfig(random, null)).close(); - - assertFalse("snapshotted commit should not exist", dir.fileExists(ic.getSegmentsFileName())); - - // Now reinit SDP from the commits in the index - the snapshot id should not - // exist anymore. - sdp = getDeletionPolicy(sdp.getSnapshots()); - new IndexWriter(dir, getConfig(random, sdp)).close(); - - try { - sdp.getSnapshot("s1"); - fail("snapshot s1 should not exist"); - } catch (IllegalStateException e) { - // expected. - } - dir.close(); - } - -}